|
| 1 | +"""Fast tests for checkpoint deletion API behavior.""" |
| 2 | + |
| 3 | +import asyncio |
| 4 | +from collections.abc import AsyncGenerator, Iterator |
| 5 | +from datetime import datetime, timezone |
| 6 | +from pathlib import Path |
| 7 | + |
| 8 | +import pytest |
| 9 | +from fastapi.testclient import TestClient |
| 10 | +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine |
| 11 | +from sqlmodel import SQLModel |
| 12 | +from sqlmodel.ext.asyncio.session import AsyncSession |
| 13 | + |
| 14 | +from skyrl.tinker import types |
| 15 | +from skyrl.tinker.api import app, get_session |
| 16 | +from skyrl.tinker.config import EngineConfig |
| 17 | +from skyrl.tinker.db_models import CheckpointDB, CheckpointStatus, ModelDB, SessionDB |
| 18 | + |
| 19 | +MODEL_ID = "model_fast_delete" |
| 20 | + |
| 21 | + |
| 22 | +@pytest.fixture |
| 23 | +def checkpoint_api(tmp_path: Path) -> Iterator[tuple[TestClient, Path, AsyncEngine]]: |
| 24 | + db_path = tmp_path / "checkpoint_delete.db" |
| 25 | + checkpoint_base = tmp_path / "checkpoints" |
| 26 | + db_engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}") |
| 27 | + |
| 28 | + async def setup_db() -> None: |
| 29 | + async with db_engine.begin() as conn: |
| 30 | + await conn.run_sync(SQLModel.metadata.create_all) |
| 31 | + |
| 32 | + async def override_get_session() -> AsyncGenerator[AsyncSession, None]: |
| 33 | + async with AsyncSession(db_engine) as session: |
| 34 | + yield session |
| 35 | + |
| 36 | + asyncio.run(setup_db()) |
| 37 | + previous_state = app.state._state.copy() |
| 38 | + app.state.db_engine = db_engine |
| 39 | + app.state.engine_config = EngineConfig( |
| 40 | + base_model="test-model", |
| 41 | + checkpoints_base=checkpoint_base, |
| 42 | + database_url=f"sqlite:///{db_path}", |
| 43 | + ) |
| 44 | + app.dependency_overrides[get_session] = override_get_session |
| 45 | + client = TestClient(app) |
| 46 | + |
| 47 | + try: |
| 48 | + yield client, checkpoint_base, db_engine |
| 49 | + finally: |
| 50 | + client.close() |
| 51 | + app.dependency_overrides.pop(get_session, None) |
| 52 | + app.state._state.clear() |
| 53 | + app.state._state.update(previous_state) |
| 54 | + asyncio.run(db_engine.dispose()) |
| 55 | + |
| 56 | + |
| 57 | +async def seed_model_and_checkpoints( |
| 58 | + db_engine: AsyncEngine, |
| 59 | + checkpoint_ids: list[str], |
| 60 | + checkpoint_type: types.CheckpointType = types.CheckpointType.TRAINING, |
| 61 | +) -> None: |
| 62 | + async with AsyncSession(db_engine) as session: |
| 63 | + if await session.get(SessionDB, "session_fast_delete") is None: |
| 64 | + session.add(SessionDB(session_id="session_fast_delete", tags=[], sdk_version="test")) |
| 65 | + if await session.get(ModelDB, MODEL_ID) is None: |
| 66 | + session.add( |
| 67 | + ModelDB( |
| 68 | + model_id=MODEL_ID, |
| 69 | + base_model="test-model", |
| 70 | + lora_config={"rank": 1}, |
| 71 | + status="created", |
| 72 | + request_id=1, |
| 73 | + session_id="session_fast_delete", |
| 74 | + ) |
| 75 | + ) |
| 76 | + for checkpoint_id in checkpoint_ids: |
| 77 | + session.add( |
| 78 | + CheckpointDB( |
| 79 | + model_id=MODEL_ID, |
| 80 | + checkpoint_id=checkpoint_id, |
| 81 | + checkpoint_type=checkpoint_type, |
| 82 | + status=CheckpointStatus.COMPLETED, |
| 83 | + completed_at=datetime.now(timezone.utc), |
| 84 | + ) |
| 85 | + ) |
| 86 | + await session.commit() |
| 87 | + |
| 88 | + |
| 89 | +def write_training_checkpoint(checkpoint_base: Path, checkpoint_id: str, directory: bool = False) -> Path: |
| 90 | + checkpoint_path = checkpoint_base / MODEL_ID / f"{checkpoint_id}.tar.gz" |
| 91 | + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) |
| 92 | + if directory: |
| 93 | + checkpoint_path.mkdir() |
| 94 | + (checkpoint_path / "checkpoint_0").write_text("tiny") |
| 95 | + else: |
| 96 | + checkpoint_path.write_text("tiny") |
| 97 | + return checkpoint_path |
| 98 | + |
| 99 | + |
| 100 | +def write_sampler_checkpoint(checkpoint_base: Path, checkpoint_id: str) -> Path: |
| 101 | + checkpoint_path = checkpoint_base / MODEL_ID / "sampler_weights" / f"{checkpoint_id}.tar.gz" |
| 102 | + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) |
| 103 | + checkpoint_path.write_text("tiny") |
| 104 | + return checkpoint_path |
| 105 | + |
| 106 | + |
| 107 | +def listed_checkpoint_ids(client: TestClient) -> set[str]: |
| 108 | + response = client.get(f"/api/v1/training_runs/{MODEL_ID}/checkpoints") |
| 109 | + assert response.status_code == 200 |
| 110 | + return {checkpoint["checkpoint_id"] for checkpoint in response.json()["checkpoints"]} |
| 111 | + |
| 112 | + |
| 113 | +def test_delete_checkpoint_removes_saved_artifact_and_list_entry( |
| 114 | + checkpoint_api: tuple[TestClient, Path, AsyncEngine], |
| 115 | +) -> None: |
| 116 | + client, checkpoint_base, db_engine = checkpoint_api |
| 117 | + asyncio.run(seed_model_and_checkpoints(db_engine, ["delete_training"])) |
| 118 | + training_checkpoint = write_training_checkpoint(checkpoint_base, "delete_training", directory=True) |
| 119 | + |
| 120 | + response = client.delete(f"/api/v1/training_runs/{MODEL_ID}/checkpoints/weights/delete_training") |
| 121 | + |
| 122 | + assert response.status_code == 204 |
| 123 | + assert not training_checkpoint.exists() |
| 124 | + assert "delete_training" not in listed_checkpoint_ids(client) |
| 125 | + |
| 126 | + asyncio.run( |
| 127 | + seed_model_and_checkpoints(db_engine, ["delete_sampler"], checkpoint_type=types.CheckpointType.SAMPLER) |
| 128 | + ) |
| 129 | + sampler_checkpoint = write_sampler_checkpoint(checkpoint_base, "delete_sampler") |
| 130 | + |
| 131 | + response = client.delete(f"/api/v1/training_runs/{MODEL_ID}/checkpoints/delete_sampler") |
| 132 | + |
| 133 | + assert response.status_code == 204 |
| 134 | + assert not sampler_checkpoint.exists() |
| 135 | + assert "delete_sampler" not in listed_checkpoint_ids(client) |
| 136 | + |
| 137 | + |
| 138 | +def test_delete_even_checkpoints_leaves_odd_checkpoints_listed( |
| 139 | + checkpoint_api: tuple[TestClient, Path, AsyncEngine], |
| 140 | +) -> None: |
| 141 | + client, checkpoint_base, db_engine = checkpoint_api |
| 142 | + checkpoint_ids = ["1", "2", "3", "4", "5"] |
| 143 | + asyncio.run(seed_model_and_checkpoints(db_engine, checkpoint_ids)) |
| 144 | + checkpoint_files = { |
| 145 | + checkpoint_id: write_training_checkpoint(checkpoint_base, checkpoint_id) for checkpoint_id in checkpoint_ids |
| 146 | + } |
| 147 | + |
| 148 | + assert listed_checkpoint_ids(client) == {"1", "2", "3", "4", "5"} |
| 149 | + |
| 150 | + assert client.delete(f"/api/v1/training_runs/{MODEL_ID}/checkpoints/2").status_code == 204 |
| 151 | + assert client.delete(f"/api/v1/training_runs/{MODEL_ID}/checkpoints/4").status_code == 204 |
| 152 | + |
| 153 | + assert checkpoint_files["1"].exists() |
| 154 | + assert not checkpoint_files["2"].exists() |
| 155 | + assert checkpoint_files["3"].exists() |
| 156 | + assert not checkpoint_files["4"].exists() |
| 157 | + assert checkpoint_files["5"].exists() |
| 158 | + assert listed_checkpoint_ids(client) == {"1", "3", "5"} |
0 commit comments