355 lines
13 KiB
Python
355 lines
13 KiB
Python
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
class OpsDatabaseSchemaTests(unittest.TestCase):
|
|
def test_initialize_database_creates_operations_tables(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
|
|
expected_tables = {
|
|
"job_runs",
|
|
"job_stages",
|
|
"job_items",
|
|
"job_workers",
|
|
"job_commands",
|
|
"job_events",
|
|
"job_logs",
|
|
"config_revisions",
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
conn = initialize_database(db_path)
|
|
conn.close()
|
|
|
|
verify_conn = initialize_database(db_path)
|
|
table_rows = verify_conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
|
).fetchall()
|
|
verify_conn.close()
|
|
|
|
tables = {row["name"] for row in table_rows}
|
|
self.assertTrue(expected_tables.issubset(tables))
|
|
|
|
def test_initialize_database_operations_core_columns_match_baseline(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
|
|
expected_job_runs = {
|
|
"id",
|
|
"job_type",
|
|
"status",
|
|
"priority",
|
|
"requested_by",
|
|
"config_snapshot_json",
|
|
"sources",
|
|
"download_sources",
|
|
"playlist_scope_json",
|
|
"created_at",
|
|
"started_at",
|
|
"ended_at",
|
|
"last_error",
|
|
"resume_token",
|
|
}
|
|
expected_job_stages = {
|
|
"id",
|
|
"job_run_id",
|
|
"stage_type",
|
|
"status",
|
|
"seq_no",
|
|
"total_items",
|
|
"pending_items",
|
|
"running_items",
|
|
"success_items",
|
|
"failed_items",
|
|
"skipped_items",
|
|
"started_at",
|
|
"ended_at",
|
|
"last_error",
|
|
}
|
|
expected_job_items = {
|
|
"id",
|
|
"job_stage_id",
|
|
"item_type",
|
|
"item_key",
|
|
"playlist_pool_id",
|
|
"playlist_id",
|
|
"song_id",
|
|
"file_location_id",
|
|
"status",
|
|
"attempt_count",
|
|
"max_attempts",
|
|
"worker_id",
|
|
"started_at",
|
|
"ended_at",
|
|
"last_error",
|
|
"last_error_code",
|
|
"payload_json",
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
conn = initialize_database(db_path)
|
|
job_runs_cols = {
|
|
row["name"] for row in conn.execute("PRAGMA table_info(job_runs)").fetchall()
|
|
}
|
|
job_stages_cols = {
|
|
row["name"] for row in conn.execute("PRAGMA table_info(job_stages)").fetchall()
|
|
}
|
|
job_items_cols = {
|
|
row["name"] for row in conn.execute("PRAGMA table_info(job_items)").fetchall()
|
|
}
|
|
conn.close()
|
|
|
|
self.assertTrue(expected_job_runs.issubset(job_runs_cols))
|
|
self.assertTrue(expected_job_stages.issubset(job_stages_cols))
|
|
self.assertTrue(expected_job_items.issubset(job_items_cols))
|
|
|
|
def test_initialize_database_config_revisions_source_type_has_default(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
conn = initialize_database(db_path)
|
|
columns = conn.execute("PRAGMA table_info(config_revisions)").fetchall()
|
|
conn.close()
|
|
|
|
source_type_col = next(row for row in columns if row["name"] == "source_type")
|
|
self.assertEqual("'env_file'", source_type_col["dflt_value"])
|
|
|
|
|
|
class OpsRepositoryTests(unittest.TestCase):
|
|
def test_create_job_requires_config_snapshot(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
from musicdl.catalogsync.ops.repository import OpsRepository
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
initialize_database(db_path).close()
|
|
repo = OpsRepository(db_path)
|
|
|
|
with self.assertRaises(ValueError):
|
|
repo.create_job(job_type="catalog_sync", config_snapshot=None)
|
|
|
|
def test_create_and_read_job_stage_item_with_default_status(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
from musicdl.catalogsync.ops.models import ItemStatus, JobStatus, StageStatus
|
|
from musicdl.catalogsync.ops.repository import OpsRepository
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
initialize_database(db_path).close()
|
|
repo = OpsRepository(db_path)
|
|
|
|
job_id = repo.create_job(
|
|
job_type="catalog_sync",
|
|
config_snapshot={
|
|
"sources": ["qq", "netease"],
|
|
"download": {"enabled": True},
|
|
},
|
|
sources=["qq", "netease"],
|
|
download_sources=["qq"],
|
|
)
|
|
stage_id = repo.create_stage(job_run_id=job_id, stage_type="collect", seq_no=1)
|
|
item_id = repo.create_item(
|
|
job_stage_id=stage_id,
|
|
item_type="song_sync",
|
|
item_key="qq:playlist:123",
|
|
song_id=12345,
|
|
payload={"platform": "qq"},
|
|
)
|
|
|
|
job = repo.get_job(job_id)
|
|
stage = repo.get_stage(stage_id)
|
|
item = repo.get_item(item_id)
|
|
|
|
self.assertIsNotNone(job)
|
|
self.assertEqual(JobStatus.QUEUED, job.status)
|
|
self.assertEqual(100, job.priority)
|
|
self.assertEqual({"sources": ["qq", "netease"], "download": {"enabled": True}}, job.config_snapshot)
|
|
self.assertEqual(["qq", "netease"], job.sources)
|
|
self.assertEqual(["qq"], job.download_sources)
|
|
|
|
self.assertIsNotNone(stage)
|
|
self.assertEqual(StageStatus.PENDING, stage.status)
|
|
self.assertEqual(job_id, stage.job_run_id)
|
|
self.assertEqual(1, stage.total_items)
|
|
self.assertEqual(1, stage.pending_items)
|
|
self.assertEqual(0, stage.running_items)
|
|
self.assertEqual(0, stage.success_items)
|
|
self.assertEqual(1, stage.seq_no)
|
|
|
|
self.assertIsNotNone(item)
|
|
self.assertEqual(ItemStatus.PENDING, item.status)
|
|
self.assertEqual(stage_id, item.job_stage_id)
|
|
self.assertEqual(0, item.attempt_count)
|
|
self.assertEqual(3, item.max_attempts)
|
|
self.assertEqual("song_sync", item.item_type)
|
|
self.assertEqual(12345, item.song_id)
|
|
self.assertEqual("qq", item.payload["platform"])
|
|
|
|
def test_create_job_allows_empty_config_snapshot_and_reads_back(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
from musicdl.catalogsync.ops.repository import OpsRepository
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
initialize_database(db_path).close()
|
|
repo = OpsRepository(db_path)
|
|
|
|
job_id = repo.create_job(
|
|
job_type="catalog_sync",
|
|
config_snapshot={},
|
|
sources=["qq", "netease"],
|
|
download_sources=["qq"],
|
|
)
|
|
job = repo.get_job(job_id)
|
|
|
|
self.assertEqual({}, job.config_snapshot)
|
|
self.assertEqual(["qq", "netease"], job.sources)
|
|
self.assertEqual(["qq"], job.download_sources)
|
|
|
|
def test_create_item_with_failed_status_updates_stage_aggregate_counts(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
from musicdl.catalogsync.ops.models import ItemStatus
|
|
from musicdl.catalogsync.ops.repository import OpsRepository
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
db_path = Path(tmpdir) / "catalogsync.db"
|
|
initialize_database(db_path).close()
|
|
repo = OpsRepository(db_path)
|
|
|
|
job_id = repo.create_job(job_type="catalog_sync", config_snapshot={})
|
|
stage_id = repo.create_stage(job_run_id=job_id, stage_type="collect", seq_no=1)
|
|
repo.create_item(
|
|
job_stage_id=stage_id,
|
|
item_type="song_sync",
|
|
item_key="qq:song:failed:1",
|
|
status=ItemStatus.FAILED,
|
|
)
|
|
stage = repo.get_stage(stage_id)
|
|
|
|
self.assertEqual(1, stage.total_items)
|
|
self.assertEqual(0, stage.pending_items)
|
|
self.assertEqual(1, stage.failed_items)
|
|
|
|
def test_list_job_playlist_progress_aggregates_download_states_per_playlist(self):
|
|
from musicdl.catalogsync.db import initialize_database
|
|
from musicdl.catalogsync.models import CatalogSong, PlaylistCandidate
|
|
from musicdl.catalogsync.ops.models import ItemStatus
|
|
from musicdl.catalogsync.ops.repository import OpsRepository
|
|
from musicdl.catalogsync.repository import CatalogRepository
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
root = Path(tmpdir)
|
|
db_path = root / "catalogsync.db"
|
|
library_root = root / "library"
|
|
initialize_database(db_path, default_library_root=library_root).close()
|
|
catalog_repo = CatalogRepository(db_path)
|
|
ops_repo = OpsRepository(db_path)
|
|
|
|
playlist_id = catalog_repo.upsert_playlist(
|
|
PlaylistCandidate(
|
|
platform="qq",
|
|
pool_kind="manual_file",
|
|
remote_id="job-progress-1",
|
|
name="Job Progress Playlist",
|
|
url="https://example.invalid/qq/job-progress-1",
|
|
)
|
|
)
|
|
pool_id = catalog_repo.upsert_playlist_pool(
|
|
platform="qq",
|
|
pool_kind="manual_file",
|
|
external_id="manual_file:job-progress-1",
|
|
name="Manual Pool",
|
|
url="https://example.invalid/pool/job-progress-1",
|
|
)
|
|
catalog_repo.link_pool_playlist(pool_id, playlist_id)
|
|
|
|
downloaded_song_id = catalog_repo.upsert_song(
|
|
CatalogSong(
|
|
platform="qq",
|
|
remote_song_id="job-song-1",
|
|
name="Downloaded Song",
|
|
singers="Singer A",
|
|
ext="mp3",
|
|
file_size_bytes=128,
|
|
quality_label="standard",
|
|
)
|
|
)
|
|
running_song_id = catalog_repo.upsert_song(
|
|
CatalogSong(
|
|
platform="qq",
|
|
remote_song_id="job-song-2",
|
|
name="Running Song",
|
|
singers="Singer A",
|
|
ext="mp3",
|
|
file_size_bytes=128,
|
|
quality_label="standard",
|
|
)
|
|
)
|
|
failed_song_id = catalog_repo.upsert_song(
|
|
CatalogSong(
|
|
platform="qq",
|
|
remote_song_id="job-song-3",
|
|
name="Failed Song",
|
|
singers="Singer A",
|
|
ext="mp3",
|
|
file_size_bytes=128,
|
|
quality_label="standard",
|
|
)
|
|
)
|
|
|
|
catalog_repo.link_playlist_song(playlist_id, downloaded_song_id, 1)
|
|
catalog_repo.link_playlist_song(playlist_id, running_song_id, 2)
|
|
catalog_repo.link_playlist_song(playlist_id, failed_song_id, 3)
|
|
|
|
backend_id = catalog_repo.get_default_backend_id()
|
|
catalog_repo.record_local_file(
|
|
song_id=downloaded_song_id,
|
|
backend_id=backend_id,
|
|
relative_path="qq/Singer A/job-song-1.mp3",
|
|
file_size_bytes=128,
|
|
ext="mp3",
|
|
quality_label="standard",
|
|
)
|
|
|
|
job_id = ops_repo.create_job(
|
|
job_type="download_only",
|
|
config_snapshot={},
|
|
playlist_scope={"playlist_ids": [playlist_id]},
|
|
)
|
|
stage_id = ops_repo.create_stage(job_run_id=job_id, stage_type="download", seq_no=1)
|
|
ops_repo.create_item(
|
|
job_stage_id=stage_id,
|
|
item_type="song_download",
|
|
item_key=f"song:{running_song_id}",
|
|
song_id=running_song_id,
|
|
status=ItemStatus.RUNNING,
|
|
)
|
|
ops_repo.create_item(
|
|
job_stage_id=stage_id,
|
|
item_type="song_download",
|
|
item_key=f"song:{failed_song_id}",
|
|
song_id=failed_song_id,
|
|
status=ItemStatus.FAILED,
|
|
)
|
|
|
|
rows = ops_repo.list_job_playlist_progress(job_id)
|
|
|
|
self.assertEqual(1, len(rows))
|
|
row = rows[0]
|
|
self.assertEqual(playlist_id, row["playlist_id"])
|
|
self.assertEqual("Job Progress Playlist", row["playlist_name"])
|
|
self.assertEqual(3, row["total_songs"])
|
|
self.assertEqual(1, row["downloaded_songs"])
|
|
self.assertEqual(1, row["running_songs"])
|
|
self.assertEqual(0, row["pending_songs"])
|
|
self.assertEqual(1, row["failed_songs"])
|
|
self.assertEqual(0, row["skipped_songs"])
|
|
self.assertEqual(33, row["progress_percent"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|