Initial import: Music_Server, MusicFree, catalog-sync
This commit is contained in:
@@ -0,0 +1,354 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class OpsDatabaseSchemaTests(unittest.TestCase):
|
||||
def test_initialize_database_creates_operations_tables(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
|
||||
expected_tables = {
|
||||
"job_runs",
|
||||
"job_stages",
|
||||
"job_items",
|
||||
"job_workers",
|
||||
"job_commands",
|
||||
"job_events",
|
||||
"job_logs",
|
||||
"config_revisions",
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
conn = initialize_database(db_path)
|
||||
conn.close()
|
||||
|
||||
verify_conn = initialize_database(db_path)
|
||||
table_rows = verify_conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
verify_conn.close()
|
||||
|
||||
tables = {row["name"] for row in table_rows}
|
||||
self.assertTrue(expected_tables.issubset(tables))
|
||||
|
||||
def test_initialize_database_operations_core_columns_match_baseline(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
|
||||
expected_job_runs = {
|
||||
"id",
|
||||
"job_type",
|
||||
"status",
|
||||
"priority",
|
||||
"requested_by",
|
||||
"config_snapshot_json",
|
||||
"sources",
|
||||
"download_sources",
|
||||
"playlist_scope_json",
|
||||
"created_at",
|
||||
"started_at",
|
||||
"ended_at",
|
||||
"last_error",
|
||||
"resume_token",
|
||||
}
|
||||
expected_job_stages = {
|
||||
"id",
|
||||
"job_run_id",
|
||||
"stage_type",
|
||||
"status",
|
||||
"seq_no",
|
||||
"total_items",
|
||||
"pending_items",
|
||||
"running_items",
|
||||
"success_items",
|
||||
"failed_items",
|
||||
"skipped_items",
|
||||
"started_at",
|
||||
"ended_at",
|
||||
"last_error",
|
||||
}
|
||||
expected_job_items = {
|
||||
"id",
|
||||
"job_stage_id",
|
||||
"item_type",
|
||||
"item_key",
|
||||
"playlist_pool_id",
|
||||
"playlist_id",
|
||||
"song_id",
|
||||
"file_location_id",
|
||||
"status",
|
||||
"attempt_count",
|
||||
"max_attempts",
|
||||
"worker_id",
|
||||
"started_at",
|
||||
"ended_at",
|
||||
"last_error",
|
||||
"last_error_code",
|
||||
"payload_json",
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
conn = initialize_database(db_path)
|
||||
job_runs_cols = {
|
||||
row["name"] for row in conn.execute("PRAGMA table_info(job_runs)").fetchall()
|
||||
}
|
||||
job_stages_cols = {
|
||||
row["name"] for row in conn.execute("PRAGMA table_info(job_stages)").fetchall()
|
||||
}
|
||||
job_items_cols = {
|
||||
row["name"] for row in conn.execute("PRAGMA table_info(job_items)").fetchall()
|
||||
}
|
||||
conn.close()
|
||||
|
||||
self.assertTrue(expected_job_runs.issubset(job_runs_cols))
|
||||
self.assertTrue(expected_job_stages.issubset(job_stages_cols))
|
||||
self.assertTrue(expected_job_items.issubset(job_items_cols))
|
||||
|
||||
def test_initialize_database_config_revisions_source_type_has_default(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
conn = initialize_database(db_path)
|
||||
columns = conn.execute("PRAGMA table_info(config_revisions)").fetchall()
|
||||
conn.close()
|
||||
|
||||
source_type_col = next(row for row in columns if row["name"] == "source_type")
|
||||
self.assertEqual("'env_file'", source_type_col["dflt_value"])
|
||||
|
||||
|
||||
class OpsRepositoryTests(unittest.TestCase):
|
||||
def test_create_job_requires_config_snapshot(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
from musicdl.catalogsync.ops.repository import OpsRepository
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
initialize_database(db_path).close()
|
||||
repo = OpsRepository(db_path)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
repo.create_job(job_type="catalog_sync", config_snapshot=None)
|
||||
|
||||
def test_create_and_read_job_stage_item_with_default_status(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
from musicdl.catalogsync.ops.models import ItemStatus, JobStatus, StageStatus
|
||||
from musicdl.catalogsync.ops.repository import OpsRepository
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
initialize_database(db_path).close()
|
||||
repo = OpsRepository(db_path)
|
||||
|
||||
job_id = repo.create_job(
|
||||
job_type="catalog_sync",
|
||||
config_snapshot={
|
||||
"sources": ["qq", "netease"],
|
||||
"download": {"enabled": True},
|
||||
},
|
||||
sources=["qq", "netease"],
|
||||
download_sources=["qq"],
|
||||
)
|
||||
stage_id = repo.create_stage(job_run_id=job_id, stage_type="collect", seq_no=1)
|
||||
item_id = repo.create_item(
|
||||
job_stage_id=stage_id,
|
||||
item_type="song_sync",
|
||||
item_key="qq:playlist:123",
|
||||
song_id=12345,
|
||||
payload={"platform": "qq"},
|
||||
)
|
||||
|
||||
job = repo.get_job(job_id)
|
||||
stage = repo.get_stage(stage_id)
|
||||
item = repo.get_item(item_id)
|
||||
|
||||
self.assertIsNotNone(job)
|
||||
self.assertEqual(JobStatus.QUEUED, job.status)
|
||||
self.assertEqual(100, job.priority)
|
||||
self.assertEqual({"sources": ["qq", "netease"], "download": {"enabled": True}}, job.config_snapshot)
|
||||
self.assertEqual(["qq", "netease"], job.sources)
|
||||
self.assertEqual(["qq"], job.download_sources)
|
||||
|
||||
self.assertIsNotNone(stage)
|
||||
self.assertEqual(StageStatus.PENDING, stage.status)
|
||||
self.assertEqual(job_id, stage.job_run_id)
|
||||
self.assertEqual(1, stage.total_items)
|
||||
self.assertEqual(1, stage.pending_items)
|
||||
self.assertEqual(0, stage.running_items)
|
||||
self.assertEqual(0, stage.success_items)
|
||||
self.assertEqual(1, stage.seq_no)
|
||||
|
||||
self.assertIsNotNone(item)
|
||||
self.assertEqual(ItemStatus.PENDING, item.status)
|
||||
self.assertEqual(stage_id, item.job_stage_id)
|
||||
self.assertEqual(0, item.attempt_count)
|
||||
self.assertEqual(3, item.max_attempts)
|
||||
self.assertEqual("song_sync", item.item_type)
|
||||
self.assertEqual(12345, item.song_id)
|
||||
self.assertEqual("qq", item.payload["platform"])
|
||||
|
||||
def test_create_job_allows_empty_config_snapshot_and_reads_back(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
from musicdl.catalogsync.ops.repository import OpsRepository
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
initialize_database(db_path).close()
|
||||
repo = OpsRepository(db_path)
|
||||
|
||||
job_id = repo.create_job(
|
||||
job_type="catalog_sync",
|
||||
config_snapshot={},
|
||||
sources=["qq", "netease"],
|
||||
download_sources=["qq"],
|
||||
)
|
||||
job = repo.get_job(job_id)
|
||||
|
||||
self.assertEqual({}, job.config_snapshot)
|
||||
self.assertEqual(["qq", "netease"], job.sources)
|
||||
self.assertEqual(["qq"], job.download_sources)
|
||||
|
||||
def test_create_item_with_failed_status_updates_stage_aggregate_counts(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
from musicdl.catalogsync.ops.models import ItemStatus
|
||||
from musicdl.catalogsync.ops.repository import OpsRepository
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
db_path = Path(tmpdir) / "catalogsync.db"
|
||||
initialize_database(db_path).close()
|
||||
repo = OpsRepository(db_path)
|
||||
|
||||
job_id = repo.create_job(job_type="catalog_sync", config_snapshot={})
|
||||
stage_id = repo.create_stage(job_run_id=job_id, stage_type="collect", seq_no=1)
|
||||
repo.create_item(
|
||||
job_stage_id=stage_id,
|
||||
item_type="song_sync",
|
||||
item_key="qq:song:failed:1",
|
||||
status=ItemStatus.FAILED,
|
||||
)
|
||||
stage = repo.get_stage(stage_id)
|
||||
|
||||
self.assertEqual(1, stage.total_items)
|
||||
self.assertEqual(0, stage.pending_items)
|
||||
self.assertEqual(1, stage.failed_items)
|
||||
|
||||
def test_list_job_playlist_progress_aggregates_download_states_per_playlist(self):
|
||||
from musicdl.catalogsync.db import initialize_database
|
||||
from musicdl.catalogsync.models import CatalogSong, PlaylistCandidate
|
||||
from musicdl.catalogsync.ops.models import ItemStatus
|
||||
from musicdl.catalogsync.ops.repository import OpsRepository
|
||||
from musicdl.catalogsync.repository import CatalogRepository
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
db_path = root / "catalogsync.db"
|
||||
library_root = root / "library"
|
||||
initialize_database(db_path, default_library_root=library_root).close()
|
||||
catalog_repo = CatalogRepository(db_path)
|
||||
ops_repo = OpsRepository(db_path)
|
||||
|
||||
playlist_id = catalog_repo.upsert_playlist(
|
||||
PlaylistCandidate(
|
||||
platform="qq",
|
||||
pool_kind="manual_file",
|
||||
remote_id="job-progress-1",
|
||||
name="Job Progress Playlist",
|
||||
url="https://example.invalid/qq/job-progress-1",
|
||||
)
|
||||
)
|
||||
pool_id = catalog_repo.upsert_playlist_pool(
|
||||
platform="qq",
|
||||
pool_kind="manual_file",
|
||||
external_id="manual_file:job-progress-1",
|
||||
name="Manual Pool",
|
||||
url="https://example.invalid/pool/job-progress-1",
|
||||
)
|
||||
catalog_repo.link_pool_playlist(pool_id, playlist_id)
|
||||
|
||||
downloaded_song_id = catalog_repo.upsert_song(
|
||||
CatalogSong(
|
||||
platform="qq",
|
||||
remote_song_id="job-song-1",
|
||||
name="Downloaded Song",
|
||||
singers="Singer A",
|
||||
ext="mp3",
|
||||
file_size_bytes=128,
|
||||
quality_label="standard",
|
||||
)
|
||||
)
|
||||
running_song_id = catalog_repo.upsert_song(
|
||||
CatalogSong(
|
||||
platform="qq",
|
||||
remote_song_id="job-song-2",
|
||||
name="Running Song",
|
||||
singers="Singer A",
|
||||
ext="mp3",
|
||||
file_size_bytes=128,
|
||||
quality_label="standard",
|
||||
)
|
||||
)
|
||||
failed_song_id = catalog_repo.upsert_song(
|
||||
CatalogSong(
|
||||
platform="qq",
|
||||
remote_song_id="job-song-3",
|
||||
name="Failed Song",
|
||||
singers="Singer A",
|
||||
ext="mp3",
|
||||
file_size_bytes=128,
|
||||
quality_label="standard",
|
||||
)
|
||||
)
|
||||
|
||||
catalog_repo.link_playlist_song(playlist_id, downloaded_song_id, 1)
|
||||
catalog_repo.link_playlist_song(playlist_id, running_song_id, 2)
|
||||
catalog_repo.link_playlist_song(playlist_id, failed_song_id, 3)
|
||||
|
||||
backend_id = catalog_repo.get_default_backend_id()
|
||||
catalog_repo.record_local_file(
|
||||
song_id=downloaded_song_id,
|
||||
backend_id=backend_id,
|
||||
relative_path="qq/Singer A/job-song-1.mp3",
|
||||
file_size_bytes=128,
|
||||
ext="mp3",
|
||||
quality_label="standard",
|
||||
)
|
||||
|
||||
job_id = ops_repo.create_job(
|
||||
job_type="download_only",
|
||||
config_snapshot={},
|
||||
playlist_scope={"playlist_ids": [playlist_id]},
|
||||
)
|
||||
stage_id = ops_repo.create_stage(job_run_id=job_id, stage_type="download", seq_no=1)
|
||||
ops_repo.create_item(
|
||||
job_stage_id=stage_id,
|
||||
item_type="song_download",
|
||||
item_key=f"song:{running_song_id}",
|
||||
song_id=running_song_id,
|
||||
status=ItemStatus.RUNNING,
|
||||
)
|
||||
ops_repo.create_item(
|
||||
job_stage_id=stage_id,
|
||||
item_type="song_download",
|
||||
item_key=f"song:{failed_song_id}",
|
||||
song_id=failed_song_id,
|
||||
status=ItemStatus.FAILED,
|
||||
)
|
||||
|
||||
rows = ops_repo.list_job_playlist_progress(job_id)
|
||||
|
||||
self.assertEqual(1, len(rows))
|
||||
row = rows[0]
|
||||
self.assertEqual(playlist_id, row["playlist_id"])
|
||||
self.assertEqual("Job Progress Playlist", row["playlist_name"])
|
||||
self.assertEqual(3, row["total_songs"])
|
||||
self.assertEqual(1, row["downloaded_songs"])
|
||||
self.assertEqual(1, row["running_songs"])
|
||||
self.assertEqual(0, row["pending_songs"])
|
||||
self.assertEqual(1, row["failed_songs"])
|
||||
self.assertEqual(0, row["skipped_songs"])
|
||||
self.assertEqual(33, row["progress_percent"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user