Initial import: Music_Server, MusicFree, catalog-sync

2026-05-23 16:51:14 +08:00
commit 069af30dba
847 changed files with 179878 additions and 0 deletions
@@ -0,0 +1,418 @@
+from __future__ import annotations
+
+import click
+
+try:
+    import uvicorn
+except Exception:  # pragma: no cover - exercised only when uvicorn is missing
+    class _MissingUvicorn:
+        def run(self, *args, **kwargs):
+            raise click.ClickException("serve command requires uvicorn. Install runtime dependencies first.")
+
+    uvicorn = _MissingUvicorn()
+
+from .db import initialize_database
+from .downloader import CatalogDownloader, DEFAULT_DOWNLOAD_WORKERS
+from .manual_playlists import parse_playlist_file
+from .repository import CatalogRepository
+from .resolver import DEFAULT_DOWNLOAD_SOURCES
+from .resolver_stats import default_resolver_stats_db_path, initialize_resolver_stats_database
+from .services import CatalogSyncService
+from .uploader import CatalogUploader
+
+
+def parse_sources(value: str) -> list[str]:
+    return [item.strip() for item in value.split(",") if item.strip()]
+
+
+def parse_int_list(value: str | None) -> list[int] | None:
+    if not value:
+        return None
+    return [int(item.strip()) for item in value.split(",") if item.strip()]
+
+
+def format_lyrics_progress(state: dict[str, object]) -> str:
+    total = int(state.get("total") or 0)
+    processed = int(state.get("processed") or 0)
+    progress_percent = int(state.get("progress_percent") or 0)
+    saved = int(state.get("saved") or 0)
+    skipped = int(state.get("skipped") or 0)
+    failed = int(state.get("failed") or 0)
+    return (
+        f"Lyrics progress: {processed}/{total} ({progress_percent}%) "
+        f"saved={saved} skipped={skipped} failed={failed}"
+    )
+
+
+PORT_RANGE = click.IntRange(1, 65535)
+
+
+def create_ops_web_app(*, db_path: str, env_path: str):
+    from .ops.web import create_app
+
+    return create_app(db_path=db_path, env_path=env_path, start_runner=True)
+
+
+class CatalogSyncApplication:
+    def __init__(self, db_path: str, library_root: str | None = None):
+        self.db_path = db_path
+        self.library_root = library_root
+        init_conn = initialize_database(db_path, default_library_root=library_root)
+        init_conn.close()
+        resolver_stats_init_conn = initialize_resolver_stats_database(default_resolver_stats_db_path(db_path))
+        resolver_stats_init_conn.close()
+        self.repository = CatalogRepository(db_path)
+        self.service = CatalogSyncService(self.repository)
+        self.downloader = CatalogDownloader(self.repository)
+
+    def init_db(self):
+        init_conn = initialize_database(self.db_path, default_library_root=self.library_root)
+        init_conn.close()
+        resolver_stats_init_conn = initialize_resolver_stats_database(
+            default_resolver_stats_db_path(self.db_path)
+        )
+        resolver_stats_init_conn.close()
+
+    def collect_playlists(self, sources: list[str], include_playlist_square: bool = True, include_toplist: bool = True):
+        return self.service.collect_playlists(sources, include_playlist_square, include_toplist)
+
+    def sync_playlist_catalog(self, sources: list[str] | None = None, limit: int | None = None):
+        return self.service.sync_playlist_catalog(sources=sources, limit=limit)
+
+    def download_pending(
+        self,
+        sources: list[str] | None = None,
+        limit: int | None = None,
+        playlist_ids: list[int] | None = None,
+        workers: int = DEFAULT_DOWNLOAD_WORKERS,
+        download_sources: list[str] | None = None,
+        lyrics_enabled: bool = True,
+        overwrite_lyrics: bool = False,
+    ):
+        if not self.library_root:
+            raise click.ClickException("download command requires --library-root")
+        downloader = CatalogDownloader(self.repository, worker_count=workers)
+        return downloader.download_pending(
+            self.library_root,
+            sources=sources,
+            limit=limit,
+            playlist_ids=playlist_ids,
+            download_sources=download_sources,
+            lyrics_enabled=lyrics_enabled,
+            overwrite_lyrics=overwrite_lyrics,
+        )
+
+    def run_playlist_file(
+        self,
+        playlist_file: str,
+        limit: int | None = None,
+        workers: int = DEFAULT_DOWNLOAD_WORKERS,
+        download_sources: list[str] | None = None,
+        lyrics_enabled: bool = True,
+        overwrite_lyrics: bool = False,
+    ) -> dict[str, int]:
+        parsed = parse_playlist_file(playlist_file)
+        if not parsed.entries:
+            raise click.ClickException("playlist file does not contain any valid playlist URLs")
+        playlist_ids = self.service.import_manual_playlists(playlist_file, parsed.entries)
+        if limit is not None:
+            playlist_ids = playlist_ids[:limit]
+        synchronized_songs = self.service.sync_specific_playlists(playlist_ids)
+        downloaded_songs = self.download_pending(
+            playlist_ids=playlist_ids,
+            workers=workers,
+            download_sources=download_sources,
+            lyrics_enabled=lyrics_enabled,
+            overwrite_lyrics=overwrite_lyrics,
+        )
+        return {
+            "total_lines": parsed.total_lines,
+            "valid_playlists": len(parsed.entries),
+            "skipped_lines": parsed.skipped_lines,
+            "synchronized_songs": synchronized_songs,
+            "downloaded_songs": downloaded_songs,
+        }
+
+    def register_object_backend(
+        self,
+        backend_name: str,
+        container_name: str,
+        endpoint: str,
+        region: str | None,
+        base_prefix: str | None,
+        credential_env_prefix: str,
+        addressing_style: str | None = None,
+        public_base_url: str | None = None,
+    ) -> int:
+        return self.repository.upsert_object_storage_backend(
+            name=backend_name,
+            container_name=container_name,
+            endpoint=endpoint,
+            region=region,
+            base_prefix=base_prefix,
+            credential_env_prefix=credential_env_prefix,
+            addressing_style=addressing_style,
+            public_base_url=public_base_url,
+        )
+
+    def upload_files(
+        self,
+        backend_name: str,
+        sources: list[str] | None = None,
+        playlist_ids: list[int] | None = None,
+        limit: int | None = None,
+        workers: int = 4,
+    ) -> dict[str, int]:
+        uploader = CatalogUploader(self.repository, worker_count=workers)
+        queued = uploader.enqueue_missing_uploads(
+            backend_name=backend_name,
+            sources=sources,
+            limit=limit,
+            playlist_ids=playlist_ids,
+        )
+        summary = uploader.run(backend_name=backend_name)
+        summary["queued"] = queued
+        return summary
+
+    def sync_local_lyrics(
+        self,
+        sources: list[str] | None = None,
+        playlist_ids: list[int] | None = None,
+        limit: int | None = None,
+        workers: int = DEFAULT_DOWNLOAD_WORKERS,
+        progress_callback=None,
+        overwrite_lyrics: bool = False,
+    ) -> dict[str, int]:
+        downloader = CatalogDownloader(self.repository, worker_count=workers)
+        return downloader.sync_local_lyrics(
+            sources=sources,
+            playlist_ids=playlist_ids,
+            limit=limit,
+            progress_callback=progress_callback,
+            overwrite_lyrics=overwrite_lyrics,
+        )
+
+
+@click.group()
+def cli():
+    """Catalog sync CLI for harvesting playlists and downloading songs."""
+
+
+@cli.command("init-db")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--library-root", type=click.Path(file_okay=False), required=False)
+def init_db_command(db_path: str, library_root: str | None):
+    app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
+    app.init_db()
+    click.echo(f"Initialized catalog database at {db_path}")
+
+
+@cli.command("collect")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--sources", default="netease,qq,kuwo", show_default=True)
+@click.option("--library-root", type=click.Path(file_okay=False), required=False)
+@click.option("--playlist-square/--no-playlist-square", default=True, show_default=True)
+@click.option("--toplist/--no-toplist", default=True, show_default=True)
+def collect_command(db_path: str, sources: str, library_root: str | None, playlist_square: bool, toplist: bool):
+    app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
+    result = app.collect_playlists(parse_sources(sources), playlist_square, toplist)
+    click.echo(f"Collected playlists: {result}")
+
+
+@cli.command("sync")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--sources", default="netease,qq,kuwo", show_default=True)
+@click.option("--library-root", type=click.Path(file_okay=False), required=False)
+@click.option("--limit", type=int, default=None)
+def sync_command(db_path: str, sources: str, library_root: str | None, limit: int | None):
+    app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
+    count = app.sync_playlist_catalog(parse_sources(sources), limit=limit)
+    click.echo(f"Synchronized songs: {count}")
+
+
+@cli.command("download")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--sources", default="netease,qq,kuwo", show_default=True)
+@click.option("--download-sources", default=",".join(DEFAULT_DOWNLOAD_SOURCES), show_default=True)
+@click.option("--library-root", type=click.Path(file_okay=False), required=True)
+@click.option("--limit", type=int, default=None)
+@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
+@click.option("--lyrics/--no-lyrics", "lyrics_enabled", default=True, show_default=True)
+@click.option("--overwrite-lyrics", is_flag=True, default=False)
+def download_command(
+    db_path: str,
+    sources: str,
+    download_sources: str,
+    library_root: str,
+    limit: int | None,
+    workers: int,
+    lyrics_enabled: bool,
+    overwrite_lyrics: bool,
+):
+    app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
+    count = app.download_pending(
+        parse_sources(sources),
+        limit=limit,
+        workers=workers,
+        download_sources=parse_sources(download_sources),
+        lyrics_enabled=lyrics_enabled,
+        overwrite_lyrics=overwrite_lyrics,
+    )
+    click.echo(f"Downloaded songs: {count}")
+
+
+@cli.command("run")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--sources", default="netease,qq,kuwo", show_default=True)
+@click.option("--download-sources", default=",".join(DEFAULT_DOWNLOAD_SOURCES), show_default=True)
+@click.option("--library-root", type=click.Path(file_okay=False), required=True)
+@click.option("--playlist-file", type=click.Path(dir_okay=False, exists=True), required=False)
+@click.option("--limit", type=int, default=None)
+@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
+@click.option("--lyrics/--no-lyrics", "lyrics_enabled", default=True, show_default=True)
+@click.option("--overwrite-lyrics", is_flag=True, default=False)
+def run_command(
+    db_path: str,
+    sources: str,
+    download_sources: str,
+    library_root: str,
+    playlist_file: str | None,
+    limit: int | None,
+    workers: int,
+    lyrics_enabled: bool,
+    overwrite_lyrics: bool,
+):
+    app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
+    parsed_download_sources = parse_sources(download_sources)
+    if playlist_file:
+        app.run_playlist_file(
+            playlist_file=playlist_file,
+            limit=limit,
+            workers=workers,
+            download_sources=parsed_download_sources,
+            lyrics_enabled=lyrics_enabled,
+            overwrite_lyrics=overwrite_lyrics,
+        )
+        click.echo("Catalog sync pipeline completed")
+        return
+    parsed_sources = parse_sources(sources)
+    app.collect_playlists(parsed_sources)
+    app.sync_playlist_catalog(parsed_sources, limit=limit)
+    app.download_pending(
+        parsed_sources,
+        limit=limit,
+        workers=workers,
+        download_sources=parsed_download_sources,
+        lyrics_enabled=lyrics_enabled,
+        overwrite_lyrics=overwrite_lyrics,
+    )
+    click.echo("Catalog sync pipeline completed")
+
+
+@cli.command("register-object-backend")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--backend", "backend_name", required=True)
+@click.option("--bucket", "container_name", required=True)
+@click.option("--endpoint", required=True)
+@click.option("--region", default=None)
+@click.option("--base-prefix", default=None)
+@click.option("--credential-env-prefix", required=True)
+@click.option("--addressing-style", default=None)
+@click.option("--public-base-url", default=None)
+def register_object_backend_command(
+    db_path: str,
+    backend_name: str,
+    container_name: str,
+    endpoint: str,
+    region: str | None,
+    base_prefix: str | None,
+    credential_env_prefix: str,
+    addressing_style: str | None,
+    public_base_url: str | None,
+):
+    app = CatalogSyncApplication(db_path=db_path)
+    backend_id = app.register_object_backend(
+        backend_name=backend_name,
+        container_name=container_name,
+        endpoint=endpoint,
+        region=region,
+        base_prefix=base_prefix,
+        credential_env_prefix=credential_env_prefix,
+        addressing_style=addressing_style,
+        public_base_url=public_base_url,
+    )
+    click.echo(f"Registered object backend: {backend_id}")
+
+
+@cli.command("upload")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--backend", "backend_name", required=True)
+@click.option("--sources", default=None)
+@click.option("--playlist-ids", default=None)
+@click.option("--limit", type=int, default=None)
+@click.option("--workers", type=int, default=4, show_default=True)
+def upload_command(
+    db_path: str,
+    backend_name: str,
+    sources: str | None,
+    playlist_ids: str | None,
+    limit: int | None,
+    workers: int,
+):
+    app = CatalogSyncApplication(db_path=db_path)
+    summary = app.upload_files(
+        backend_name=backend_name,
+        sources=parse_sources(sources) if sources else None,
+        playlist_ids=parse_int_list(playlist_ids),
+        limit=limit,
+        workers=workers,
+    )
+    click.echo(f"Upload summary: {summary}")
+
+
+@cli.command("lyrics")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--sources", default=None)
+@click.option("--playlist-ids", default=None)
+@click.option("--limit", type=int, default=None)
+@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
+@click.option("--overwrite-lyrics", is_flag=True, default=False)
+def lyrics_command(
+    db_path: str,
+    sources: str | None,
+    playlist_ids: str | None,
+    limit: int | None,
+    workers: int,
+    overwrite_lyrics: bool,
+):
+    app = CatalogSyncApplication(db_path=db_path)
+    def progress_callback(**state):
+        click.echo(format_lyrics_progress(state))
+
+    summary = app.sync_local_lyrics(
+        sources=parse_sources(sources) if sources else None,
+        playlist_ids=parse_int_list(playlist_ids),
+        limit=limit,
+        workers=workers,
+        progress_callback=progress_callback,
+        overwrite_lyrics=overwrite_lyrics,
+    )
+    click.echo(f"Lyrics summary: {summary}")
+
+
+@cli.command("serve")
+@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
+@click.option("--env-file", required=True, type=click.Path(dir_okay=False))
+@click.option("--host", default="127.0.0.1", show_default=True)
+@click.option("--port", type=PORT_RANGE, default=18080, show_default=True)
+def serve_command(db_path: str, env_file: str, host: str, port: int):
+    app = create_ops_web_app(db_path=db_path, env_path=env_file)
+    uvicorn.run(app, host=host, port=port)
+
+
+def main():
+    cli()
+
+
+if __name__ == "__main__":
+    main()