419 lines
16 KiB
Python
419 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import click
|
|
|
|
try:
|
|
import uvicorn
|
|
except Exception: # pragma: no cover - exercised only when uvicorn is missing
|
|
class _MissingUvicorn:
|
|
def run(self, *args, **kwargs):
|
|
raise click.ClickException("serve command requires uvicorn. Install runtime dependencies first.")
|
|
|
|
uvicorn = _MissingUvicorn()
|
|
|
|
from .db import initialize_database
|
|
from .downloader import CatalogDownloader, DEFAULT_DOWNLOAD_WORKERS
|
|
from .manual_playlists import parse_playlist_file
|
|
from .repository import CatalogRepository
|
|
from .resolver import DEFAULT_DOWNLOAD_SOURCES
|
|
from .resolver_stats import default_resolver_stats_db_path, initialize_resolver_stats_database
|
|
from .services import CatalogSyncService
|
|
from .uploader import CatalogUploader
|
|
|
|
|
|
def parse_sources(value: str) -> list[str]:
|
|
return [item.strip() for item in value.split(",") if item.strip()]
|
|
|
|
|
|
def parse_int_list(value: str | None) -> list[int] | None:
|
|
if not value:
|
|
return None
|
|
return [int(item.strip()) for item in value.split(",") if item.strip()]
|
|
|
|
|
|
def format_lyrics_progress(state: dict[str, object]) -> str:
|
|
total = int(state.get("total") or 0)
|
|
processed = int(state.get("processed") or 0)
|
|
progress_percent = int(state.get("progress_percent") or 0)
|
|
saved = int(state.get("saved") or 0)
|
|
skipped = int(state.get("skipped") or 0)
|
|
failed = int(state.get("failed") or 0)
|
|
return (
|
|
f"Lyrics progress: {processed}/{total} ({progress_percent}%) "
|
|
f"saved={saved} skipped={skipped} failed={failed}"
|
|
)
|
|
|
|
|
|
PORT_RANGE = click.IntRange(1, 65535)
|
|
|
|
|
|
def create_ops_web_app(*, db_path: str, env_path: str):
|
|
from .ops.web import create_app
|
|
|
|
return create_app(db_path=db_path, env_path=env_path, start_runner=True)
|
|
|
|
|
|
class CatalogSyncApplication:
|
|
def __init__(self, db_path: str, library_root: str | None = None):
|
|
self.db_path = db_path
|
|
self.library_root = library_root
|
|
init_conn = initialize_database(db_path, default_library_root=library_root)
|
|
init_conn.close()
|
|
resolver_stats_init_conn = initialize_resolver_stats_database(default_resolver_stats_db_path(db_path))
|
|
resolver_stats_init_conn.close()
|
|
self.repository = CatalogRepository(db_path)
|
|
self.service = CatalogSyncService(self.repository)
|
|
self.downloader = CatalogDownloader(self.repository)
|
|
|
|
def init_db(self):
|
|
init_conn = initialize_database(self.db_path, default_library_root=self.library_root)
|
|
init_conn.close()
|
|
resolver_stats_init_conn = initialize_resolver_stats_database(
|
|
default_resolver_stats_db_path(self.db_path)
|
|
)
|
|
resolver_stats_init_conn.close()
|
|
|
|
def collect_playlists(self, sources: list[str], include_playlist_square: bool = True, include_toplist: bool = True):
|
|
return self.service.collect_playlists(sources, include_playlist_square, include_toplist)
|
|
|
|
def sync_playlist_catalog(self, sources: list[str] | None = None, limit: int | None = None):
|
|
return self.service.sync_playlist_catalog(sources=sources, limit=limit)
|
|
|
|
def download_pending(
|
|
self,
|
|
sources: list[str] | None = None,
|
|
limit: int | None = None,
|
|
playlist_ids: list[int] | None = None,
|
|
workers: int = DEFAULT_DOWNLOAD_WORKERS,
|
|
download_sources: list[str] | None = None,
|
|
lyrics_enabled: bool = True,
|
|
overwrite_lyrics: bool = False,
|
|
):
|
|
if not self.library_root:
|
|
raise click.ClickException("download command requires --library-root")
|
|
downloader = CatalogDownloader(self.repository, worker_count=workers)
|
|
return downloader.download_pending(
|
|
self.library_root,
|
|
sources=sources,
|
|
limit=limit,
|
|
playlist_ids=playlist_ids,
|
|
download_sources=download_sources,
|
|
lyrics_enabled=lyrics_enabled,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
|
|
def run_playlist_file(
|
|
self,
|
|
playlist_file: str,
|
|
limit: int | None = None,
|
|
workers: int = DEFAULT_DOWNLOAD_WORKERS,
|
|
download_sources: list[str] | None = None,
|
|
lyrics_enabled: bool = True,
|
|
overwrite_lyrics: bool = False,
|
|
) -> dict[str, int]:
|
|
parsed = parse_playlist_file(playlist_file)
|
|
if not parsed.entries:
|
|
raise click.ClickException("playlist file does not contain any valid playlist URLs")
|
|
playlist_ids = self.service.import_manual_playlists(playlist_file, parsed.entries)
|
|
if limit is not None:
|
|
playlist_ids = playlist_ids[:limit]
|
|
synchronized_songs = self.service.sync_specific_playlists(playlist_ids)
|
|
downloaded_songs = self.download_pending(
|
|
playlist_ids=playlist_ids,
|
|
workers=workers,
|
|
download_sources=download_sources,
|
|
lyrics_enabled=lyrics_enabled,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
return {
|
|
"total_lines": parsed.total_lines,
|
|
"valid_playlists": len(parsed.entries),
|
|
"skipped_lines": parsed.skipped_lines,
|
|
"synchronized_songs": synchronized_songs,
|
|
"downloaded_songs": downloaded_songs,
|
|
}
|
|
|
|
def register_object_backend(
|
|
self,
|
|
backend_name: str,
|
|
container_name: str,
|
|
endpoint: str,
|
|
region: str | None,
|
|
base_prefix: str | None,
|
|
credential_env_prefix: str,
|
|
addressing_style: str | None = None,
|
|
public_base_url: str | None = None,
|
|
) -> int:
|
|
return self.repository.upsert_object_storage_backend(
|
|
name=backend_name,
|
|
container_name=container_name,
|
|
endpoint=endpoint,
|
|
region=region,
|
|
base_prefix=base_prefix,
|
|
credential_env_prefix=credential_env_prefix,
|
|
addressing_style=addressing_style,
|
|
public_base_url=public_base_url,
|
|
)
|
|
|
|
def upload_files(
|
|
self,
|
|
backend_name: str,
|
|
sources: list[str] | None = None,
|
|
playlist_ids: list[int] | None = None,
|
|
limit: int | None = None,
|
|
workers: int = 4,
|
|
) -> dict[str, int]:
|
|
uploader = CatalogUploader(self.repository, worker_count=workers)
|
|
queued = uploader.enqueue_missing_uploads(
|
|
backend_name=backend_name,
|
|
sources=sources,
|
|
limit=limit,
|
|
playlist_ids=playlist_ids,
|
|
)
|
|
summary = uploader.run(backend_name=backend_name)
|
|
summary["queued"] = queued
|
|
return summary
|
|
|
|
def sync_local_lyrics(
|
|
self,
|
|
sources: list[str] | None = None,
|
|
playlist_ids: list[int] | None = None,
|
|
limit: int | None = None,
|
|
workers: int = DEFAULT_DOWNLOAD_WORKERS,
|
|
progress_callback=None,
|
|
overwrite_lyrics: bool = False,
|
|
) -> dict[str, int]:
|
|
downloader = CatalogDownloader(self.repository, worker_count=workers)
|
|
return downloader.sync_local_lyrics(
|
|
sources=sources,
|
|
playlist_ids=playlist_ids,
|
|
limit=limit,
|
|
progress_callback=progress_callback,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
"""Catalog sync CLI for harvesting playlists and downloading songs."""
|
|
|
|
|
|
@cli.command("init-db")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--library-root", type=click.Path(file_okay=False), required=False)
|
|
def init_db_command(db_path: str, library_root: str | None):
|
|
app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
|
|
app.init_db()
|
|
click.echo(f"Initialized catalog database at {db_path}")
|
|
|
|
|
|
@cli.command("collect")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--sources", default="netease,qq,kuwo", show_default=True)
|
|
@click.option("--library-root", type=click.Path(file_okay=False), required=False)
|
|
@click.option("--playlist-square/--no-playlist-square", default=True, show_default=True)
|
|
@click.option("--toplist/--no-toplist", default=True, show_default=True)
|
|
def collect_command(db_path: str, sources: str, library_root: str | None, playlist_square: bool, toplist: bool):
|
|
app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
|
|
result = app.collect_playlists(parse_sources(sources), playlist_square, toplist)
|
|
click.echo(f"Collected playlists: {result}")
|
|
|
|
|
|
@cli.command("sync")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--sources", default="netease,qq,kuwo", show_default=True)
|
|
@click.option("--library-root", type=click.Path(file_okay=False), required=False)
|
|
@click.option("--limit", type=int, default=None)
|
|
def sync_command(db_path: str, sources: str, library_root: str | None, limit: int | None):
|
|
app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
|
|
count = app.sync_playlist_catalog(parse_sources(sources), limit=limit)
|
|
click.echo(f"Synchronized songs: {count}")
|
|
|
|
|
|
@cli.command("download")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--sources", default="netease,qq,kuwo", show_default=True)
|
|
@click.option("--download-sources", default=",".join(DEFAULT_DOWNLOAD_SOURCES), show_default=True)
|
|
@click.option("--library-root", type=click.Path(file_okay=False), required=True)
|
|
@click.option("--limit", type=int, default=None)
|
|
@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
|
|
@click.option("--lyrics/--no-lyrics", "lyrics_enabled", default=True, show_default=True)
|
|
@click.option("--overwrite-lyrics", is_flag=True, default=False)
|
|
def download_command(
|
|
db_path: str,
|
|
sources: str,
|
|
download_sources: str,
|
|
library_root: str,
|
|
limit: int | None,
|
|
workers: int,
|
|
lyrics_enabled: bool,
|
|
overwrite_lyrics: bool,
|
|
):
|
|
app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
|
|
count = app.download_pending(
|
|
parse_sources(sources),
|
|
limit=limit,
|
|
workers=workers,
|
|
download_sources=parse_sources(download_sources),
|
|
lyrics_enabled=lyrics_enabled,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
click.echo(f"Downloaded songs: {count}")
|
|
|
|
|
|
@cli.command("run")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--sources", default="netease,qq,kuwo", show_default=True)
|
|
@click.option("--download-sources", default=",".join(DEFAULT_DOWNLOAD_SOURCES), show_default=True)
|
|
@click.option("--library-root", type=click.Path(file_okay=False), required=True)
|
|
@click.option("--playlist-file", type=click.Path(dir_okay=False, exists=True), required=False)
|
|
@click.option("--limit", type=int, default=None)
|
|
@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
|
|
@click.option("--lyrics/--no-lyrics", "lyrics_enabled", default=True, show_default=True)
|
|
@click.option("--overwrite-lyrics", is_flag=True, default=False)
|
|
def run_command(
|
|
db_path: str,
|
|
sources: str,
|
|
download_sources: str,
|
|
library_root: str,
|
|
playlist_file: str | None,
|
|
limit: int | None,
|
|
workers: int,
|
|
lyrics_enabled: bool,
|
|
overwrite_lyrics: bool,
|
|
):
|
|
app = CatalogSyncApplication(db_path=db_path, library_root=library_root)
|
|
parsed_download_sources = parse_sources(download_sources)
|
|
if playlist_file:
|
|
app.run_playlist_file(
|
|
playlist_file=playlist_file,
|
|
limit=limit,
|
|
workers=workers,
|
|
download_sources=parsed_download_sources,
|
|
lyrics_enabled=lyrics_enabled,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
click.echo("Catalog sync pipeline completed")
|
|
return
|
|
parsed_sources = parse_sources(sources)
|
|
app.collect_playlists(parsed_sources)
|
|
app.sync_playlist_catalog(parsed_sources, limit=limit)
|
|
app.download_pending(
|
|
parsed_sources,
|
|
limit=limit,
|
|
workers=workers,
|
|
download_sources=parsed_download_sources,
|
|
lyrics_enabled=lyrics_enabled,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
click.echo("Catalog sync pipeline completed")
|
|
|
|
|
|
@cli.command("register-object-backend")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--backend", "backend_name", required=True)
|
|
@click.option("--bucket", "container_name", required=True)
|
|
@click.option("--endpoint", required=True)
|
|
@click.option("--region", default=None)
|
|
@click.option("--base-prefix", default=None)
|
|
@click.option("--credential-env-prefix", required=True)
|
|
@click.option("--addressing-style", default=None)
|
|
@click.option("--public-base-url", default=None)
|
|
def register_object_backend_command(
|
|
db_path: str,
|
|
backend_name: str,
|
|
container_name: str,
|
|
endpoint: str,
|
|
region: str | None,
|
|
base_prefix: str | None,
|
|
credential_env_prefix: str,
|
|
addressing_style: str | None,
|
|
public_base_url: str | None,
|
|
):
|
|
app = CatalogSyncApplication(db_path=db_path)
|
|
backend_id = app.register_object_backend(
|
|
backend_name=backend_name,
|
|
container_name=container_name,
|
|
endpoint=endpoint,
|
|
region=region,
|
|
base_prefix=base_prefix,
|
|
credential_env_prefix=credential_env_prefix,
|
|
addressing_style=addressing_style,
|
|
public_base_url=public_base_url,
|
|
)
|
|
click.echo(f"Registered object backend: {backend_id}")
|
|
|
|
|
|
@cli.command("upload")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--backend", "backend_name", required=True)
|
|
@click.option("--sources", default=None)
|
|
@click.option("--playlist-ids", default=None)
|
|
@click.option("--limit", type=int, default=None)
|
|
@click.option("--workers", type=int, default=4, show_default=True)
|
|
def upload_command(
|
|
db_path: str,
|
|
backend_name: str,
|
|
sources: str | None,
|
|
playlist_ids: str | None,
|
|
limit: int | None,
|
|
workers: int,
|
|
):
|
|
app = CatalogSyncApplication(db_path=db_path)
|
|
summary = app.upload_files(
|
|
backend_name=backend_name,
|
|
sources=parse_sources(sources) if sources else None,
|
|
playlist_ids=parse_int_list(playlist_ids),
|
|
limit=limit,
|
|
workers=workers,
|
|
)
|
|
click.echo(f"Upload summary: {summary}")
|
|
|
|
|
|
@cli.command("lyrics")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--sources", default=None)
|
|
@click.option("--playlist-ids", default=None)
|
|
@click.option("--limit", type=int, default=None)
|
|
@click.option("--workers", type=int, default=DEFAULT_DOWNLOAD_WORKERS, envvar="DOWNLOAD_WORKERS", show_default=True)
|
|
@click.option("--overwrite-lyrics", is_flag=True, default=False)
|
|
def lyrics_command(
|
|
db_path: str,
|
|
sources: str | None,
|
|
playlist_ids: str | None,
|
|
limit: int | None,
|
|
workers: int,
|
|
overwrite_lyrics: bool,
|
|
):
|
|
app = CatalogSyncApplication(db_path=db_path)
|
|
def progress_callback(**state):
|
|
click.echo(format_lyrics_progress(state))
|
|
|
|
summary = app.sync_local_lyrics(
|
|
sources=parse_sources(sources) if sources else None,
|
|
playlist_ids=parse_int_list(playlist_ids),
|
|
limit=limit,
|
|
workers=workers,
|
|
progress_callback=progress_callback,
|
|
overwrite_lyrics=overwrite_lyrics,
|
|
)
|
|
click.echo(f"Lyrics summary: {summary}")
|
|
|
|
|
|
@cli.command("serve")
|
|
@click.option("--db", "db_path", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--env-file", required=True, type=click.Path(dir_okay=False))
|
|
@click.option("--host", default="127.0.0.1", show_default=True)
|
|
@click.option("--port", type=PORT_RANGE, default=18080, show_default=True)
|
|
def serve_command(db_path: str, env_file: str, host: str, port: int):
|
|
app = create_ops_web_app(db_path=db_path, env_path=env_file)
|
|
uvicorn.run(app, host=host, port=port)
|
|
|
|
|
|
def main():
|
|
cli()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|