Initial import: Music_Server, MusicFree, catalog-sync
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
param(
|
||||
[string]$HostName = "192.168.5.43",
|
||||
[int]$Port = 222,
|
||||
[string]$User = "xiaoming",
|
||||
[string]$RemoteAppHome = "/volume4/Music_Cloud/Music_Server",
|
||||
[string]$Password = $(if ($env:NAS_192168543_PASSWORD) { $env:NAS_192168543_PASSWORD } else { "Nie@159357" }),
|
||||
[switch]$SkipHealthCheck
|
||||
)
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$PythonScript = Join-Path $ScriptDir "deploy_to_nas.py"
|
||||
|
||||
if (-not (Test-Path -LiteralPath $PythonScript)) {
|
||||
throw "Python script not found: $PythonScript"
|
||||
}
|
||||
|
||||
$arguments = @(
|
||||
$PythonScript,
|
||||
"--host", $HostName,
|
||||
"--port", "$Port",
|
||||
"--user", $User,
|
||||
"--remote-app-home", $RemoteAppHome
|
||||
)
|
||||
|
||||
if ($Password) {
|
||||
$arguments += @("--password", $Password)
|
||||
}
|
||||
if ($SkipHealthCheck) {
|
||||
$arguments += "--skip-health-check"
|
||||
}
|
||||
|
||||
python @arguments
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
DEFAULT_PASSWORD = "Nie@159357"
|
||||
SKIP_DIR_NAMES = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".git"}
|
||||
SKIP_FILE_SUFFIXES = {".pyc", ".pyo", ".DS_Store"}
|
||||
SKIP_FILE_NAMES = {"music_server_deploy.tar", "music_server_deploy.zip"}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
project_root = script_dir.parent
|
||||
default_template_dir = script_dir / "templates"
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload Music_Server to NAS staging and trigger deploy_and_restart.sh"
|
||||
)
|
||||
parser.add_argument("--host", default="192.168.5.43")
|
||||
parser.add_argument("--port", type=int, default=222)
|
||||
parser.add_argument("--user", default="xiaoming")
|
||||
parser.add_argument(
|
||||
"--password",
|
||||
default=os.environ.get("NAS_192168543_PASSWORD") or DEFAULT_PASSWORD,
|
||||
)
|
||||
parser.add_argument("--remote-app-home", default="/volume4/Music_Cloud/Music_Server")
|
||||
parser.add_argument("--source-dir", default=str(project_root))
|
||||
parser.add_argument("--template-dir", default=str(default_template_dir))
|
||||
parser.add_argument("--skip-health-check", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def to_sftp_path(shell_path: str) -> str:
|
||||
normalized = shell_path.rstrip("/")
|
||||
if normalized.startswith("/volume4"):
|
||||
mapped = normalized[len("/volume4") :]
|
||||
return mapped or "/"
|
||||
return normalized or "/"
|
||||
|
||||
|
||||
def shell_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
def ensure_remote_dir(sftp, path: str) -> None:
|
||||
path = path.rstrip("/")
|
||||
if not path or path == "/":
|
||||
return
|
||||
parts: list[str] = []
|
||||
current = path
|
||||
while current not in ("", "/"):
|
||||
parts.append(current)
|
||||
current = posixpath.dirname(current)
|
||||
for part in reversed(parts):
|
||||
try:
|
||||
sftp.stat(part)
|
||||
except OSError:
|
||||
sftp.mkdir(part)
|
||||
|
||||
|
||||
def exists_remote(sftp, path: str) -> bool:
|
||||
try:
|
||||
sftp.stat(path)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def remove_remote_tree(sftp, root: str) -> None:
|
||||
if not exists_remote(sftp, root):
|
||||
return
|
||||
for entry in sftp.listdir_attr(root):
|
||||
child = posixpath.join(root, entry.filename)
|
||||
if stat.S_ISDIR(entry.st_mode):
|
||||
remove_remote_tree(sftp, child)
|
||||
else:
|
||||
sftp.remove(child)
|
||||
sftp.rmdir(root)
|
||||
|
||||
|
||||
def iter_local_files(local_root: Path) -> Iterable[Path]:
|
||||
for current_root, dir_names, file_names in os.walk(local_root):
|
||||
dir_names[:] = [name for name in dir_names if name not in SKIP_DIR_NAMES]
|
||||
for file_name in file_names:
|
||||
path = Path(current_root) / file_name
|
||||
if path.suffix in SKIP_FILE_SUFFIXES:
|
||||
continue
|
||||
if path.name in SKIP_FILE_NAMES:
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def upload_tree(sftp, local_root: Path, remote_root: str) -> int:
|
||||
ensure_remote_dir(sftp, remote_root)
|
||||
uploaded = 0
|
||||
for local_file in iter_local_files(local_root):
|
||||
relative_path = local_file.relative_to(local_root).as_posix()
|
||||
remote_file = posixpath.join(remote_root, relative_path)
|
||||
remote_dir = posixpath.dirname(remote_file)
|
||||
ensure_remote_dir(sftp, remote_dir)
|
||||
sftp.put(str(local_file), remote_file)
|
||||
uploaded += 1
|
||||
return uploaded
|
||||
|
||||
|
||||
def read_channel_text(channel_file) -> str:
|
||||
data = channel_file.read()
|
||||
if isinstance(data, bytes):
|
||||
return data.decode("utf-8", "replace")
|
||||
return str(data)
|
||||
|
||||
|
||||
def run_remote_command(client, command: str, sudo_password: str | None = None) -> int:
|
||||
remote_command = command
|
||||
if sudo_password:
|
||||
remote_command = f"sudo -S sh -lc {shell_quote(command)}"
|
||||
stdin, stdout, stderr = client.exec_command(remote_command)
|
||||
if sudo_password:
|
||||
stdin.write(sudo_password + "\n")
|
||||
stdin.flush()
|
||||
out_text = read_channel_text(stdout)
|
||||
err_text = read_channel_text(stderr)
|
||||
if out_text:
|
||||
print(out_text, end="")
|
||||
if err_text:
|
||||
print(err_text, file=sys.stderr, end="")
|
||||
return stdout.channel.recv_exit_status()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import paramiko
|
||||
|
||||
args = parse_args()
|
||||
source_dir = Path(args.source_dir).resolve()
|
||||
template_dir = Path(args.template_dir).resolve()
|
||||
|
||||
if not source_dir.exists():
|
||||
print(f"Source dir not found: {source_dir}", file=sys.stderr)
|
||||
return 2
|
||||
deploy_template = template_dir / "deploy_and_restart.sh"
|
||||
if not deploy_template.exists():
|
||||
print(f"Missing template: {deploy_template}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
remote_app_home_shell = args.remote_app_home.rstrip("/")
|
||||
remote_app_home_sftp = to_sftp_path(remote_app_home_shell)
|
||||
remote_staging_shell = f"{remote_app_home_shell}/deploy/staging/music-server-app"
|
||||
remote_staging_sftp = to_sftp_path(remote_staging_shell)
|
||||
remote_bin_sftp = to_sftp_path(f"{remote_app_home_shell}/bin")
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
print(
|
||||
f"[deploy_to_nas] Connecting {args.user}@{args.host}:{args.port} ...",
|
||||
flush=True,
|
||||
)
|
||||
client.connect(
|
||||
hostname=args.host,
|
||||
port=args.port,
|
||||
username=args.user,
|
||||
password=args.password,
|
||||
timeout=15,
|
||||
banner_timeout=15,
|
||||
auth_timeout=15,
|
||||
)
|
||||
|
||||
try:
|
||||
sftp = client.open_sftp()
|
||||
try:
|
||||
ensure_remote_dir(sftp, remote_app_home_sftp)
|
||||
ensure_remote_dir(sftp, remote_bin_sftp)
|
||||
if exists_remote(sftp, remote_staging_sftp):
|
||||
print(
|
||||
f"[deploy_to_nas] Clearing staging: {remote_staging_sftp}",
|
||||
flush=True,
|
||||
)
|
||||
remove_remote_tree(sftp, remote_staging_sftp)
|
||||
ensure_remote_dir(sftp, remote_staging_sftp)
|
||||
uploaded_count = upload_tree(sftp, source_dir, remote_staging_sftp)
|
||||
print(
|
||||
f"[deploy_to_nas] Uploaded {uploaded_count} files to staging.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
sftp.put(
|
||||
str(deploy_template),
|
||||
posixpath.join(remote_bin_sftp, "deploy_and_restart.sh"),
|
||||
)
|
||||
finally:
|
||||
sftp.close()
|
||||
|
||||
chmod_cmd = (
|
||||
f"chmod +x {shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')}"
|
||||
)
|
||||
chmod_exit = run_remote_command(client, chmod_cmd)
|
||||
if chmod_exit != 0:
|
||||
print(
|
||||
f"[deploy_to_nas] chmod failed with exit code {chmod_exit}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return chmod_exit
|
||||
|
||||
deploy_cmd = (
|
||||
f"bash {shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')} "
|
||||
f"--staging-dir {shell_quote(remote_staging_shell)}"
|
||||
)
|
||||
if args.skip_health_check:
|
||||
deploy_cmd += " --skip-health-check"
|
||||
|
||||
print("[deploy_to_nas] Running deploy command...", flush=True)
|
||||
deploy_exit = run_remote_command(client, deploy_cmd, sudo_password=args.password)
|
||||
if deploy_exit != 0:
|
||||
print(
|
||||
f"[deploy_to_nas] Deploy failed with exit code {deploy_exit}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return deploy_exit
|
||||
|
||||
print("[deploy_to_nas] Deploy completed successfully.", flush=True)
|
||||
return 0
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TOPLIST_PARSE_STRATEGIES = {"netease_toplist", "qq_toplist", "kuwo_toplist"}
|
||||
TOPLIST_GROUP_NAMES = {
|
||||
"qq": "QQ音乐",
|
||||
"netease": "网易云",
|
||||
"kuwo": "酷我",
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build catalog_read.db from catalogsync.db for Music_Server."
|
||||
)
|
||||
parser.add_argument("--source-db", required=True, help="Path to catalogsync.db")
|
||||
parser.add_argument("--target-db", required=True, help="Path to catalog_read.db")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def connect(path: str) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def create_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(
|
||||
"""
|
||||
create table catalog_playlists (
|
||||
playlist_id integer primary key,
|
||||
platform text not null,
|
||||
remote_playlist_id text not null,
|
||||
name text not null,
|
||||
description text,
|
||||
cover_url text,
|
||||
play_count integer not null,
|
||||
song_count integer not null,
|
||||
playable_song_count integer not null
|
||||
);
|
||||
|
||||
create table catalog_tracks (
|
||||
song_id integer primary key,
|
||||
platform text not null,
|
||||
remote_song_id text not null,
|
||||
name text not null,
|
||||
singers text,
|
||||
album text,
|
||||
cover_url text,
|
||||
duration_ms integer,
|
||||
metadata_json text
|
||||
);
|
||||
|
||||
create table catalog_playlist_tracks (
|
||||
playlist_id integer not null,
|
||||
song_id integer not null,
|
||||
position integer not null
|
||||
);
|
||||
|
||||
create table catalog_toplists (
|
||||
toplist_id text primary key,
|
||||
platform text not null,
|
||||
name text not null,
|
||||
description text,
|
||||
cover_url text,
|
||||
play_count integer not null,
|
||||
song_count integer not null,
|
||||
playable_song_count integer not null,
|
||||
group_name text not null
|
||||
);
|
||||
|
||||
create table catalog_toplist_tracks (
|
||||
toplist_id text not null,
|
||||
song_id integer not null,
|
||||
position integer not null
|
||||
);
|
||||
|
||||
create table catalog_track_files (
|
||||
song_id integer not null,
|
||||
quality_label text,
|
||||
ext text,
|
||||
file_size_bytes integer,
|
||||
backend_type text,
|
||||
backend_name text,
|
||||
locator text,
|
||||
public_url text,
|
||||
status text,
|
||||
is_primary integer
|
||||
);
|
||||
|
||||
create table catalog_artists (
|
||||
artist_id integer primary key,
|
||||
artist_key text not null unique,
|
||||
platform text not null,
|
||||
remote_artist_id text,
|
||||
name text not null,
|
||||
normalized_name text not null,
|
||||
avatar_url text,
|
||||
description text,
|
||||
playable_song_count integer not null
|
||||
);
|
||||
|
||||
create table catalog_artist_tracks (
|
||||
artist_id integer not null,
|
||||
song_id integer not null,
|
||||
position integer not null
|
||||
);
|
||||
|
||||
create index idx_catalog_playlist_tracks_playlist on catalog_playlist_tracks (playlist_id, position);
|
||||
create index idx_catalog_toplist_tracks_toplist on catalog_toplist_tracks (toplist_id, position);
|
||||
create index idx_catalog_track_files_song on catalog_track_files (song_id, status, is_primary);
|
||||
create index idx_catalog_artist_tracks_artist on catalog_artist_tracks (artist_id, position);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _extract_song_cover(metadata_json: str | None) -> str | None:
|
||||
if not metadata_json:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
snapshot = payload.get("snapshot") or {}
|
||||
raw_data = snapshot.get("raw_data") or {}
|
||||
if isinstance(snapshot.get("cover_url"), str) and snapshot.get("cover_url"):
|
||||
return snapshot["cover_url"]
|
||||
|
||||
search = raw_data.get("search") or {}
|
||||
album = search.get("al") or {}
|
||||
if isinstance(album.get("picUrl"), str) and album.get("picUrl"):
|
||||
return album["picUrl"]
|
||||
return None
|
||||
|
||||
|
||||
def _extract_duration_ms(duration_seconds: int | None, metadata_json: str | None) -> int | None:
|
||||
if duration_seconds is not None:
|
||||
return int(duration_seconds) * 1000
|
||||
if not metadata_json:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
snapshot = payload.get("snapshot") or {}
|
||||
raw_data = snapshot.get("raw_data") or {}
|
||||
search = raw_data.get("search") or {}
|
||||
duration_ms = search.get("dt")
|
||||
if duration_ms is None:
|
||||
duration_s = snapshot.get("duration_s")
|
||||
return int(duration_s) * 1000 if duration_s is not None else None
|
||||
return int(duration_ms)
|
||||
|
||||
|
||||
def _extract_artist_avatar(metadata_json: str | None) -> str | None:
|
||||
if not metadata_json:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
avatar = payload.get("avatar") or payload.get("avatar_url") or payload.get("cover_url")
|
||||
return avatar if isinstance(avatar, str) and avatar else None
|
||||
|
||||
|
||||
def _extract_artist_description(metadata_json: str | None) -> str | None:
|
||||
if not metadata_json:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
description = payload.get("description") or payload.get("desc")
|
||||
return description if isinstance(description, str) and description else None
|
||||
|
||||
|
||||
def export_playlists(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
rows = source.execute(
|
||||
"""
|
||||
select
|
||||
p.id,
|
||||
p.platform,
|
||||
p.remote_playlist_id,
|
||||
p.name,
|
||||
p.creator_name,
|
||||
p.cover_url,
|
||||
p.play_count,
|
||||
coalesce(ps.song_count, p.collected_song_count, 0) as song_count,
|
||||
coalesce(pps.playable_song_count, 0) as playable_song_count
|
||||
from playlists p
|
||||
left join (
|
||||
select playlist_id, count(*) as song_count
|
||||
from playlist_songs
|
||||
group by playlist_id
|
||||
) ps on ps.playlist_id = p.id
|
||||
left join (
|
||||
select
|
||||
playlist_song_keys.playlist_id,
|
||||
count(*) as playable_song_count
|
||||
from (
|
||||
select distinct playlist_id, song_id
|
||||
from playlist_songs
|
||||
) playlist_song_keys
|
||||
where exists (
|
||||
select 1
|
||||
from file_assets fa
|
||||
join file_locations fl on fl.file_asset_id = fa.id
|
||||
where fa.song_id = playlist_song_keys.song_id
|
||||
and fl.status = 'active'
|
||||
)
|
||||
group by playlist_song_keys.playlist_id
|
||||
) pps on pps.playlist_id = p.id
|
||||
where p.parse_strategy not in ({})
|
||||
""".format(",".join("?" for _ in TOPLIST_PARSE_STRATEGIES)),
|
||||
tuple(sorted(TOPLIST_PARSE_STRATEGIES)),
|
||||
).fetchall()
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_playlists (
|
||||
playlist_id,
|
||||
platform,
|
||||
remote_playlist_id,
|
||||
name,
|
||||
description,
|
||||
cover_url,
|
||||
play_count,
|
||||
song_count,
|
||||
playable_song_count
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
int(row["id"]),
|
||||
row["platform"],
|
||||
row["remote_playlist_id"],
|
||||
row["name"],
|
||||
row["creator_name"],
|
||||
row["cover_url"],
|
||||
int(row["play_count"] or 0),
|
||||
int(row["song_count"] or 0),
|
||||
int(row["playable_song_count"] or 0),
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def export_tracks(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
rows = source.execute(
|
||||
"""
|
||||
select
|
||||
id,
|
||||
platform,
|
||||
remote_song_id,
|
||||
name,
|
||||
singers,
|
||||
album,
|
||||
duration_seconds,
|
||||
metadata_json
|
||||
from songs
|
||||
"""
|
||||
).fetchall()
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_tracks (
|
||||
song_id,
|
||||
platform,
|
||||
remote_song_id,
|
||||
name,
|
||||
singers,
|
||||
album,
|
||||
cover_url,
|
||||
duration_ms,
|
||||
metadata_json
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
int(row["id"]),
|
||||
row["platform"],
|
||||
row["remote_song_id"],
|
||||
row["name"],
|
||||
row["singers"],
|
||||
row["album"],
|
||||
_extract_song_cover(row["metadata_json"]),
|
||||
_extract_duration_ms(row["duration_seconds"], row["metadata_json"]),
|
||||
row["metadata_json"],
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def export_artists(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
exportable_rows = source.execute(
|
||||
"""
|
||||
select distinct
|
||||
a.id as artist_id,
|
||||
a.artist_key,
|
||||
a.platform,
|
||||
a.remote_artist_id,
|
||||
a.name,
|
||||
a.normalized_name,
|
||||
a.metadata_json,
|
||||
songs.id as song_id,
|
||||
songs.name as song_name
|
||||
from artists a
|
||||
join artist_songs s on s.artist_id = a.id
|
||||
join songs songs on songs.id = s.song_id
|
||||
where exists (
|
||||
select 1
|
||||
from file_assets fa
|
||||
join file_locations fl on fl.file_asset_id = fa.id
|
||||
where fa.song_id = s.song_id
|
||||
and fl.status = 'active'
|
||||
)
|
||||
order by a.id asc, lower(songs.name) asc, songs.id asc
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
artist_rows: dict[int, sqlite3.Row] = {}
|
||||
playable_song_counts: dict[int, int] = {}
|
||||
positions: dict[int, int] = {}
|
||||
track_payload: list[tuple[int, int, int]] = []
|
||||
for row in exportable_rows:
|
||||
artist_id = int(row["artist_id"])
|
||||
if artist_id not in artist_rows:
|
||||
artist_rows[artist_id] = row
|
||||
playable_song_counts[artist_id] = playable_song_counts.get(artist_id, 0) + 1
|
||||
positions[artist_id] = positions.get(artist_id, 0) + 1
|
||||
track_payload.append((artist_id, int(row["song_id"]), positions[artist_id]))
|
||||
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_artists (
|
||||
artist_id,
|
||||
artist_key,
|
||||
platform,
|
||||
remote_artist_id,
|
||||
name,
|
||||
normalized_name,
|
||||
avatar_url,
|
||||
description,
|
||||
playable_song_count
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
artist_id,
|
||||
artist_rows[artist_id]["artist_key"],
|
||||
artist_rows[artist_id]["platform"],
|
||||
artist_rows[artist_id]["remote_artist_id"],
|
||||
artist_rows[artist_id]["name"],
|
||||
artist_rows[artist_id]["normalized_name"],
|
||||
_extract_artist_avatar(artist_rows[artist_id]["metadata_json"]),
|
||||
_extract_artist_description(artist_rows[artist_id]["metadata_json"]),
|
||||
playable_song_counts[artist_id],
|
||||
)
|
||||
for artist_id in sorted(artist_rows)
|
||||
],
|
||||
)
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_artist_tracks (artist_id, song_id, position)
|
||||
values (?, ?, ?)
|
||||
""",
|
||||
track_payload,
|
||||
)
|
||||
|
||||
|
||||
def export_playlist_tracks(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
rows = source.execute(
|
||||
"""
|
||||
select ps.playlist_id, ps.song_id, coalesce(ps.position, 0) as position
|
||||
from playlist_songs ps
|
||||
join playlists p on p.id = ps.playlist_id
|
||||
where p.parse_strategy not in ({})
|
||||
""".format(",".join("?" for _ in TOPLIST_PARSE_STRATEGIES)),
|
||||
tuple(sorted(TOPLIST_PARSE_STRATEGIES)),
|
||||
).fetchall()
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_playlist_tracks (playlist_id, song_id, position)
|
||||
values (?, ?, ?)
|
||||
""",
|
||||
[(int(row["playlist_id"]), int(row["song_id"]), int(row["position"])) for row in rows],
|
||||
)
|
||||
|
||||
|
||||
def export_toplists(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
toplist_rows = source.execute(
|
||||
"""
|
||||
select
|
||||
p.id,
|
||||
p.platform,
|
||||
p.remote_playlist_id,
|
||||
p.name,
|
||||
p.creator_name,
|
||||
p.cover_url,
|
||||
p.play_count,
|
||||
coalesce(ps.song_count, p.collected_song_count, 0) as song_count,
|
||||
coalesce(pps.playable_song_count, 0) as playable_song_count,
|
||||
p.parse_strategy
|
||||
from playlists p
|
||||
left join (
|
||||
select playlist_id, count(*) as song_count
|
||||
from playlist_songs
|
||||
group by playlist_id
|
||||
) ps on ps.playlist_id = p.id
|
||||
left join (
|
||||
select
|
||||
playlist_song_keys.playlist_id,
|
||||
count(*) as playable_song_count
|
||||
from (
|
||||
select distinct playlist_id, song_id
|
||||
from playlist_songs
|
||||
) playlist_song_keys
|
||||
where exists (
|
||||
select 1
|
||||
from file_assets fa
|
||||
join file_locations fl on fl.file_asset_id = fa.id
|
||||
where fa.song_id = playlist_song_keys.song_id
|
||||
and fl.status = 'active'
|
||||
)
|
||||
group by playlist_song_keys.playlist_id
|
||||
) pps on pps.playlist_id = p.id
|
||||
where p.parse_strategy in ({})
|
||||
""".format(",".join("?" for _ in TOPLIST_PARSE_STRATEGIES)),
|
||||
tuple(sorted(TOPLIST_PARSE_STRATEGIES)),
|
||||
).fetchall()
|
||||
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_toplists (
|
||||
toplist_id,
|
||||
platform,
|
||||
name,
|
||||
description,
|
||||
cover_url,
|
||||
play_count,
|
||||
song_count,
|
||||
playable_song_count,
|
||||
group_name
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
f"{row['platform']}_top_{row['remote_playlist_id']}",
|
||||
row["platform"],
|
||||
row["name"],
|
||||
row["creator_name"],
|
||||
row["cover_url"],
|
||||
int(row["play_count"] or 0),
|
||||
int(row["song_count"] or 0),
|
||||
int(row["playable_song_count"] or 0),
|
||||
TOPLIST_GROUP_NAMES.get(row["platform"], row["platform"]),
|
||||
)
|
||||
for row in toplist_rows
|
||||
],
|
||||
)
|
||||
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_toplist_tracks (toplist_id, song_id, position)
|
||||
values (?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
f"{row['platform']}_top_{row['remote_playlist_id']}",
|
||||
int(track["song_id"]),
|
||||
int(track["position"] or 0),
|
||||
)
|
||||
for row in toplist_rows
|
||||
for track in source.execute(
|
||||
"""
|
||||
select song_id, position
|
||||
from playlist_songs
|
||||
where playlist_id = ?
|
||||
order by position asc
|
||||
""",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def export_track_files(source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
rows = source.execute(
|
||||
"""
|
||||
select
|
||||
fa.song_id,
|
||||
fa.quality_label,
|
||||
fa.ext,
|
||||
fa.file_size_bytes,
|
||||
sb.backend_type,
|
||||
sb.name as backend_name,
|
||||
fl.locator,
|
||||
coalesce(fl.public_url, fl.download_url) as public_url,
|
||||
fl.status,
|
||||
fl.is_primary
|
||||
from file_locations fl
|
||||
join file_assets fa on fa.id = fl.file_asset_id
|
||||
join storage_backends sb on sb.id = fl.backend_id
|
||||
"""
|
||||
).fetchall()
|
||||
target.executemany(
|
||||
"""
|
||||
insert into catalog_track_files (
|
||||
song_id,
|
||||
quality_label,
|
||||
ext,
|
||||
file_size_bytes,
|
||||
backend_type,
|
||||
backend_name,
|
||||
locator,
|
||||
public_url,
|
||||
status,
|
||||
is_primary
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
(
|
||||
int(row["song_id"]),
|
||||
row["quality_label"],
|
||||
row["ext"],
|
||||
row["file_size_bytes"],
|
||||
row["backend_type"],
|
||||
row["backend_name"],
|
||||
row["locator"],
|
||||
row["public_url"],
|
||||
row["status"],
|
||||
int(row["is_primary"] or 0),
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def build_catalog_read(source_db: str, target_db: str) -> None:
|
||||
source_path = Path(source_db).resolve()
|
||||
target_path = Path(target_db).resolve()
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fd, temp_path_str = tempfile.mkstemp(
|
||||
prefix=target_path.stem + ".",
|
||||
suffix=".tmp",
|
||||
dir=str(target_path.parent),
|
||||
)
|
||||
os.close(fd)
|
||||
temp_path = Path(temp_path_str)
|
||||
source: sqlite3.Connection | None = None
|
||||
target: sqlite3.Connection | None = None
|
||||
|
||||
try:
|
||||
source = connect(str(source_path))
|
||||
target = connect(str(temp_path))
|
||||
create_schema(target)
|
||||
export_playlists(source, target)
|
||||
export_tracks(source, target)
|
||||
export_artists(source, target)
|
||||
export_playlist_tracks(source, target)
|
||||
export_toplists(source, target)
|
||||
export_track_files(source, target)
|
||||
target.commit()
|
||||
source.close()
|
||||
source = None
|
||||
target.close()
|
||||
target = None
|
||||
os.replace(temp_path, target_path)
|
||||
finally:
|
||||
if source is not None:
|
||||
source.close()
|
||||
if target is not None:
|
||||
target.close()
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
build_catalog_read(source_db=args.source_db, target_db=args.target_db)
|
||||
print(f"catalog_read.db written to {args.target_db}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
RUN_DIR="${APP_HOME}/run"
|
||||
DEPLOY_DIR="${APP_HOME}/deploy"
|
||||
LOCK_DIR="${RUN_DIR}/deploy.lock"
|
||||
APP_DIR="${APP_HOME}/app"
|
||||
CONFIG_DIR="${APP_HOME}/config"
|
||||
DATA_DIR="${APP_HOME}/data"
|
||||
CONFIG_FILE="${CONFIG_DIR}/music_server.env"
|
||||
LEGACY_CONFIG_FILE="${APP_DIR}/config/music_server.env"
|
||||
LEGACY_CONFIG_EXAMPLE="${APP_DIR}/config/music_server.env.example"
|
||||
LEGACY_DATA_DIR="${APP_DIR}/data"
|
||||
DEFAULT_STAGING_DIR="${APP_HOME}/deploy/staging/music-server-app"
|
||||
BACKUP_ROOT="${APP_HOME}/deploy/backups"
|
||||
COMPOSE_FILE="${APP_DIR}/docker-compose.nas.yml"
|
||||
DOCKER_BIN_DIR="/var/packages/Docker/target/usr/bin"
|
||||
|
||||
STAGING_DIR="${DEFAULT_STAGING_DIR}"
|
||||
HEALTH_URL="http://127.0.0.1:18081/healthz"
|
||||
HEALTH_RETRIES=45
|
||||
HEALTH_INTERVAL_SECONDS=2
|
||||
KEEP_BACKUPS=3
|
||||
SKIP_HEALTH_CHECK=0
|
||||
BACKUP_DIR=""
|
||||
HAS_BACKUP=0
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
$(basename "$0") [options]
|
||||
|
||||
Options:
|
||||
--staging-dir PATH Repo staging directory to deploy into ${APP_HOME}/app
|
||||
--health-url URL Health-check URL (default: ${HEALTH_URL})
|
||||
--health-retries N Max health-check retries (default: 45)
|
||||
--health-interval-sec N Health-check interval seconds (default: 2)
|
||||
--keep-backups N Number of app backups to keep (default: 3)
|
||||
--skip-health-check Skip HTTP health check
|
||||
-h, --help Show help
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
echo "[deploy_and_restart.sh] $*"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "[deploy_and_restart.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
validate_positive_integer() {
|
||||
local value="$1"
|
||||
local name="$2"
|
||||
if ! [[ "${value}" =~ ^[0-9]+$ ]] || (( value < 1 )); then
|
||||
fail "${name} must be a positive integer: ${value}"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_deploy_lock() {
|
||||
mkdir -p "${RUN_DIR}"
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner_pid=""
|
||||
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
|
||||
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
|
||||
fail "Another deploy is running (owner_pid=${owner_pid})"
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
fail "Cannot acquire deploy lock: ${LOCK_DIR}"
|
||||
fi
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
}
|
||||
|
||||
cleanup_lock() {
|
||||
rm -rf "${LOCK_DIR}"
|
||||
}
|
||||
|
||||
ensure_runtime_layout() {
|
||||
mkdir -p "${CONFIG_DIR}" "${DATA_DIR}" "${DEPLOY_DIR}" "${BACKUP_ROOT}" "${RUN_DIR}"
|
||||
|
||||
if [[ ! -f "${CONFIG_FILE}" ]]; then
|
||||
if [[ -f "${LEGACY_CONFIG_FILE}" ]]; then
|
||||
cp -a "${LEGACY_CONFIG_FILE}" "${CONFIG_FILE}"
|
||||
log "Copied legacy runtime config to ${CONFIG_FILE}"
|
||||
elif [[ -f "${LEGACY_CONFIG_EXAMPLE}" ]]; then
|
||||
cp -a "${LEGACY_CONFIG_EXAMPLE}" "${CONFIG_FILE}"
|
||||
log "Bootstrapped runtime config from example: ${CONFIG_FILE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -d "${LEGACY_DATA_DIR}" ]] && [[ -z "$(ls -A "${DATA_DIR}" 2>/dev/null || true)" ]]; then
|
||||
cp -a "${LEGACY_DATA_DIR}/." "${DATA_DIR}/"
|
||||
log "Copied legacy runtime data into ${DATA_DIR}"
|
||||
fi
|
||||
}
|
||||
|
||||
require_staging_checkout() {
|
||||
if [[ ! -d "${STAGING_DIR}" ]]; then
|
||||
fail "Staging directory not found: ${STAGING_DIR}"
|
||||
fi
|
||||
if [[ ! -f "${STAGING_DIR}/docker-compose.nas.yml" ]]; then
|
||||
fail "Invalid staging checkout (missing docker-compose.nas.yml): ${STAGING_DIR}"
|
||||
fi
|
||||
if [[ ! -f "${STAGING_DIR}/Dockerfile" ]]; then
|
||||
fail "Invalid staging checkout (missing Dockerfile): ${STAGING_DIR}"
|
||||
fi
|
||||
}
|
||||
|
||||
sync_app_checkout() {
|
||||
require_staging_checkout
|
||||
|
||||
BACKUP_DIR="${BACKUP_ROOT}/app_$(date +%Y%m%d_%H%M%S)"
|
||||
if [[ -d "${APP_DIR}" ]]; then
|
||||
mv "${APP_DIR}" "${BACKUP_DIR}"
|
||||
HAS_BACKUP=1
|
||||
log "Backed up current app to ${BACKUP_DIR}"
|
||||
fi
|
||||
|
||||
cp -a "${STAGING_DIR}" "${APP_DIR}"
|
||||
log "Synced new app checkout from ${STAGING_DIR} -> ${APP_DIR}"
|
||||
}
|
||||
|
||||
run_compose() {
|
||||
export PATH="${DOCKER_BIN_DIR}:${PATH}"
|
||||
cd "${APP_DIR}"
|
||||
docker-compose -f "${COMPOSE_FILE}" "$@"
|
||||
}
|
||||
|
||||
remove_conflicting_named_containers() {
|
||||
export PATH="${DOCKER_BIN_DIR}:${PATH}"
|
||||
docker rm -f music-server >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
restart_service() {
|
||||
run_compose build catalog-export music-server
|
||||
remove_conflicting_named_containers
|
||||
run_compose down --remove-orphans || true
|
||||
run_compose run --rm catalog-export
|
||||
run_compose up -d music-server
|
||||
}
|
||||
|
||||
wait_health() {
|
||||
if (( SKIP_HEALTH_CHECK == 1 )); then
|
||||
log "Health check skipped by --skip-health-check"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
fail "curl is required for health check"
|
||||
fi
|
||||
|
||||
log "Health checking: ${HEALTH_URL}"
|
||||
for _ in $(seq 1 "${HEALTH_RETRIES}"); do
|
||||
if curl -fsS "${HEALTH_URL}" >/dev/null; then
|
||||
log "Health check passed"
|
||||
return 0
|
||||
fi
|
||||
sleep "${HEALTH_INTERVAL_SECONDS}"
|
||||
done
|
||||
|
||||
log "Health check failed: ${HEALTH_URL}"
|
||||
return 1
|
||||
}
|
||||
|
||||
rollback() {
|
||||
log "Starting rollback..."
|
||||
if (( HAS_BACKUP == 0 )) || [[ ! -d "${BACKUP_DIR}" ]]; then
|
||||
log "No backup available; rollback skipped"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -rf "${APP_DIR}"
|
||||
mv "${BACKUP_DIR}" "${APP_DIR}"
|
||||
HAS_BACKUP=0
|
||||
log "Restored backup to ${APP_DIR}"
|
||||
|
||||
restart_service
|
||||
wait_health
|
||||
}
|
||||
|
||||
prune_backups() {
|
||||
if (( KEEP_BACKUPS < 1 )) || [[ ! -d "${BACKUP_ROOT}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
mapfile -t backups < <(ls -1dt "${BACKUP_ROOT}"/app_* 2>/dev/null || true)
|
||||
if (( ${#backups[@]} <= KEEP_BACKUPS )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for old_backup in "${backups[@]:KEEP_BACKUPS}"; do
|
||||
rm -rf "${old_backup}"
|
||||
log "Pruned old backup: ${old_backup}"
|
||||
done
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--staging-dir)
|
||||
STAGING_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-url)
|
||||
HEALTH_URL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-retries)
|
||||
HEALTH_RETRIES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-interval-sec)
|
||||
HEALTH_INTERVAL_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-backups)
|
||||
KEEP_BACKUPS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--skip-health-check)
|
||||
SKIP_HEALTH_CHECK=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
fail "Unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_positive_integer "${HEALTH_RETRIES}" "HEALTH_RETRIES"
|
||||
validate_positive_integer "${HEALTH_INTERVAL_SECONDS}" "HEALTH_INTERVAL_SECONDS"
|
||||
validate_positive_integer "${KEEP_BACKUPS}" "KEEP_BACKUPS"
|
||||
|
||||
ensure_runtime_layout
|
||||
acquire_deploy_lock
|
||||
trap cleanup_lock EXIT INT TERM
|
||||
|
||||
LOG_FILE="${DEPLOY_DIR}/deploy_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
log "Starting deploy. staging=${STAGING_DIR}"
|
||||
log "App home: ${APP_HOME}"
|
||||
log "Runtime config: ${CONFIG_FILE}"
|
||||
log "Runtime data: ${DATA_DIR}"
|
||||
log "Deploy log: ${LOG_FILE}"
|
||||
|
||||
if ! sync_app_checkout; then
|
||||
fail "Sync step failed"
|
||||
fi
|
||||
|
||||
ensure_runtime_layout
|
||||
if [[ ! -f "${CONFIG_FILE}" ]]; then
|
||||
fail "Runtime config not found: ${CONFIG_FILE}"
|
||||
fi
|
||||
|
||||
if ! restart_service; then
|
||||
log "Restart failed; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed during restart; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed during restart; rollback failed."
|
||||
fi
|
||||
|
||||
if ! wait_health; then
|
||||
log "New version failed health check; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed during health check; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed during health check; rollback failed."
|
||||
fi
|
||||
|
||||
prune_backups
|
||||
log "Deploy succeeded."
|
||||
Reference in New Issue
Block a user