xiaoming/musicdl-catalog-sync-suite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: Music_Server, MusicFree, catalog-sync

Browse Source
This commit is contained in:
Codex
2026-05-23 16:51:14 +08:00
commit 069af30dba
847 changed files with 179878 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalog-sync/scripts/catalogsync/bootstrap_to_linux.ps1
+92
View File
@@ -0,0 +1,92 @@
param(
[Parameter(Mandatory = $true)]
[string]$RemoteHost,
[int]$Port = 22,
[Parameter(Mandatory = $true)]
[string]$User,
[string]$RootDir = "/volume4/Music_Cloud"
)
function Quote-RemotePathForPosixShell {
param(
[Parameter(Mandatory = $true)]
[string]$Path
)
$EscapedPath = $Path.Replace("'", "'\''")
return "'$EscapedPath'"
}
function New-ScpRemoteTarget {
param(
[Parameter(Mandatory = $true)]
[string]$Remote,
[Parameter(Mandatory = $true)]
[string]$Path
)
return "${Remote}:$(Quote-RemotePathForPosixShell -Path $Path)"
}
$AppHome = "$RootDir/catalogsync"
$RemoteDirs = @(
$RootDir,
"$RootDir/library",
"$AppHome/app",
"$AppHome/bin",
"$AppHome/config",
"$AppHome/data",
"$AppHome/inputs",
"$AppHome/logs"
)
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$ProjectRoot = Resolve-Path (Join-Path $ScriptDir "..\..")
$Remote = "$User@$RemoteHost"
$QuotedDirs = $RemoteDirs | ForEach-Object { Quote-RemotePathForPosixShell -Path $_ }
$RemoteMkdirCommand = "mkdir -p " + ($QuotedDirs -join " ")
ssh -p $Port $Remote $RemoteMkdirCommand
if ($LASTEXITCODE -ne 0) {
throw "Failed to create remote directories."
}
$AppSources = @(
(Join-Path $ProjectRoot "musicdl"),
(Join-Path $ProjectRoot "setup.py"),
(Join-Path $ProjectRoot "README.md"),
(Join-Path $ProjectRoot "LICENSE"),
(Join-Path $ProjectRoot "requirements.txt"),
(Join-Path $ProjectRoot "requirements-optional.txt")
)
$RemoteAppDirTarget = New-ScpRemoteTarget -Remote $Remote -Path "$AppHome/app/"
foreach ($Source in $AppSources) {
scp -P $Port -r $Source $RemoteAppDirTarget
if ($LASTEXITCODE -ne 0) {
throw "Failed to copy app source: $Source"
}
}
$BinTemplates = @(
(Join-Path $ScriptDir "templates\download_all.sh"),
(Join-Path $ScriptDir "templates\download_from_file.sh"),
(Join-Path $ScriptDir "templates\upload_all.sh"),
(Join-Path $ScriptDir "templates\install_runtime.sh"),
(Join-Path $ScriptDir "templates\serve_console.sh"),
(Join-Path $ScriptDir "templates\deploy_and_restart.sh")
)
$RemoteBinDirTarget = New-ScpRemoteTarget -Remote $Remote -Path "$AppHome/bin/"
foreach ($Template in $BinTemplates) {
scp -P $Port $Template $RemoteBinDirTarget
if ($LASTEXITCODE -ne 0) {
throw "Failed to copy bin template: $Template"
}
}
$RemoteEnvExample = "$AppHome/config/catalogsync.env.example"
$LocalEnvExample = Join-Path $ScriptDir "templates\catalogsync.env.example"
$RemoteEnvExampleTarget = New-ScpRemoteTarget -Remote $Remote -Path $RemoteEnvExample
scp -P $Port $LocalEnvExample $RemoteEnvExampleTarget
if ($LASTEXITCODE -ne 0) {
throw "Failed to copy catalogsync.env.example."
}
catalog-sync/scripts/catalogsync/deploy_to_nas.ps1
+33
View File
@@ -0,0 +1,33 @@
param(
[string]$HostName = "192.168.5.43",
[int]$Port = 222,
[string]$User = "xiaoming",
[string]$RemoteAppHome = "/volume4/Music_Cloud/catalogsync",
[string]$Password = $(if ($env:NAS_192168543_PASSWORD) { $env:NAS_192168543_PASSWORD } else { "Nie@159357" }),
[switch]$SkipHealthCheck
)
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$PythonScript = Join-Path $ScriptDir "deploy_to_nas.py"
if (-not (Test-Path -LiteralPath $PythonScript)) {
throw "Python script not found: $PythonScript"
}
$arguments = @(
$PythonScript,
"--host", $HostName,
"--port", "$Port",
"--user", $User,
"--remote-app-home", $RemoteAppHome
)
if ($Password) {
$arguments += @("--password", $Password)
}
if ($SkipHealthCheck) {
$arguments += "--skip-health-check"
}
python @arguments
exit $LASTEXITCODE
catalog-sync/scripts/catalogsync/deploy_to_nas.py
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import posixpath
import stat
import sys
from pathlib import Path
from typing import Iterable
import paramiko
DEFAULT_PASSWORD = "Nie@159357"
SKIP_DIR_NAMES = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".git"}
SKIP_FILE_SUFFIXES = {".pyc", ".pyo", ".DS_Store"}
def parse_args() -> argparse.Namespace:
script_dir = Path(__file__).resolve().parent
project_root = script_dir.parent.parent
default_source_dir = project_root / "musicdl" / "catalogsync"
default_template_dir = script_dir / "templates"
parser = argparse.ArgumentParser(
description="Upload catalogsync to NAS staging and trigger deploy_and_restart.sh"
)
parser.add_argument("--host", default="192.168.5.43")
parser.add_argument("--port", type=int, default=222)
parser.add_argument("--user", default="xiaoming")
parser.add_argument(
"--password",
default=os.environ.get("NAS_192168543_PASSWORD") or DEFAULT_PASSWORD,
)
parser.add_argument("--remote-app-home", default="/volume4/Music_Cloud/catalogsync")
parser.add_argument("--source-dir", default=str(default_source_dir))
parser.add_argument("--template-dir", default=str(default_template_dir))
parser.add_argument("--skip-health-check", action="store_true")
return parser.parse_args()
def to_sftp_path(shell_path: str) -> str:
normalized = shell_path.rstrip("/")
if normalized.startswith("/volume4"):
mapped = normalized[len("/volume4") :]
return mapped or "/"
return normalized or "/"
def shell_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
def ensure_remote_dir(sftp: paramiko.SFTPClient, path: str) -> None:
path = path.rstrip("/")
if not path:
return
if path == "/":
return
parts: list[str] = []
current = path
while current not in ("", "/"):
parts.append(current)
current = posixpath.dirname(current)
for part in reversed(parts):
try:
sftp.stat(part)
except OSError:
sftp.mkdir(part)
def exists_remote(sftp: paramiko.SFTPClient, path: str) -> bool:
try:
sftp.stat(path)
return True
except OSError:
return False
def remove_remote_tree(sftp: paramiko.SFTPClient, root: str) -> None:
if not exists_remote(sftp, root):
return
for entry in sftp.listdir_attr(root):
child = posixpath.join(root, entry.filename)
if stat.S_ISDIR(entry.st_mode):
remove_remote_tree(sftp, child)
else:
sftp.remove(child)
sftp.rmdir(root)
def iter_local_files(local_root: Path) -> Iterable[Path]:
for current_root, dir_names, file_names in os.walk(local_root):
dir_names[:] = [name for name in dir_names if name not in SKIP_DIR_NAMES]
for file_name in file_names:
path = Path(current_root) / file_name
if path.suffix in SKIP_FILE_SUFFIXES:
continue
yield path
def upload_tree(sftp: paramiko.SFTPClient, local_root: Path, remote_root: str) -> int:
ensure_remote_dir(sftp, remote_root)
uploaded = 0
for local_file in iter_local_files(local_root):
relative_path = local_file.relative_to(local_root).as_posix()
remote_file = posixpath.join(remote_root, relative_path)
remote_dir = posixpath.dirname(remote_file)
ensure_remote_dir(sftp, remote_dir)
sftp.put(str(local_file), remote_file)
uploaded += 1
return uploaded
def read_channel_text(channel_file) -> str:
data = channel_file.read()
if isinstance(data, bytes):
return data.decode("utf-8", "replace")
return str(data)
def run_remote_command(client: paramiko.SSHClient, command: str) -> int:
stdin, stdout, stderr = client.exec_command(command)
_ = stdin
out_text = read_channel_text(stdout)
err_text = read_channel_text(stderr)
if out_text:
print(out_text, end="")
if err_text:
print(err_text, file=sys.stderr, end="")
return stdout.channel.recv_exit_status()
def main() -> int:
args = parse_args()
source_dir = Path(args.source_dir).resolve()
template_dir = Path(args.template_dir).resolve()
if not source_dir.exists():
print(f"Source dir not found: {source_dir}", file=sys.stderr)
return 2
if not (template_dir / "serve_console.sh").exists():
print(f"Missing template: {template_dir / 'serve_console.sh'}", file=sys.stderr)
return 2
if not (template_dir / "deploy_and_restart.sh").exists():
print(f"Missing template: {template_dir / 'deploy_and_restart.sh'}", file=sys.stderr)
return 2
remote_app_home_shell = args.remote_app_home.rstrip("/")
remote_app_home_sftp = to_sftp_path(remote_app_home_shell)
remote_staging_shell = f"{remote_app_home_shell}/deploy/staging/catalogsync"
remote_staging_sftp = to_sftp_path(remote_staging_shell)
remote_bin_sftp = to_sftp_path(f"{remote_app_home_shell}/bin")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print(
f"[deploy_to_nas] Connecting {args.user}@{args.host}:{args.port} ...",
flush=True,
)
client.connect(
hostname=args.host,
port=args.port,
username=args.user,
password=args.password,
timeout=15,
banner_timeout=15,
auth_timeout=15,
)
try:
sftp = client.open_sftp()
try:
ensure_remote_dir(sftp, remote_bin_sftp)
if exists_remote(sftp, remote_staging_sftp):
print(
f"[deploy_to_nas] Clearing staging: {remote_staging_sftp}",
flush=True,
)
remove_remote_tree(sftp, remote_staging_sftp)
ensure_remote_dir(sftp, remote_staging_sftp)
uploaded_count = upload_tree(sftp, source_dir, remote_staging_sftp)
print(
f"[deploy_to_nas] Uploaded {uploaded_count} files to staging.",
flush=True,
)
sftp.put(
str(template_dir / "serve_console.sh"),
posixpath.join(remote_bin_sftp, "serve_console.sh"),
)
sftp.put(
str(template_dir / "deploy_and_restart.sh"),
posixpath.join(remote_bin_sftp, "deploy_and_restart.sh"),
)
finally:
sftp.close()
chmod_cmd = (
f"chmod +x {shell_quote(remote_app_home_shell + '/bin/serve_console.sh')} "
f"{shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')}"
)
chmod_exit = run_remote_command(client, chmod_cmd)
if chmod_exit != 0:
print(
f"[deploy_to_nas] chmod failed with exit code {chmod_exit}",
file=sys.stderr,
)
return chmod_exit
deploy_cmd = (
f"bash {shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')} "
f"--staging-dir {shell_quote(remote_staging_shell)}"
)
if args.skip_health_check:
deploy_cmd += " --skip-health-check"
print(f"[deploy_to_nas] Running deploy command...", flush=True)
deploy_exit = run_remote_command(client, deploy_cmd)
if deploy_exit != 0:
print(
f"[deploy_to_nas] Deploy failed with exit code {deploy_exit}",
file=sys.stderr,
)
return deploy_exit
print("[deploy_to_nas] Deploy completed successfully.", flush=True)
return 0
finally:
client.close()
if __name__ == "__main__":
raise SystemExit(main())
catalog-sync/scripts/catalogsync/list_suspected_live_songs.py
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env python3
from __future__ import annotations
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from musicdl.catalogsync.suspected_live import main
if __name__ == "__main__":
raise SystemExit(main())
catalog-sync/scripts/catalogsync/templates/catalogsync.env.example
+36
View File
@@ -0,0 +1,36 @@
ROOT_DIR=/volume4/Music_Cloud
APP_HOME=/volume4/Music_Cloud/catalogsync
LIBRARY_DIR=/volume4/Music_Cloud/library
DB_PATH=/volume4/Music_Cloud/catalogsync/data/catalogsync.db
INPUT_DIR=/volume4/Music_Cloud/catalogsync/inputs
LOG_DIR=/volume4/Music_Cloud/catalogsync/logs
ENV_FILE=/volume4/Music_Cloud/catalogsync/config/catalogsync.env
WEB_HOST=127.0.0.1
WEB_PORT=18080
PYTHON_BIN=python3
VENV_DIR=/volume4/Music_Cloud/catalogsync/app/.venv
DOWNLOAD_LAYOUT=platform_first_artist
DOWNLOAD_SOURCES=qq,kuwo,migu,qianqian,kugou,netease
DOWNLOAD_WORKERS=10
SYNC_WORKERS=4
OBJECT_BACKEND_NAME=main-s3
OBJECT_BUCKET=music-bucket
OBJECT_ENDPOINT=https://s3.example.com
OBJECT_REGION=auto
OBJECT_BASE_PREFIX=music
OBJECT_ADDRESSING_STYLE=
OBJECT_PUBLIC_BASE_URL=
OBJECT_CREDENTIAL_ENV_PREFIX=CATALOGSYNC_MAIN_S3
UPLOAD_WORKERS=4
UPLOAD_SOURCES=
UPLOAD_PLAYLIST_IDS=
UPLOAD_LIMIT=
CATALOGSYNC_MAIN_S3_ACCESS_KEY_ID=
CATALOGSYNC_MAIN_S3_SECRET_ACCESS_KEY=
CATALOGSYNC_MAIN_S3_SESSION_TOKEN=
catalog-sync/scripts/catalogsync/templates/deploy_and_restart.sh
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
RUN_DIR="${APP_HOME}/run"
DEPLOY_DIR="${APP_HOME}/deploy"
LOCK_DIR="${RUN_DIR}/deploy.lock"
PID_FILE="${RUN_DIR}/serve.pid"
TARGET_DIR="${APP_HOME}/app/musicdl/catalogsync"
DEFAULT_STAGING_DIR="${APP_HOME}/deploy/staging/catalogsync"
BACKUP_ROOT="${APP_HOME}/deploy/backups"
STAGING_DIR="${DEFAULT_STAGING_DIR}"
HEALTH_URL=""
HEALTH_RETRIES=45
HEALTH_INTERVAL_SECONDS=1
KEEP_BACKUPS=5
SKIP_HEALTH_CHECK=0
BACKUP_DIR=""
HAS_BACKUP=0
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
usage() {
cat <<EOF
Usage:
$(basename "$0") [options]
Options:
--staging-dir PATH Source directory to sync into app/musicdl/catalogsync
--health-url URL Health-check URL (default: http://127.0.0.1:\${WEB_PORT}/dashboard)
--health-retries N Max health-check retries (default: 45)
--health-interval-sec N Health-check interval seconds (default: 1)
--keep-backups N Number of backups to keep (default: 5)
--skip-health-check Skip HTTP health check
-h, --help Show help
EOF
}
log() {
echo "[deploy_and_restart.sh] $*"
}
fail() {
echo "[deploy_and_restart.sh] ERROR: $*" >&2
exit 1
}
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
validate_positive_integer() {
local value="$1"
local name="$2"
if ! [[ "${value}" =~ ^[0-9]+$ ]] || (( value < 1 )); then
fail "${name} must be a positive integer: ${value}"
fi
}
acquire_deploy_lock() {
mkdir -p "${RUN_DIR}"
if mkdir "${LOCK_DIR}" 2>/dev/null; then
echo "$$" > "${LOCK_DIR}/owner_pid"
return 0
fi
local owner_pid=""
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
fi
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
fail "Another deploy is running (owner_pid=${owner_pid})"
fi
rm -rf "${LOCK_DIR}"
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
fail "Cannot acquire deploy lock: ${LOCK_DIR}"
fi
echo "$$" > "${LOCK_DIR}/owner_pid"
}
cleanup_lock() {
rm -rf "${LOCK_DIR}"
}
stop_service() {
local pid=""
if [[ -f "${PID_FILE}" ]]; then
pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
fi
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
log "Stopping running service from PID file (pid=${pid})"
kill -TERM "${pid}" 2>/dev/null || true
for _ in $(seq 1 20); do
if ! kill -0 "${pid}" 2>/dev/null; then
break
fi
sleep 1
done
if kill -0 "${pid}" 2>/dev/null; then
log "Service still alive; force killing pid=${pid}"
kill -KILL "${pid}" 2>/dev/null || true
fi
fi
rm -f "${PID_FILE}"
local serve_pattern="musicdl.catalogsync.cli serve"
local wrapper_pattern="${APP_HOME}/bin/serve_console.sh"
for _ in $(seq 1 10); do
local serve_count
local wrapper_count
serve_count="$(count_matching_processes "${serve_pattern}")"
wrapper_count="$(count_matching_processes "${wrapper_pattern}")"
if [[ "${serve_count}" == "0" && "${wrapper_count}" == "0" ]]; then
break
fi
kill_matching_processes "TERM" "${serve_pattern}"
kill_matching_processes "TERM" "${wrapper_pattern}"
sleep 1
done
kill_matching_processes "KILL" "${serve_pattern}"
kill_matching_processes "KILL" "${wrapper_pattern}"
}
start_service() {
local launch_log="${LOG_DIR}/serve_console_launch_$(date +%Y%m%d_%H%M%S).log"
nohup bash "${APP_HOME}/bin/serve_console.sh" >"${launch_log}" 2>&1 &
local launcher_pid=$!
log "Started service launcher pid=${launcher_pid}, launch_log=${launch_log}"
}
sync_catalogsync() {
if [[ ! -d "${STAGING_DIR}" ]]; then
fail "Staging directory not found: ${STAGING_DIR}"
fi
if [[ ! -f "${STAGING_DIR}/__init__.py" ]]; then
fail "Invalid staging directory (missing __init__.py): ${STAGING_DIR}"
fi
mkdir -p "${BACKUP_ROOT}" "$(dirname "${TARGET_DIR}")"
BACKUP_DIR="${BACKUP_ROOT}/catalogsync_$(date +%Y%m%d_%H%M%S)"
if [[ -d "${TARGET_DIR}" ]]; then
mv "${TARGET_DIR}" "${BACKUP_DIR}"
HAS_BACKUP=1
log "Backed up current catalogsync to ${BACKUP_DIR}"
fi
cp -a "${STAGING_DIR}" "${TARGET_DIR}"
log "Synced new catalogsync from ${STAGING_DIR} -> ${TARGET_DIR}"
}
wait_health() {
if (( SKIP_HEALTH_CHECK == 1 )); then
log "Health check skipped by --skip-health-check"
return 0
fi
if ! command -v curl >/dev/null 2>&1; then
fail "curl is required for health check"
fi
log "Health checking: ${HEALTH_URL}"
for _ in $(seq 1 "${HEALTH_RETRIES}"); do
local code
code="$(curl -s -o /dev/null -w '%{http_code}' "${HEALTH_URL}" || true)"
if [[ "${code}" == "200" ]]; then
log "Health check passed (HTTP 200)"
return 0
fi
sleep "${HEALTH_INTERVAL_SECONDS}"
done
log "Health check failed: ${HEALTH_URL}"
return 1
}
verify_single_instance() {
local serve_count
serve_count="$(count_matching_processes 'musicdl.catalogsync.cli serve')"
if [[ "${serve_count}" != "1" ]]; then
log "Unexpected serve process count: ${serve_count}"
return 1
fi
log "Single-instance check passed (serve_count=${serve_count})"
return 0
}
list_matching_processes() {
local pattern="$1"
ps -ef | grep -F "${pattern}" | grep -v grep | awk '{print $2}' || true
}
count_matching_processes() {
local pattern="$1"
list_matching_processes "${pattern}" | awk 'NF {count++} END {print count+0}'
}
kill_matching_processes() {
local signal_name="$1"
local pattern="$2"
local pid
while IFS= read -r pid; do
if [[ -n "${pid}" ]]; then
kill "-${signal_name}" "${pid}" 2>/dev/null || true
fi
done < <(list_matching_processes "${pattern}")
}
rollback() {
log "Starting rollback..."
stop_service
if (( HAS_BACKUP == 0 )) || [[ ! -d "${BACKUP_DIR}" ]]; then
log "No backup available; rollback skipped"
return 1
fi
rm -rf "${TARGET_DIR}"
mv "${BACKUP_DIR}" "${TARGET_DIR}"
HAS_BACKUP=0
log "Restored backup to ${TARGET_DIR}"
start_service
if ! wait_health; then
log "Rollback service failed health check"
return 1
fi
verify_single_instance
}
prune_backups() {
if (( KEEP_BACKUPS < 1 )); then
return 0
fi
if [[ ! -d "${BACKUP_ROOT}" ]]; then
return 0
fi
mapfile -t backups < <(ls -1dt "${BACKUP_ROOT}"/catalogsync_* 2>/dev/null || true)
if (( ${#backups[@]} <= KEEP_BACKUPS )); then
return 0
fi
for old_backup in "${backups[@]:KEEP_BACKUPS}"; do
rm -rf "${old_backup}"
log "Pruned old backup: ${old_backup}"
done
}
while [[ $# -gt 0 ]]; do
case "$1" in
--staging-dir)
STAGING_DIR="${2:-}"
shift 2
;;
--health-url)
HEALTH_URL="${2:-}"
shift 2
;;
--health-retries)
HEALTH_RETRIES="${2:-}"
shift 2
;;
--health-interval-sec)
HEALTH_INTERVAL_SECONDS="${2:-}"
shift 2
;;
--keep-backups)
KEEP_BACKUPS="${2:-}"
shift 2
;;
--skip-health-check)
SKIP_HEALTH_CHECK=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
fail "Unknown argument: $1"
;;
esac
done
validate_positive_integer "${HEALTH_RETRIES}" "HEALTH_RETRIES"
validate_positive_integer "${HEALTH_INTERVAL_SECONDS}" "HEALTH_INTERVAL_SECONDS"
validate_positive_integer "${KEEP_BACKUPS}" "KEEP_BACKUPS"
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in WEB_PORT LOG_DIR; do
require_var "${required_var}"
done
if [[ -z "${HEALTH_URL}" ]]; then
HEALTH_URL="http://127.0.0.1:${WEB_PORT}/dashboard"
fi
mkdir -p "${DEPLOY_DIR}" "${RUN_DIR}" "${LOG_DIR}" "${BACKUP_ROOT}" "${APP_HOME}/app/musicdl"
acquire_deploy_lock
trap cleanup_lock EXIT INT TERM
LOG_FILE="${LOG_DIR}/deploy_and_restart_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
log "Starting deploy. staging=${STAGING_DIR}"
log "Deploy log: ${LOG_FILE}"
if ! sync_catalogsync; then
fail "Sync step failed"
fi
stop_service
start_service
if ! wait_health; then
log "New version failed health check; attempting rollback."
if rollback; then
fail "Deploy failed; rollback succeeded."
fi
fail "Deploy failed; rollback failed."
fi
if ! verify_single_instance; then
log "Single-instance check failed; attempting rollback."
if rollback; then
fail "Deploy failed by single-instance check; rollback succeeded."
fi
fail "Deploy failed by single-instance check; rollback failed."
fi
prune_backups
log "Deploy succeeded."
catalog-sync/scripts/catalogsync/templates/download_all.sh
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
fail() {
echo "[download_all.sh] ERROR: $*" >&2
exit 1
}
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
require_var "${required_var}"
done
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
PYTHON_BIN="${VENV_DIR}/bin/python"
fi
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
fi
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
LOG_FILE="${LOG_DIR}/download_all_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "[download_all.sh] logging to ${LOG_FILE}"
EXTRA_ARGS=()
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
fi
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
--db "${DB_PATH}" \
--library-root "${LIBRARY_DIR}" \
--workers "${DOWNLOAD_WORKERS:-10}" \
"${EXTRA_ARGS[@]}" \
"$@"
catalog-sync/scripts/catalogsync/templates/download_from_file.sh
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
set -euo pipefail
fail() {
echo "[download_from_file.sh] ERROR: $*" >&2
exit 1
}
if [[ $# -lt 1 ]]; then
fail "usage: $0 <playlist-file> [extra args...]"
fi
PLAYLIST_FILE="$1"
shift
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
if [[ ! -f "${PLAYLIST_FILE}" ]]; then
fail "playlist file not found: ${PLAYLIST_FILE}"
fi
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
require_var "${required_var}"
done
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
PYTHON_BIN="${VENV_DIR}/bin/python"
fi
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
fi
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
LOG_FILE="${LOG_DIR}/download_from_file_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "[download_from_file.sh] logging to ${LOG_FILE}"
EXTRA_ARGS=()
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
fi
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
--db "${DB_PATH}" \
--library-root "${LIBRARY_DIR}" \
--playlist-file "${PLAYLIST_FILE}" \
"${EXTRA_ARGS[@]}" \
"$@"
catalog-sync/scripts/catalogsync/templates/install_runtime.sh
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
fail() {
echo "[install_runtime.sh] ERROR: $*" >&2
exit 1
}
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
require_requirement() {
local requirement_name="$1"
if ! grep -Eq "^${requirement_name}([<>=!~].*)?$" "${NAS_REQUIREMENTS_FILE}"; then
fail "Missing required runtime dependency in requirements.txt: ${requirement_name}"
fi
}
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in APP_HOME DB_PATH LOG_DIR PYTHON_BIN VENV_DIR; do
require_var "${required_var}"
done
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
fi
APP_DIR="${APP_HOME}/app"
REQUIREMENTS_FILE="${APP_DIR}/requirements.txt"
SETUP_FILE="${APP_DIR}/setup.py"
if [[ ! -f "${REQUIREMENTS_FILE}" ]]; then
fail "requirements.txt not found: ${REQUIREMENTS_FILE}"
fi
if [[ ! -f "${SETUP_FILE}" ]]; then
fail "setup.py not found: ${SETUP_FILE}"
fi
mkdir -p "${APP_HOME}" "${APP_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
export PYTHONPATH="${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
LOG_FILE="${LOG_DIR}/install_runtime_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "[install_runtime.sh] logging to ${LOG_FILE}"
if [[ ! -d "${VENV_DIR}" ]]; then
"${PYTHON_BIN}" -m venv "${VENV_DIR}"
fi
RUNTIME_PYTHON_BIN="${VENV_DIR}/bin/python"
if [[ ! -x "${RUNTIME_PYTHON_BIN}" ]]; then
fail "Virtualenv python not found after setup: ${RUNTIME_PYTHON_BIN}"
fi
"${RUNTIME_PYTHON_BIN}" -m pip install --upgrade pip setuptools wheel
NAS_REQUIREMENTS_FILE="${APP_DIR}/requirements.nas.txt"
grep -v '^nodejs-wheel$' "${REQUIREMENTS_FILE}" > "${NAS_REQUIREMENTS_FILE}"
for runtime_requirement in fastapi uvicorn python-multipart; do
require_requirement "${runtime_requirement}"
done
"${RUNTIME_PYTHON_BIN}" -m pip install -r "${NAS_REQUIREMENTS_FILE}"
"${RUNTIME_PYTHON_BIN}" -m pip install --no-deps -e "${APP_DIR}"
echo "[install_runtime.sh] runtime ready: ${RUNTIME_PYTHON_BIN}"
catalog-sync/scripts/catalogsync/templates/load_env.sh
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env bash
trim_env_whitespace() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "${value}"
}
load_env_file() {
local env_file="$1"
local raw_line=""
local normalized=""
local key=""
local value=""
local trimmed_value=""
local quote_char=""
[[ -f "${env_file}" ]] || return 1
while IFS= read -r raw_line || [[ -n "${raw_line}" ]]; do
raw_line="${raw_line%$'\r'}"
normalized="$(trim_env_whitespace "${raw_line}")"
if [[ -z "${normalized}" || "${normalized:0:1}" == "#" ]]; then
continue
fi
if [[ "${normalized}" == export\ * ]]; then
normalized="${normalized#export }"
fi
if [[ "${normalized}" != *=* ]]; then
continue
fi
key="$(trim_env_whitespace "${normalized%%=*}")"
if [[ -z "${key}" ]]; then
continue
fi
value="${normalized#*=}"
trimmed_value="$(trim_env_whitespace "${value}")"
if [[ ${#trimmed_value} -ge 2 ]]; then
quote_char="${trimmed_value:0:1}"
if [[ ( "${quote_char}" == "'" || "${quote_char}" == '"' ) && "${trimmed_value: -1}" == "${quote_char}" ]]; then
value="${trimmed_value:1:${#trimmed_value}-2}"
else
value="${trimmed_value}"
fi
else
value="${trimmed_value}"
fi
printf -v "${key}" '%s' "${value}"
export "${key}"
done < "${env_file}"
}
catalog-sync/scripts/catalogsync/templates/serve_console.sh
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
RUN_DIR="${APP_HOME}/run"
LOCK_DIR="${RUN_DIR}/serve.lock"
PID_FILE="${RUN_DIR}/serve.pid"
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
fail() {
echo "[serve_console.sh] ERROR: $*" >&2
exit 1
}
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
acquire_serve_lock() {
mkdir -p "${RUN_DIR}"
if mkdir "${LOCK_DIR}" 2>/dev/null; then
echo "$$" > "${LOCK_DIR}/owner_pid"
return 0
fi
local owner_pid=""
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
fi
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
fail "Another serve_console instance is running (owner_pid=${owner_pid})"
fi
rm -rf "${LOCK_DIR}"
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
fail "Cannot acquire serve lock: ${LOCK_DIR}"
fi
echo "$$" > "${LOCK_DIR}/owner_pid"
}
cleanup() {
local exit_code=$?
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
kill -TERM "${SERVER_PID}" 2>/dev/null || true
wait "${SERVER_PID}" 2>/dev/null || true
fi
rm -f "${PID_FILE}"
rm -rf "${LOCK_DIR}"
return "${exit_code}"
}
validate_port() {
local port_value="$1"
if ! [[ "${port_value}" =~ ^[0-9]+$ ]] || (( port_value < 1 || port_value > 65535 )); then
fail "WEB_PORT must be an integer in range 1..65535: ${port_value}"
fi
}
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in DB_PATH ENV_FILE WEB_HOST WEB_PORT LOG_DIR PYTHON_BIN VENV_DIR; do
require_var "${required_var}"
done
validate_port "${WEB_PORT}"
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
PYTHON_BIN="${VENV_DIR}/bin/python"
fi
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
fi
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")" "$(dirname "${ENV_FILE}")"
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
acquire_serve_lock
trap cleanup EXIT INT TERM
if [[ -f "${PID_FILE}" ]]; then
existing_pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
fail "Server process already running (pid=${existing_pid})"
fi
rm -f "${PID_FILE}"
fi
LOG_FILE="${LOG_DIR}/serve_console_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "[serve_console.sh] logging to ${LOG_FILE}"
# Run from the deployed app directory so `python -m musicdl...` does not
# accidentally import an older checkout from the current working directory.
cd "${APP_HOME}/app"
"${PYTHON_BIN}" -m musicdl.catalogsync.cli serve \
--db "${DB_PATH}" \
--env-file "${ENV_FILE}" \
--host "${WEB_HOST}" \
--port "${WEB_PORT}" \
"$@" &
SERVER_PID=$!
echo "${SERVER_PID}" > "${PID_FILE}"
echo "[serve_console.sh] server pid=${SERVER_PID}"
wait "${SERVER_PID}"
catalog-sync/scripts/catalogsync/templates/upload_all.sh
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
# shellcheck source=./load_env.sh
source "${SCRIPT_DIR}/load_env.sh"
fail() {
echo "[upload_all.sh] ERROR: $*" >&2
exit 1
}
require_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
fi
}
if [[ -f "${CONFIG_FILE}" ]]; then
load_env_file "${CONFIG_FILE}"
else
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
fi
for required_var in DB_PATH LOG_DIR PYTHON_BIN VENV_DIR OBJECT_BACKEND_NAME OBJECT_BUCKET OBJECT_ENDPOINT OBJECT_CREDENTIAL_ENV_PREFIX; do
require_var "${required_var}"
done
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
PYTHON_BIN="${VENV_DIR}/bin/python"
fi
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
fi
ACCESS_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_ACCESS_KEY_ID"
SECRET_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_SECRET_ACCESS_KEY"
require_var "${ACCESS_KEY_VAR}"
require_var "${SECRET_KEY_VAR}"
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
LOG_FILE="${LOG_DIR}/upload_all_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "[upload_all.sh] logging to ${LOG_FILE}"
REGISTER_ARGS=(
-m musicdl.catalogsync.cli register-object-backend
--db "${DB_PATH}"
--backend "${OBJECT_BACKEND_NAME}"
--bucket "${OBJECT_BUCKET}"
--endpoint "${OBJECT_ENDPOINT}"
--credential-env-prefix "${OBJECT_CREDENTIAL_ENV_PREFIX}"
)
if [[ -n "${OBJECT_REGION:-}" ]]; then
REGISTER_ARGS+=(--region "${OBJECT_REGION}")
fi
if [[ -n "${OBJECT_BASE_PREFIX:-}" ]]; then
REGISTER_ARGS+=(--base-prefix "${OBJECT_BASE_PREFIX}")
fi
if [[ -n "${OBJECT_ADDRESSING_STYLE:-}" ]]; then
REGISTER_ARGS+=(--addressing-style "${OBJECT_ADDRESSING_STYLE}")
fi
if [[ -n "${OBJECT_PUBLIC_BASE_URL:-}" ]]; then
REGISTER_ARGS+=(--public-base-url "${OBJECT_PUBLIC_BASE_URL}")
fi
"${PYTHON_BIN}" "${REGISTER_ARGS[@]}"
UPLOAD_ARGS=(
-m musicdl.catalogsync.cli upload
--db "${DB_PATH}"
--backend "${OBJECT_BACKEND_NAME}"
--workers "${UPLOAD_WORKERS:-4}"
)
if [[ -n "${UPLOAD_SOURCES:-}" ]]; then
UPLOAD_ARGS+=(--sources "${UPLOAD_SOURCES}")
fi
if [[ -n "${UPLOAD_PLAYLIST_IDS:-}" ]]; then
UPLOAD_ARGS+=(--playlist-ids "${UPLOAD_PLAYLIST_IDS}")
fi
if [[ -n "${UPLOAD_LIMIT:-}" ]]; then
UPLOAD_ARGS+=(--limit "${UPLOAD_LIMIT}")
fi
"${PYTHON_BIN}" "${UPLOAD_ARGS[@]}" "$@"