Initial import: Music_Server, MusicFree, catalog-sync
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
ROOT_DIR=/volume4/Music_Cloud
|
||||
APP_HOME=/volume4/Music_Cloud/catalogsync
|
||||
LIBRARY_DIR=/volume4/Music_Cloud/library
|
||||
|
||||
DB_PATH=/volume4/Music_Cloud/catalogsync/data/catalogsync.db
|
||||
INPUT_DIR=/volume4/Music_Cloud/catalogsync/inputs
|
||||
LOG_DIR=/volume4/Music_Cloud/catalogsync/logs
|
||||
ENV_FILE=/volume4/Music_Cloud/catalogsync/config/catalogsync.env
|
||||
WEB_HOST=127.0.0.1
|
||||
WEB_PORT=18080
|
||||
|
||||
PYTHON_BIN=python3
|
||||
VENV_DIR=/volume4/Music_Cloud/catalogsync/app/.venv
|
||||
|
||||
DOWNLOAD_LAYOUT=platform_first_artist
|
||||
DOWNLOAD_SOURCES=qq,kuwo,migu,qianqian,kugou,netease
|
||||
DOWNLOAD_WORKERS=10
|
||||
SYNC_WORKERS=4
|
||||
|
||||
OBJECT_BACKEND_NAME=main-s3
|
||||
OBJECT_BUCKET=music-bucket
|
||||
OBJECT_ENDPOINT=https://s3.example.com
|
||||
OBJECT_REGION=auto
|
||||
OBJECT_BASE_PREFIX=music
|
||||
OBJECT_ADDRESSING_STYLE=
|
||||
OBJECT_PUBLIC_BASE_URL=
|
||||
OBJECT_CREDENTIAL_ENV_PREFIX=CATALOGSYNC_MAIN_S3
|
||||
|
||||
UPLOAD_WORKERS=4
|
||||
UPLOAD_SOURCES=
|
||||
UPLOAD_PLAYLIST_IDS=
|
||||
UPLOAD_LIMIT=
|
||||
|
||||
CATALOGSYNC_MAIN_S3_ACCESS_KEY_ID=
|
||||
CATALOGSYNC_MAIN_S3_SECRET_ACCESS_KEY=
|
||||
CATALOGSYNC_MAIN_S3_SESSION_TOKEN=
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
RUN_DIR="${APP_HOME}/run"
|
||||
DEPLOY_DIR="${APP_HOME}/deploy"
|
||||
LOCK_DIR="${RUN_DIR}/deploy.lock"
|
||||
PID_FILE="${RUN_DIR}/serve.pid"
|
||||
TARGET_DIR="${APP_HOME}/app/musicdl/catalogsync"
|
||||
DEFAULT_STAGING_DIR="${APP_HOME}/deploy/staging/catalogsync"
|
||||
BACKUP_ROOT="${APP_HOME}/deploy/backups"
|
||||
|
||||
STAGING_DIR="${DEFAULT_STAGING_DIR}"
|
||||
HEALTH_URL=""
|
||||
HEALTH_RETRIES=45
|
||||
HEALTH_INTERVAL_SECONDS=1
|
||||
KEEP_BACKUPS=5
|
||||
SKIP_HEALTH_CHECK=0
|
||||
BACKUP_DIR=""
|
||||
HAS_BACKUP=0
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
$(basename "$0") [options]
|
||||
|
||||
Options:
|
||||
--staging-dir PATH Source directory to sync into app/musicdl/catalogsync
|
||||
--health-url URL Health-check URL (default: http://127.0.0.1:\${WEB_PORT}/dashboard)
|
||||
--health-retries N Max health-check retries (default: 45)
|
||||
--health-interval-sec N Health-check interval seconds (default: 1)
|
||||
--keep-backups N Number of backups to keep (default: 5)
|
||||
--skip-health-check Skip HTTP health check
|
||||
-h, --help Show help
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
echo "[deploy_and_restart.sh] $*"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "[deploy_and_restart.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_positive_integer() {
|
||||
local value="$1"
|
||||
local name="$2"
|
||||
if ! [[ "${value}" =~ ^[0-9]+$ ]] || (( value < 1 )); then
|
||||
fail "${name} must be a positive integer: ${value}"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_deploy_lock() {
|
||||
mkdir -p "${RUN_DIR}"
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner_pid=""
|
||||
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
|
||||
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
|
||||
fail "Another deploy is running (owner_pid=${owner_pid})"
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
fail "Cannot acquire deploy lock: ${LOCK_DIR}"
|
||||
fi
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
}
|
||||
|
||||
cleanup_lock() {
|
||||
rm -rf "${LOCK_DIR}"
|
||||
}
|
||||
|
||||
stop_service() {
|
||||
local pid=""
|
||||
if [[ -f "${PID_FILE}" ]]; then
|
||||
pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
|
||||
log "Stopping running service from PID file (pid=${pid})"
|
||||
kill -TERM "${pid}" 2>/dev/null || true
|
||||
for _ in $(seq 1 20); do
|
||||
if ! kill -0 "${pid}" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "${pid}" 2>/dev/null; then
|
||||
log "Service still alive; force killing pid=${pid}"
|
||||
kill -KILL "${pid}" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
|
||||
local serve_pattern="musicdl.catalogsync.cli serve"
|
||||
local wrapper_pattern="${APP_HOME}/bin/serve_console.sh"
|
||||
for _ in $(seq 1 10); do
|
||||
local serve_count
|
||||
local wrapper_count
|
||||
serve_count="$(count_matching_processes "${serve_pattern}")"
|
||||
wrapper_count="$(count_matching_processes "${wrapper_pattern}")"
|
||||
if [[ "${serve_count}" == "0" && "${wrapper_count}" == "0" ]]; then
|
||||
break
|
||||
fi
|
||||
kill_matching_processes "TERM" "${serve_pattern}"
|
||||
kill_matching_processes "TERM" "${wrapper_pattern}"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
kill_matching_processes "KILL" "${serve_pattern}"
|
||||
kill_matching_processes "KILL" "${wrapper_pattern}"
|
||||
}
|
||||
|
||||
start_service() {
|
||||
local launch_log="${LOG_DIR}/serve_console_launch_$(date +%Y%m%d_%H%M%S).log"
|
||||
nohup bash "${APP_HOME}/bin/serve_console.sh" >"${launch_log}" 2>&1 &
|
||||
local launcher_pid=$!
|
||||
log "Started service launcher pid=${launcher_pid}, launch_log=${launch_log}"
|
||||
}
|
||||
|
||||
sync_catalogsync() {
|
||||
if [[ ! -d "${STAGING_DIR}" ]]; then
|
||||
fail "Staging directory not found: ${STAGING_DIR}"
|
||||
fi
|
||||
if [[ ! -f "${STAGING_DIR}/__init__.py" ]]; then
|
||||
fail "Invalid staging directory (missing __init__.py): ${STAGING_DIR}"
|
||||
fi
|
||||
|
||||
mkdir -p "${BACKUP_ROOT}" "$(dirname "${TARGET_DIR}")"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/catalogsync_$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
if [[ -d "${TARGET_DIR}" ]]; then
|
||||
mv "${TARGET_DIR}" "${BACKUP_DIR}"
|
||||
HAS_BACKUP=1
|
||||
log "Backed up current catalogsync to ${BACKUP_DIR}"
|
||||
fi
|
||||
|
||||
cp -a "${STAGING_DIR}" "${TARGET_DIR}"
|
||||
log "Synced new catalogsync from ${STAGING_DIR} -> ${TARGET_DIR}"
|
||||
}
|
||||
|
||||
wait_health() {
|
||||
if (( SKIP_HEALTH_CHECK == 1 )); then
|
||||
log "Health check skipped by --skip-health-check"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
fail "curl is required for health check"
|
||||
fi
|
||||
|
||||
log "Health checking: ${HEALTH_URL}"
|
||||
for _ in $(seq 1 "${HEALTH_RETRIES}"); do
|
||||
local code
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' "${HEALTH_URL}" || true)"
|
||||
if [[ "${code}" == "200" ]]; then
|
||||
log "Health check passed (HTTP 200)"
|
||||
return 0
|
||||
fi
|
||||
sleep "${HEALTH_INTERVAL_SECONDS}"
|
||||
done
|
||||
|
||||
log "Health check failed: ${HEALTH_URL}"
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_single_instance() {
|
||||
local serve_count
|
||||
serve_count="$(count_matching_processes 'musicdl.catalogsync.cli serve')"
|
||||
if [[ "${serve_count}" != "1" ]]; then
|
||||
log "Unexpected serve process count: ${serve_count}"
|
||||
return 1
|
||||
fi
|
||||
log "Single-instance check passed (serve_count=${serve_count})"
|
||||
return 0
|
||||
}
|
||||
|
||||
list_matching_processes() {
|
||||
local pattern="$1"
|
||||
ps -ef | grep -F "${pattern}" | grep -v grep | awk '{print $2}' || true
|
||||
}
|
||||
|
||||
count_matching_processes() {
|
||||
local pattern="$1"
|
||||
list_matching_processes "${pattern}" | awk 'NF {count++} END {print count+0}'
|
||||
}
|
||||
|
||||
kill_matching_processes() {
|
||||
local signal_name="$1"
|
||||
local pattern="$2"
|
||||
local pid
|
||||
while IFS= read -r pid; do
|
||||
if [[ -n "${pid}" ]]; then
|
||||
kill "-${signal_name}" "${pid}" 2>/dev/null || true
|
||||
fi
|
||||
done < <(list_matching_processes "${pattern}")
|
||||
}
|
||||
|
||||
rollback() {
|
||||
log "Starting rollback..."
|
||||
stop_service
|
||||
if (( HAS_BACKUP == 0 )) || [[ ! -d "${BACKUP_DIR}" ]]; then
|
||||
log "No backup available; rollback skipped"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -rf "${TARGET_DIR}"
|
||||
mv "${BACKUP_DIR}" "${TARGET_DIR}"
|
||||
HAS_BACKUP=0
|
||||
log "Restored backup to ${TARGET_DIR}"
|
||||
|
||||
start_service
|
||||
if ! wait_health; then
|
||||
log "Rollback service failed health check"
|
||||
return 1
|
||||
fi
|
||||
verify_single_instance
|
||||
}
|
||||
|
||||
prune_backups() {
|
||||
if (( KEEP_BACKUPS < 1 )); then
|
||||
return 0
|
||||
fi
|
||||
if [[ ! -d "${BACKUP_ROOT}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
mapfile -t backups < <(ls -1dt "${BACKUP_ROOT}"/catalogsync_* 2>/dev/null || true)
|
||||
if (( ${#backups[@]} <= KEEP_BACKUPS )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for old_backup in "${backups[@]:KEEP_BACKUPS}"; do
|
||||
rm -rf "${old_backup}"
|
||||
log "Pruned old backup: ${old_backup}"
|
||||
done
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--staging-dir)
|
||||
STAGING_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-url)
|
||||
HEALTH_URL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-retries)
|
||||
HEALTH_RETRIES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-interval-sec)
|
||||
HEALTH_INTERVAL_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-backups)
|
||||
KEEP_BACKUPS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--skip-health-check)
|
||||
SKIP_HEALTH_CHECK=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
fail "Unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_positive_integer "${HEALTH_RETRIES}" "HEALTH_RETRIES"
|
||||
validate_positive_integer "${HEALTH_INTERVAL_SECONDS}" "HEALTH_INTERVAL_SECONDS"
|
||||
validate_positive_integer "${KEEP_BACKUPS}" "KEEP_BACKUPS"
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in WEB_PORT LOG_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -z "${HEALTH_URL}" ]]; then
|
||||
HEALTH_URL="http://127.0.0.1:${WEB_PORT}/dashboard"
|
||||
fi
|
||||
|
||||
mkdir -p "${DEPLOY_DIR}" "${RUN_DIR}" "${LOG_DIR}" "${BACKUP_ROOT}" "${APP_HOME}/app/musicdl"
|
||||
acquire_deploy_lock
|
||||
trap cleanup_lock EXIT INT TERM
|
||||
|
||||
LOG_FILE="${LOG_DIR}/deploy_and_restart_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
log "Starting deploy. staging=${STAGING_DIR}"
|
||||
log "Deploy log: ${LOG_FILE}"
|
||||
|
||||
if ! sync_catalogsync; then
|
||||
fail "Sync step failed"
|
||||
fi
|
||||
|
||||
stop_service
|
||||
start_service
|
||||
|
||||
if ! wait_health; then
|
||||
log "New version failed health check; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed; rollback failed."
|
||||
fi
|
||||
|
||||
if ! verify_single_instance; then
|
||||
log "Single-instance check failed; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed by single-instance check; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed by single-instance check; rollback failed."
|
||||
fi
|
||||
|
||||
prune_backups
|
||||
log "Deploy succeeded."
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[download_all.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/download_all_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[download_all.sh] logging to ${LOG_FILE}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
|
||||
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
|
||||
--db "${DB_PATH}" \
|
||||
--library-root "${LIBRARY_DIR}" \
|
||||
--workers "${DOWNLOAD_WORKERS:-10}" \
|
||||
"${EXTRA_ARGS[@]}" \
|
||||
"$@"
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
fail() {
|
||||
echo "[download_from_file.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
fail "usage: $0 <playlist-file> [extra args...]"
|
||||
fi
|
||||
|
||||
PLAYLIST_FILE="$1"
|
||||
shift
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
if [[ ! -f "${PLAYLIST_FILE}" ]]; then
|
||||
fail "playlist file not found: ${PLAYLIST_FILE}"
|
||||
fi
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/download_from_file_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[download_from_file.sh] logging to ${LOG_FILE}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
|
||||
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
|
||||
--db "${DB_PATH}" \
|
||||
--library-root "${LIBRARY_DIR}" \
|
||||
--playlist-file "${PLAYLIST_FILE}" \
|
||||
"${EXTRA_ARGS[@]}" \
|
||||
"$@"
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[install_runtime.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
require_requirement() {
|
||||
local requirement_name="$1"
|
||||
if ! grep -Eq "^${requirement_name}([<>=!~].*)?$" "${NAS_REQUIREMENTS_FILE}"; then
|
||||
fail "Missing required runtime dependency in requirements.txt: ${requirement_name}"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in APP_HOME DB_PATH LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
APP_DIR="${APP_HOME}/app"
|
||||
REQUIREMENTS_FILE="${APP_DIR}/requirements.txt"
|
||||
SETUP_FILE="${APP_DIR}/setup.py"
|
||||
if [[ ! -f "${REQUIREMENTS_FILE}" ]]; then
|
||||
fail "requirements.txt not found: ${REQUIREMENTS_FILE}"
|
||||
fi
|
||||
if [[ ! -f "${SETUP_FILE}" ]]; then
|
||||
fail "setup.py not found: ${SETUP_FILE}"
|
||||
fi
|
||||
|
||||
mkdir -p "${APP_HOME}" "${APP_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/install_runtime_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[install_runtime.sh] logging to ${LOG_FILE}"
|
||||
|
||||
if [[ ! -d "${VENV_DIR}" ]]; then
|
||||
"${PYTHON_BIN}" -m venv "${VENV_DIR}"
|
||||
fi
|
||||
|
||||
RUNTIME_PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
if [[ ! -x "${RUNTIME_PYTHON_BIN}" ]]; then
|
||||
fail "Virtualenv python not found after setup: ${RUNTIME_PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install --upgrade pip setuptools wheel
|
||||
|
||||
NAS_REQUIREMENTS_FILE="${APP_DIR}/requirements.nas.txt"
|
||||
grep -v '^nodejs-wheel$' "${REQUIREMENTS_FILE}" > "${NAS_REQUIREMENTS_FILE}"
|
||||
for runtime_requirement in fastapi uvicorn python-multipart; do
|
||||
require_requirement "${runtime_requirement}"
|
||||
done
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install -r "${NAS_REQUIREMENTS_FILE}"
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install --no-deps -e "${APP_DIR}"
|
||||
|
||||
echo "[install_runtime.sh] runtime ready: ${RUNTIME_PYTHON_BIN}"
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
trim_env_whitespace() {
|
||||
local value="$1"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
load_env_file() {
|
||||
local env_file="$1"
|
||||
local raw_line=""
|
||||
local normalized=""
|
||||
local key=""
|
||||
local value=""
|
||||
local trimmed_value=""
|
||||
local quote_char=""
|
||||
|
||||
[[ -f "${env_file}" ]] || return 1
|
||||
|
||||
while IFS= read -r raw_line || [[ -n "${raw_line}" ]]; do
|
||||
raw_line="${raw_line%$'\r'}"
|
||||
normalized="$(trim_env_whitespace "${raw_line}")"
|
||||
if [[ -z "${normalized}" || "${normalized:0:1}" == "#" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ "${normalized}" == export\ * ]]; then
|
||||
normalized="${normalized#export }"
|
||||
fi
|
||||
if [[ "${normalized}" != *=* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
key="$(trim_env_whitespace "${normalized%%=*}")"
|
||||
if [[ -z "${key}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
value="${normalized#*=}"
|
||||
trimmed_value="$(trim_env_whitespace "${value}")"
|
||||
if [[ ${#trimmed_value} -ge 2 ]]; then
|
||||
quote_char="${trimmed_value:0:1}"
|
||||
if [[ ( "${quote_char}" == "'" || "${quote_char}" == '"' ) && "${trimmed_value: -1}" == "${quote_char}" ]]; then
|
||||
value="${trimmed_value:1:${#trimmed_value}-2}"
|
||||
else
|
||||
value="${trimmed_value}"
|
||||
fi
|
||||
else
|
||||
value="${trimmed_value}"
|
||||
fi
|
||||
|
||||
printf -v "${key}" '%s' "${value}"
|
||||
export "${key}"
|
||||
done < "${env_file}"
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
RUN_DIR="${APP_HOME}/run"
|
||||
LOCK_DIR="${RUN_DIR}/serve.lock"
|
||||
PID_FILE="${RUN_DIR}/serve.pid"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[serve_console.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_serve_lock() {
|
||||
mkdir -p "${RUN_DIR}"
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner_pid=""
|
||||
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
|
||||
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
|
||||
fail "Another serve_console instance is running (owner_pid=${owner_pid})"
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
fail "Cannot acquire serve lock: ${LOCK_DIR}"
|
||||
fi
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
kill -TERM "${SERVER_PID}" 2>/dev/null || true
|
||||
wait "${SERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
rm -rf "${LOCK_DIR}"
|
||||
return "${exit_code}"
|
||||
}
|
||||
|
||||
validate_port() {
|
||||
local port_value="$1"
|
||||
if ! [[ "${port_value}" =~ ^[0-9]+$ ]] || (( port_value < 1 || port_value > 65535 )); then
|
||||
fail "WEB_PORT must be an integer in range 1..65535: ${port_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in DB_PATH ENV_FILE WEB_HOST WEB_PORT LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
validate_port "${WEB_PORT}"
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")" "$(dirname "${ENV_FILE}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
acquire_serve_lock
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
if [[ -f "${PID_FILE}" ]]; then
|
||||
existing_pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
|
||||
if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
|
||||
fail "Server process already running (pid=${existing_pid})"
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
fi
|
||||
|
||||
LOG_FILE="${LOG_DIR}/serve_console_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[serve_console.sh] logging to ${LOG_FILE}"
|
||||
|
||||
# Run from the deployed app directory so `python -m musicdl...` does not
|
||||
# accidentally import an older checkout from the current working directory.
|
||||
cd "${APP_HOME}/app"
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli serve \
|
||||
--db "${DB_PATH}" \
|
||||
--env-file "${ENV_FILE}" \
|
||||
--host "${WEB_HOST}" \
|
||||
--port "${WEB_PORT}" \
|
||||
"$@" &
|
||||
SERVER_PID=$!
|
||||
echo "${SERVER_PID}" > "${PID_FILE}"
|
||||
echo "[serve_console.sh] server pid=${SERVER_PID}"
|
||||
wait "${SERVER_PID}"
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[upload_all.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in DB_PATH LOG_DIR PYTHON_BIN VENV_DIR OBJECT_BACKEND_NAME OBJECT_BUCKET OBJECT_ENDPOINT OBJECT_CREDENTIAL_ENV_PREFIX; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
ACCESS_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_ACCESS_KEY_ID"
|
||||
SECRET_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_SECRET_ACCESS_KEY"
|
||||
require_var "${ACCESS_KEY_VAR}"
|
||||
require_var "${SECRET_KEY_VAR}"
|
||||
|
||||
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/upload_all_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[upload_all.sh] logging to ${LOG_FILE}"
|
||||
|
||||
REGISTER_ARGS=(
|
||||
-m musicdl.catalogsync.cli register-object-backend
|
||||
--db "${DB_PATH}"
|
||||
--backend "${OBJECT_BACKEND_NAME}"
|
||||
--bucket "${OBJECT_BUCKET}"
|
||||
--endpoint "${OBJECT_ENDPOINT}"
|
||||
--credential-env-prefix "${OBJECT_CREDENTIAL_ENV_PREFIX}"
|
||||
)
|
||||
|
||||
if [[ -n "${OBJECT_REGION:-}" ]]; then
|
||||
REGISTER_ARGS+=(--region "${OBJECT_REGION}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_BASE_PREFIX:-}" ]]; then
|
||||
REGISTER_ARGS+=(--base-prefix "${OBJECT_BASE_PREFIX}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_ADDRESSING_STYLE:-}" ]]; then
|
||||
REGISTER_ARGS+=(--addressing-style "${OBJECT_ADDRESSING_STYLE}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_PUBLIC_BASE_URL:-}" ]]; then
|
||||
REGISTER_ARGS+=(--public-base-url "${OBJECT_PUBLIC_BASE_URL}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" "${REGISTER_ARGS[@]}"
|
||||
|
||||
UPLOAD_ARGS=(
|
||||
-m musicdl.catalogsync.cli upload
|
||||
--db "${DB_PATH}"
|
||||
--backend "${OBJECT_BACKEND_NAME}"
|
||||
--workers "${UPLOAD_WORKERS:-4}"
|
||||
)
|
||||
|
||||
if [[ -n "${UPLOAD_SOURCES:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--sources "${UPLOAD_SOURCES}")
|
||||
fi
|
||||
if [[ -n "${UPLOAD_PLAYLIST_IDS:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--playlist-ids "${UPLOAD_PLAYLIST_IDS}")
|
||||
fi
|
||||
if [[ -n "${UPLOAD_LIMIT:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--limit "${UPLOAD_LIMIT}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" "${UPLOAD_ARGS[@]}" "$@"
|
||||
Reference in New Issue
Block a user