musicdl-catalog-sync-suite/Music_Server/scripts/templates/deploy_and_restart.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
RUN_DIR="${APP_HOME}/run"
DEPLOY_DIR="${APP_HOME}/deploy"
LOCK_DIR="${RUN_DIR}/deploy.lock"
APP_DIR="${APP_HOME}/app"
CONFIG_DIR="${APP_HOME}/config"
DATA_DIR="${APP_HOME}/data"
CONFIG_FILE="${CONFIG_DIR}/music_server.env"
LEGACY_CONFIG_FILE="${APP_DIR}/config/music_server.env"
LEGACY_CONFIG_EXAMPLE="${APP_DIR}/config/music_server.env.example"
LEGACY_DATA_DIR="${APP_DIR}/data"
DEFAULT_STAGING_DIR="${APP_HOME}/deploy/staging/music-server-app"
BACKUP_ROOT="${APP_HOME}/deploy/backups"
COMPOSE_FILE="${APP_DIR}/docker-compose.nas.yml"
DOCKER_BIN_DIR="/var/packages/Docker/target/usr/bin"

STAGING_DIR="${DEFAULT_STAGING_DIR}"
HEALTH_URL="http://127.0.0.1:18081/healthz"
HEALTH_RETRIES=45
HEALTH_INTERVAL_SECONDS=2
KEEP_BACKUPS=3
SKIP_HEALTH_CHECK=0
BACKUP_DIR=""
HAS_BACKUP=0

usage() {
  cat <<EOF
Usage:
  $(basename "$0") [options]

Options:
  --staging-dir PATH         Repo staging directory to deploy into ${APP_HOME}/app
  --health-url URL           Health-check URL (default: ${HEALTH_URL})
  --health-retries N         Max health-check retries (default: 45)
  --health-interval-sec N    Health-check interval seconds (default: 2)
  --keep-backups N           Number of app backups to keep (default: 3)
  --skip-health-check        Skip HTTP health check
  -h, --help                 Show help
EOF
}

log() {
  echo "[deploy_and_restart.sh] $*"
}

fail() {
  echo "[deploy_and_restart.sh] ERROR: $*" >&2
  exit 1
}

validate_positive_integer() {
  local value="$1"
  local name="$2"
  if ! [[ "${value}" =~ ^[0-9]+$ ]] || (( value < 1 )); then
    fail "${name} must be a positive integer: ${value}"
  fi
}

acquire_deploy_lock() {
  mkdir -p "${RUN_DIR}"
  if mkdir "${LOCK_DIR}" 2>/dev/null; then
    echo "$$" > "${LOCK_DIR}/owner_pid"
    return 0
  fi

  local owner_pid=""
  if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
    owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
  fi
  if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
    fail "Another deploy is running (owner_pid=${owner_pid})"
  fi

  rm -rf "${LOCK_DIR}"
  if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
    fail "Cannot acquire deploy lock: ${LOCK_DIR}"
  fi
  echo "$$" > "${LOCK_DIR}/owner_pid"
}

cleanup_lock() {
  rm -rf "${LOCK_DIR}"
}

ensure_runtime_layout() {
  mkdir -p "${CONFIG_DIR}" "${DATA_DIR}" "${DEPLOY_DIR}" "${BACKUP_ROOT}" "${RUN_DIR}"

  if [[ ! -f "${CONFIG_FILE}" ]]; then
    if [[ -f "${LEGACY_CONFIG_FILE}" ]]; then
      cp -a "${LEGACY_CONFIG_FILE}" "${CONFIG_FILE}"
      log "Copied legacy runtime config to ${CONFIG_FILE}"
    elif [[ -f "${LEGACY_CONFIG_EXAMPLE}" ]]; then
      cp -a "${LEGACY_CONFIG_EXAMPLE}" "${CONFIG_FILE}"
      log "Bootstrapped runtime config from example: ${CONFIG_FILE}"
    fi
  fi

  if [[ -d "${LEGACY_DATA_DIR}" ]] && [[ -z "$(ls -A "${DATA_DIR}" 2>/dev/null || true)" ]]; then
    cp -a "${LEGACY_DATA_DIR}/." "${DATA_DIR}/"
    log "Copied legacy runtime data into ${DATA_DIR}"
  fi
}

require_staging_checkout() {
  if [[ ! -d "${STAGING_DIR}" ]]; then
    fail "Staging directory not found: ${STAGING_DIR}"
  fi
  if [[ ! -f "${STAGING_DIR}/docker-compose.nas.yml" ]]; then
    fail "Invalid staging checkout (missing docker-compose.nas.yml): ${STAGING_DIR}"
  fi
  if [[ ! -f "${STAGING_DIR}/Dockerfile" ]]; then
    fail "Invalid staging checkout (missing Dockerfile): ${STAGING_DIR}"
  fi
}

sync_app_checkout() {
  require_staging_checkout

  BACKUP_DIR="${BACKUP_ROOT}/app_$(date +%Y%m%d_%H%M%S)"
  if [[ -d "${APP_DIR}" ]]; then
    mv "${APP_DIR}" "${BACKUP_DIR}"
    HAS_BACKUP=1
    log "Backed up current app to ${BACKUP_DIR}"
  fi

  cp -a "${STAGING_DIR}" "${APP_DIR}"
  log "Synced new app checkout from ${STAGING_DIR} -> ${APP_DIR}"
}

run_compose() {
  export PATH="${DOCKER_BIN_DIR}:${PATH}"
  cd "${APP_DIR}"
  docker-compose -f "${COMPOSE_FILE}" "$@"
}

remove_conflicting_named_containers() {
  export PATH="${DOCKER_BIN_DIR}:${PATH}"
  docker rm -f music-server >/dev/null 2>&1 || true
}

restart_service() {
  run_compose build catalog-export music-server
  remove_conflicting_named_containers
  run_compose down --remove-orphans || true
  run_compose run --rm catalog-export
  run_compose up -d music-server
}

wait_health() {
  if (( SKIP_HEALTH_CHECK == 1 )); then
    log "Health check skipped by --skip-health-check"
    return 0
  fi

  if ! command -v curl >/dev/null 2>&1; then
    fail "curl is required for health check"
  fi

  log "Health checking: ${HEALTH_URL}"
  for _ in $(seq 1 "${HEALTH_RETRIES}"); do
    if curl -fsS "${HEALTH_URL}" >/dev/null; then
      log "Health check passed"
      return 0
    fi
    sleep "${HEALTH_INTERVAL_SECONDS}"
  done

  log "Health check failed: ${HEALTH_URL}"
  return 1
}

rollback() {
  log "Starting rollback..."
  if (( HAS_BACKUP == 0 )) || [[ ! -d "${BACKUP_DIR}" ]]; then
    log "No backup available; rollback skipped"
    return 1
  fi

  rm -rf "${APP_DIR}"
  mv "${BACKUP_DIR}" "${APP_DIR}"
  HAS_BACKUP=0
  log "Restored backup to ${APP_DIR}"

  restart_service
  wait_health
}

prune_backups() {
  if (( KEEP_BACKUPS < 1 )) || [[ ! -d "${BACKUP_ROOT}" ]]; then
    return 0
  fi

  mapfile -t backups < <(ls -1dt "${BACKUP_ROOT}"/app_* 2>/dev/null || true)
  if (( ${#backups[@]} <= KEEP_BACKUPS )); then
    return 0
  fi

  for old_backup in "${backups[@]:KEEP_BACKUPS}"; do
    rm -rf "${old_backup}"
    log "Pruned old backup: ${old_backup}"
  done
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --staging-dir)
      STAGING_DIR="${2:-}"
      shift 2
      ;;
    --health-url)
      HEALTH_URL="${2:-}"
      shift 2
      ;;
    --health-retries)
      HEALTH_RETRIES="${2:-}"
      shift 2
      ;;
    --health-interval-sec)
      HEALTH_INTERVAL_SECONDS="${2:-}"
      shift 2
      ;;
    --keep-backups)
      KEEP_BACKUPS="${2:-}"
      shift 2
      ;;
    --skip-health-check)
      SKIP_HEALTH_CHECK=1
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      fail "Unknown argument: $1"
      ;;
  esac
done

validate_positive_integer "${HEALTH_RETRIES}" "HEALTH_RETRIES"
validate_positive_integer "${HEALTH_INTERVAL_SECONDS}" "HEALTH_INTERVAL_SECONDS"
validate_positive_integer "${KEEP_BACKUPS}" "KEEP_BACKUPS"

ensure_runtime_layout
acquire_deploy_lock
trap cleanup_lock EXIT INT TERM

LOG_FILE="${DEPLOY_DIR}/deploy_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "${LOG_FILE}") 2>&1
log "Starting deploy. staging=${STAGING_DIR}"
log "App home: ${APP_HOME}"
log "Runtime config: ${CONFIG_FILE}"
log "Runtime data: ${DATA_DIR}"
log "Deploy log: ${LOG_FILE}"

if ! sync_app_checkout; then
  fail "Sync step failed"
fi

ensure_runtime_layout
if [[ ! -f "${CONFIG_FILE}" ]]; then
  fail "Runtime config not found: ${CONFIG_FILE}"
fi

if ! restart_service; then
  log "Restart failed; attempting rollback."
  if rollback; then
    fail "Deploy failed during restart; rollback succeeded."
  fi
  fail "Deploy failed during restart; rollback failed."
fi

if ! wait_health; then
  log "New version failed health check; attempting rollback."
  if rollback; then
    fail "Deploy failed during health check; rollback succeeded."
  fi
  fail "Deploy failed during health check; rollback failed."
fi

prune_backups
log "Deploy succeeded."