Initial import: Music_Server, MusicFree, catalog-sync
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of JooxMusicClient Cookies Builder
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
import time
|
||||
import hashlib
|
||||
import requests
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
'''settings'''
|
||||
USERNAME = 'Your Email Here'
|
||||
PASSWORD = 'Your Password Here'
|
||||
|
||||
|
||||
'''buildjooxcookies'''
|
||||
def buildjooxcookies():
|
||||
session, epoch = requests.Session(), int(time.time()) - 60
|
||||
encoded_email, md5_password = quote(quote(USERNAME)), hashlib.md5(PASSWORD.encode('utf-8')).hexdigest()
|
||||
url_auth = f"https://api.joox.com/web-fcgi-bin/web_wmauth?country=id&lang=id&wxopenid={encoded_email}&password={md5_password}&wmauth_type=0&authtype=2&time={epoch}294&_={epoch}295&callback=axiosJsonpCallback1"
|
||||
resp = session.get(url_auth)
|
||||
cookies: dict = requests.utils.dict_from_cookiejar(resp.cookies)
|
||||
cookies.update(requests.utils.dict_from_cookiejar(session.cookies))
|
||||
return cookies
|
||||
|
||||
|
||||
'''tests'''
|
||||
if __name__ == '__main__':
|
||||
print(buildjooxcookies())
|
||||
@@ -0,0 +1,58 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of KugouMusicClient Cookies Builder
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
import time
|
||||
import qrcode
|
||||
import requests
|
||||
from musicdl.modules.utils.kugouutils import KugouMusicClientUtils, APPID, safeextractfromdict
|
||||
|
||||
|
||||
'''settings'''
|
||||
session, cookies = requests.Session(), KugouMusicClientUtils.initdevice()
|
||||
|
||||
|
||||
'''loginqrkey'''
|
||||
def loginqrkey(use_web: bool = False):
|
||||
qr_appid = 1014 if use_web else 1001
|
||||
params = {"appid": qr_appid, "type": 1, "plat": 4, "qrcode_txt": f"https://h5.kugou.com/apps/loginQRCode/html/index.html?appid={APPID}&", "srcappid": 2919}
|
||||
return KugouMusicClientUtils.sendrequest(session, "GET", "/v2/qrcode", params=params, base_url="https://login-user.kugou.com", encrypt_type="web", cookies=cookies)
|
||||
|
||||
|
||||
'''loginqrcheck'''
|
||||
def loginqrcheck(key: str):
|
||||
params = {"plat": 4, "appid": APPID, "srcappid": 2919, "qrcode": key}
|
||||
result = KugouMusicClientUtils.sendrequest(session, "GET", "/v2/get_userinfo_qrcode", params=params, base_url="https://login-user.kugou.com", encrypt_type="web", cookies=cookies)
|
||||
if isinstance(result, dict) and safeextractfromdict(result, ['data', 'status'], None) == 4:
|
||||
token = safeextractfromdict(result, ['data', 'token'], None)
|
||||
userid = safeextractfromdict(result, ['data', 'userid'], None)
|
||||
if token: cookies["token"] = token
|
||||
if userid: cookies["userid"] = str(userid)
|
||||
return result
|
||||
|
||||
|
||||
'''buildkugoucookies'''
|
||||
def buildkugoucookies():
|
||||
# prepare for scan qr code
|
||||
qr_resp = loginqrkey()
|
||||
qr_key = qr_resp["data"]["qrcode"]
|
||||
img = qrcode.make(f"https://h5.kugou.com/apps/loginQRCode/html/index.html?qrcode={qr_key}")
|
||||
img.save("kugou_login_qr.png"); img.show()
|
||||
# wait for scan
|
||||
while True:
|
||||
check = loginqrcheck(qr_key)
|
||||
if safeextractfromdict(check, ['data', 'status'], None) == 4: break
|
||||
time.sleep(2)
|
||||
# register device
|
||||
KugouMusicClientUtils.registerdevice(session, cookies)
|
||||
# return
|
||||
return cookies
|
||||
|
||||
|
||||
'''tests'''
|
||||
if __name__ == '__main__':
|
||||
print(buildkugoucookies())
|
||||
@@ -0,0 +1,33 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of QingtingMusicClient Cookies Builder
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
import warnings
|
||||
import requests
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
'''settings'''
|
||||
USERNAME = 'Your Phone Number Here'
|
||||
PASSWORD = 'Your Password Here'
|
||||
|
||||
|
||||
'''buildqingtingfmcookies'''
|
||||
def buildqingtingfmcookies():
|
||||
data = {'account_type': '5', 'device_id': 'web', 'user_id': USERNAME, 'password': PASSWORD}
|
||||
headers = {
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36'
|
||||
}
|
||||
resp = requests.post('https://u2.qingting.fm/u2/api/v4/user/login', headers=headers, data=data, verify=False)
|
||||
resp.raise_for_status()
|
||||
raw_data = resp.json()['data']
|
||||
return {'qingting_id': raw_data['qingting_id'], 'access_token': raw_data['access_token'], 'refresh_token': raw_data['refresh_token']}
|
||||
|
||||
|
||||
'''tests'''
|
||||
if __name__ == '__main__':
|
||||
print(buildqingtingfmcookies())
|
||||
@@ -0,0 +1,38 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of QobuzMusicClient Cookies Builder
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
import re
|
||||
import requests
|
||||
from urllib.parse import urljoin
|
||||
|
||||
|
||||
'''settings'''
|
||||
USERNAME = 'Your Email or UserID Here'
|
||||
PASSWORD = 'Your Password or Token Here'
|
||||
LOGIN_BY_PASSWORD = True # modify as False if you use token to login in Qobuz
|
||||
|
||||
|
||||
'''buildqobuzcookies'''
|
||||
def buildqobuzcookies():
|
||||
(session := requests.Session()).headers.update({"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"})
|
||||
(resp := session.get("https://play.qobuz.com/login")).raise_for_status()
|
||||
bundle_url = re.search(r'<script src="(/resources/\d+\.\d+\.\d+-[a-z]\d{3}/bundle\.js)"></script>', resp.text).group(1)
|
||||
(resp := session.get(urljoin("https://play.qobuz.com", bundle_url))).raise_for_status()
|
||||
app_id = str(re.search(r'production:{api:{appId:"(?P<app_id>\d{9})",appSecret:"(\w{32})', resp.text).group("app_id"))
|
||||
session.headers.update({"X-App-Id": str(app_id)})
|
||||
params = {"user_id": USERNAME, "user_auth_token": PASSWORD, "app_id": str(app_id),} if not LOGIN_BY_PASSWORD else {"email": USERNAME, "password": PASSWORD, "app_id": str(app_id),}
|
||||
(resp := session.get("https://www.qobuz.com/api.json/0.2/user/login", params=params)).raise_for_status()
|
||||
cookies: dict = requests.utils.dict_from_cookiejar(resp.cookies)
|
||||
cookies.update(requests.utils.dict_from_cookiejar(session.cookies))
|
||||
cookies.update(resp.json()); cookies['x-user-auth-token'] = cookies['user_auth_token']
|
||||
return cookies
|
||||
|
||||
|
||||
'''tests'''
|
||||
if __name__ == '__main__':
|
||||
print(buildqobuzcookies())
|
||||
@@ -0,0 +1,21 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of KugouMusicClient Cookies Builder
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
from musicdl.modules.utils.tidalutils import TidalTvSession
|
||||
|
||||
|
||||
'''buildtidalcookies'''
|
||||
def buildtidalcookies():
|
||||
cli = TidalTvSession()
|
||||
cli.auth()
|
||||
return cli.getstorage().tojson()
|
||||
|
||||
|
||||
'''tests'''
|
||||
if __name__ == '__main__':
|
||||
print(buildtidalcookies())
|
||||
@@ -0,0 +1,92 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RemoteHost,
|
||||
[int]$Port = 22,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$User,
|
||||
[string]$RootDir = "/volume4/Music_Cloud"
|
||||
)
|
||||
|
||||
function Quote-RemotePathForPosixShell {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Path
|
||||
)
|
||||
|
||||
$EscapedPath = $Path.Replace("'", "'\''")
|
||||
return "'$EscapedPath'"
|
||||
}
|
||||
|
||||
function New-ScpRemoteTarget {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Remote,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Path
|
||||
)
|
||||
|
||||
return "${Remote}:$(Quote-RemotePathForPosixShell -Path $Path)"
|
||||
}
|
||||
|
||||
$AppHome = "$RootDir/catalogsync"
|
||||
$RemoteDirs = @(
|
||||
$RootDir,
|
||||
"$RootDir/library",
|
||||
"$AppHome/app",
|
||||
"$AppHome/bin",
|
||||
"$AppHome/config",
|
||||
"$AppHome/data",
|
||||
"$AppHome/inputs",
|
||||
"$AppHome/logs"
|
||||
)
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$ProjectRoot = Resolve-Path (Join-Path $ScriptDir "..\..")
|
||||
$Remote = "$User@$RemoteHost"
|
||||
|
||||
$QuotedDirs = $RemoteDirs | ForEach-Object { Quote-RemotePathForPosixShell -Path $_ }
|
||||
$RemoteMkdirCommand = "mkdir -p " + ($QuotedDirs -join " ")
|
||||
ssh -p $Port $Remote $RemoteMkdirCommand
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to create remote directories."
|
||||
}
|
||||
|
||||
$AppSources = @(
|
||||
(Join-Path $ProjectRoot "musicdl"),
|
||||
(Join-Path $ProjectRoot "setup.py"),
|
||||
(Join-Path $ProjectRoot "README.md"),
|
||||
(Join-Path $ProjectRoot "LICENSE"),
|
||||
(Join-Path $ProjectRoot "requirements.txt"),
|
||||
(Join-Path $ProjectRoot "requirements-optional.txt")
|
||||
)
|
||||
$RemoteAppDirTarget = New-ScpRemoteTarget -Remote $Remote -Path "$AppHome/app/"
|
||||
foreach ($Source in $AppSources) {
|
||||
scp -P $Port -r $Source $RemoteAppDirTarget
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to copy app source: $Source"
|
||||
}
|
||||
}
|
||||
|
||||
$BinTemplates = @(
|
||||
(Join-Path $ScriptDir "templates\download_all.sh"),
|
||||
(Join-Path $ScriptDir "templates\download_from_file.sh"),
|
||||
(Join-Path $ScriptDir "templates\upload_all.sh"),
|
||||
(Join-Path $ScriptDir "templates\install_runtime.sh"),
|
||||
(Join-Path $ScriptDir "templates\serve_console.sh"),
|
||||
(Join-Path $ScriptDir "templates\deploy_and_restart.sh")
|
||||
)
|
||||
$RemoteBinDirTarget = New-ScpRemoteTarget -Remote $Remote -Path "$AppHome/bin/"
|
||||
foreach ($Template in $BinTemplates) {
|
||||
scp -P $Port $Template $RemoteBinDirTarget
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to copy bin template: $Template"
|
||||
}
|
||||
}
|
||||
|
||||
$RemoteEnvExample = "$AppHome/config/catalogsync.env.example"
|
||||
$LocalEnvExample = Join-Path $ScriptDir "templates\catalogsync.env.example"
|
||||
$RemoteEnvExampleTarget = New-ScpRemoteTarget -Remote $Remote -Path $RemoteEnvExample
|
||||
scp -P $Port $LocalEnvExample $RemoteEnvExampleTarget
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to copy catalogsync.env.example."
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
param(
|
||||
[string]$HostName = "192.168.5.43",
|
||||
[int]$Port = 222,
|
||||
[string]$User = "xiaoming",
|
||||
[string]$RemoteAppHome = "/volume4/Music_Cloud/catalogsync",
|
||||
[string]$Password = $(if ($env:NAS_192168543_PASSWORD) { $env:NAS_192168543_PASSWORD } else { "Nie@159357" }),
|
||||
[switch]$SkipHealthCheck
|
||||
)
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$PythonScript = Join-Path $ScriptDir "deploy_to_nas.py"
|
||||
|
||||
if (-not (Test-Path -LiteralPath $PythonScript)) {
|
||||
throw "Python script not found: $PythonScript"
|
||||
}
|
||||
|
||||
$arguments = @(
|
||||
$PythonScript,
|
||||
"--host", $HostName,
|
||||
"--port", "$Port",
|
||||
"--user", $User,
|
||||
"--remote-app-home", $RemoteAppHome
|
||||
)
|
||||
|
||||
if ($Password) {
|
||||
$arguments += @("--password", $Password)
|
||||
}
|
||||
if ($SkipHealthCheck) {
|
||||
$arguments += "--skip-health-check"
|
||||
}
|
||||
|
||||
python @arguments
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
import paramiko
|
||||
|
||||
DEFAULT_PASSWORD = "Nie@159357"
|
||||
SKIP_DIR_NAMES = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".git"}
|
||||
SKIP_FILE_SUFFIXES = {".pyc", ".pyo", ".DS_Store"}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
project_root = script_dir.parent.parent
|
||||
default_source_dir = project_root / "musicdl" / "catalogsync"
|
||||
default_template_dir = script_dir / "templates"
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload catalogsync to NAS staging and trigger deploy_and_restart.sh"
|
||||
)
|
||||
parser.add_argument("--host", default="192.168.5.43")
|
||||
parser.add_argument("--port", type=int, default=222)
|
||||
parser.add_argument("--user", default="xiaoming")
|
||||
parser.add_argument(
|
||||
"--password",
|
||||
default=os.environ.get("NAS_192168543_PASSWORD") or DEFAULT_PASSWORD,
|
||||
)
|
||||
parser.add_argument("--remote-app-home", default="/volume4/Music_Cloud/catalogsync")
|
||||
parser.add_argument("--source-dir", default=str(default_source_dir))
|
||||
parser.add_argument("--template-dir", default=str(default_template_dir))
|
||||
parser.add_argument("--skip-health-check", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def to_sftp_path(shell_path: str) -> str:
|
||||
normalized = shell_path.rstrip("/")
|
||||
if normalized.startswith("/volume4"):
|
||||
mapped = normalized[len("/volume4") :]
|
||||
return mapped or "/"
|
||||
return normalized or "/"
|
||||
|
||||
|
||||
def shell_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
def ensure_remote_dir(sftp: paramiko.SFTPClient, path: str) -> None:
|
||||
path = path.rstrip("/")
|
||||
if not path:
|
||||
return
|
||||
if path == "/":
|
||||
return
|
||||
parts: list[str] = []
|
||||
current = path
|
||||
while current not in ("", "/"):
|
||||
parts.append(current)
|
||||
current = posixpath.dirname(current)
|
||||
for part in reversed(parts):
|
||||
try:
|
||||
sftp.stat(part)
|
||||
except OSError:
|
||||
sftp.mkdir(part)
|
||||
|
||||
|
||||
def exists_remote(sftp: paramiko.SFTPClient, path: str) -> bool:
|
||||
try:
|
||||
sftp.stat(path)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def remove_remote_tree(sftp: paramiko.SFTPClient, root: str) -> None:
|
||||
if not exists_remote(sftp, root):
|
||||
return
|
||||
for entry in sftp.listdir_attr(root):
|
||||
child = posixpath.join(root, entry.filename)
|
||||
if stat.S_ISDIR(entry.st_mode):
|
||||
remove_remote_tree(sftp, child)
|
||||
else:
|
||||
sftp.remove(child)
|
||||
sftp.rmdir(root)
|
||||
|
||||
|
||||
def iter_local_files(local_root: Path) -> Iterable[Path]:
|
||||
for current_root, dir_names, file_names in os.walk(local_root):
|
||||
dir_names[:] = [name for name in dir_names if name not in SKIP_DIR_NAMES]
|
||||
for file_name in file_names:
|
||||
path = Path(current_root) / file_name
|
||||
if path.suffix in SKIP_FILE_SUFFIXES:
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def upload_tree(sftp: paramiko.SFTPClient, local_root: Path, remote_root: str) -> int:
|
||||
ensure_remote_dir(sftp, remote_root)
|
||||
uploaded = 0
|
||||
for local_file in iter_local_files(local_root):
|
||||
relative_path = local_file.relative_to(local_root).as_posix()
|
||||
remote_file = posixpath.join(remote_root, relative_path)
|
||||
remote_dir = posixpath.dirname(remote_file)
|
||||
ensure_remote_dir(sftp, remote_dir)
|
||||
sftp.put(str(local_file), remote_file)
|
||||
uploaded += 1
|
||||
return uploaded
|
||||
|
||||
|
||||
def read_channel_text(channel_file) -> str:
|
||||
data = channel_file.read()
|
||||
if isinstance(data, bytes):
|
||||
return data.decode("utf-8", "replace")
|
||||
return str(data)
|
||||
|
||||
|
||||
def run_remote_command(client: paramiko.SSHClient, command: str) -> int:
|
||||
stdin, stdout, stderr = client.exec_command(command)
|
||||
_ = stdin
|
||||
out_text = read_channel_text(stdout)
|
||||
err_text = read_channel_text(stderr)
|
||||
if out_text:
|
||||
print(out_text, end="")
|
||||
if err_text:
|
||||
print(err_text, file=sys.stderr, end="")
|
||||
return stdout.channel.recv_exit_status()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
source_dir = Path(args.source_dir).resolve()
|
||||
template_dir = Path(args.template_dir).resolve()
|
||||
|
||||
if not source_dir.exists():
|
||||
print(f"Source dir not found: {source_dir}", file=sys.stderr)
|
||||
return 2
|
||||
if not (template_dir / "serve_console.sh").exists():
|
||||
print(f"Missing template: {template_dir / 'serve_console.sh'}", file=sys.stderr)
|
||||
return 2
|
||||
if not (template_dir / "deploy_and_restart.sh").exists():
|
||||
print(f"Missing template: {template_dir / 'deploy_and_restart.sh'}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
remote_app_home_shell = args.remote_app_home.rstrip("/")
|
||||
remote_app_home_sftp = to_sftp_path(remote_app_home_shell)
|
||||
remote_staging_shell = f"{remote_app_home_shell}/deploy/staging/catalogsync"
|
||||
remote_staging_sftp = to_sftp_path(remote_staging_shell)
|
||||
remote_bin_sftp = to_sftp_path(f"{remote_app_home_shell}/bin")
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
print(
|
||||
f"[deploy_to_nas] Connecting {args.user}@{args.host}:{args.port} ...",
|
||||
flush=True,
|
||||
)
|
||||
client.connect(
|
||||
hostname=args.host,
|
||||
port=args.port,
|
||||
username=args.user,
|
||||
password=args.password,
|
||||
timeout=15,
|
||||
banner_timeout=15,
|
||||
auth_timeout=15,
|
||||
)
|
||||
|
||||
try:
|
||||
sftp = client.open_sftp()
|
||||
try:
|
||||
ensure_remote_dir(sftp, remote_bin_sftp)
|
||||
if exists_remote(sftp, remote_staging_sftp):
|
||||
print(
|
||||
f"[deploy_to_nas] Clearing staging: {remote_staging_sftp}",
|
||||
flush=True,
|
||||
)
|
||||
remove_remote_tree(sftp, remote_staging_sftp)
|
||||
ensure_remote_dir(sftp, remote_staging_sftp)
|
||||
uploaded_count = upload_tree(sftp, source_dir, remote_staging_sftp)
|
||||
print(
|
||||
f"[deploy_to_nas] Uploaded {uploaded_count} files to staging.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
sftp.put(
|
||||
str(template_dir / "serve_console.sh"),
|
||||
posixpath.join(remote_bin_sftp, "serve_console.sh"),
|
||||
)
|
||||
sftp.put(
|
||||
str(template_dir / "deploy_and_restart.sh"),
|
||||
posixpath.join(remote_bin_sftp, "deploy_and_restart.sh"),
|
||||
)
|
||||
finally:
|
||||
sftp.close()
|
||||
|
||||
chmod_cmd = (
|
||||
f"chmod +x {shell_quote(remote_app_home_shell + '/bin/serve_console.sh')} "
|
||||
f"{shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')}"
|
||||
)
|
||||
chmod_exit = run_remote_command(client, chmod_cmd)
|
||||
if chmod_exit != 0:
|
||||
print(
|
||||
f"[deploy_to_nas] chmod failed with exit code {chmod_exit}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return chmod_exit
|
||||
|
||||
deploy_cmd = (
|
||||
f"bash {shell_quote(remote_app_home_shell + '/bin/deploy_and_restart.sh')} "
|
||||
f"--staging-dir {shell_quote(remote_staging_shell)}"
|
||||
)
|
||||
if args.skip_health_check:
|
||||
deploy_cmd += " --skip-health-check"
|
||||
|
||||
print(f"[deploy_to_nas] Running deploy command...", flush=True)
|
||||
deploy_exit = run_remote_command(client, deploy_cmd)
|
||||
if deploy_exit != 0:
|
||||
print(
|
||||
f"[deploy_to_nas] Deploy failed with exit code {deploy_exit}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return deploy_exit
|
||||
|
||||
print("[deploy_to_nas] Deploy completed successfully.", flush=True)
|
||||
return 0
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from musicdl.catalogsync.suspected_live import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
ROOT_DIR=/volume4/Music_Cloud
|
||||
APP_HOME=/volume4/Music_Cloud/catalogsync
|
||||
LIBRARY_DIR=/volume4/Music_Cloud/library
|
||||
|
||||
DB_PATH=/volume4/Music_Cloud/catalogsync/data/catalogsync.db
|
||||
INPUT_DIR=/volume4/Music_Cloud/catalogsync/inputs
|
||||
LOG_DIR=/volume4/Music_Cloud/catalogsync/logs
|
||||
ENV_FILE=/volume4/Music_Cloud/catalogsync/config/catalogsync.env
|
||||
WEB_HOST=127.0.0.1
|
||||
WEB_PORT=18080
|
||||
|
||||
PYTHON_BIN=python3
|
||||
VENV_DIR=/volume4/Music_Cloud/catalogsync/app/.venv
|
||||
|
||||
DOWNLOAD_LAYOUT=platform_first_artist
|
||||
DOWNLOAD_SOURCES=qq,kuwo,migu,qianqian,kugou,netease
|
||||
DOWNLOAD_WORKERS=10
|
||||
SYNC_WORKERS=4
|
||||
|
||||
OBJECT_BACKEND_NAME=main-s3
|
||||
OBJECT_BUCKET=music-bucket
|
||||
OBJECT_ENDPOINT=https://s3.example.com
|
||||
OBJECT_REGION=auto
|
||||
OBJECT_BASE_PREFIX=music
|
||||
OBJECT_ADDRESSING_STYLE=
|
||||
OBJECT_PUBLIC_BASE_URL=
|
||||
OBJECT_CREDENTIAL_ENV_PREFIX=CATALOGSYNC_MAIN_S3
|
||||
|
||||
UPLOAD_WORKERS=4
|
||||
UPLOAD_SOURCES=
|
||||
UPLOAD_PLAYLIST_IDS=
|
||||
UPLOAD_LIMIT=
|
||||
|
||||
CATALOGSYNC_MAIN_S3_ACCESS_KEY_ID=
|
||||
CATALOGSYNC_MAIN_S3_SECRET_ACCESS_KEY=
|
||||
CATALOGSYNC_MAIN_S3_SESSION_TOKEN=
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
RUN_DIR="${APP_HOME}/run"
|
||||
DEPLOY_DIR="${APP_HOME}/deploy"
|
||||
LOCK_DIR="${RUN_DIR}/deploy.lock"
|
||||
PID_FILE="${RUN_DIR}/serve.pid"
|
||||
TARGET_DIR="${APP_HOME}/app/musicdl/catalogsync"
|
||||
DEFAULT_STAGING_DIR="${APP_HOME}/deploy/staging/catalogsync"
|
||||
BACKUP_ROOT="${APP_HOME}/deploy/backups"
|
||||
|
||||
STAGING_DIR="${DEFAULT_STAGING_DIR}"
|
||||
HEALTH_URL=""
|
||||
HEALTH_RETRIES=45
|
||||
HEALTH_INTERVAL_SECONDS=1
|
||||
KEEP_BACKUPS=5
|
||||
SKIP_HEALTH_CHECK=0
|
||||
BACKUP_DIR=""
|
||||
HAS_BACKUP=0
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
$(basename "$0") [options]
|
||||
|
||||
Options:
|
||||
--staging-dir PATH Source directory to sync into app/musicdl/catalogsync
|
||||
--health-url URL Health-check URL (default: http://127.0.0.1:\${WEB_PORT}/dashboard)
|
||||
--health-retries N Max health-check retries (default: 45)
|
||||
--health-interval-sec N Health-check interval seconds (default: 1)
|
||||
--keep-backups N Number of backups to keep (default: 5)
|
||||
--skip-health-check Skip HTTP health check
|
||||
-h, --help Show help
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
echo "[deploy_and_restart.sh] $*"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "[deploy_and_restart.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_positive_integer() {
|
||||
local value="$1"
|
||||
local name="$2"
|
||||
if ! [[ "${value}" =~ ^[0-9]+$ ]] || (( value < 1 )); then
|
||||
fail "${name} must be a positive integer: ${value}"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_deploy_lock() {
|
||||
mkdir -p "${RUN_DIR}"
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner_pid=""
|
||||
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
|
||||
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
|
||||
fail "Another deploy is running (owner_pid=${owner_pid})"
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
fail "Cannot acquire deploy lock: ${LOCK_DIR}"
|
||||
fi
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
}
|
||||
|
||||
cleanup_lock() {
|
||||
rm -rf "${LOCK_DIR}"
|
||||
}
|
||||
|
||||
stop_service() {
|
||||
local pid=""
|
||||
if [[ -f "${PID_FILE}" ]]; then
|
||||
pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
|
||||
log "Stopping running service from PID file (pid=${pid})"
|
||||
kill -TERM "${pid}" 2>/dev/null || true
|
||||
for _ in $(seq 1 20); do
|
||||
if ! kill -0 "${pid}" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "${pid}" 2>/dev/null; then
|
||||
log "Service still alive; force killing pid=${pid}"
|
||||
kill -KILL "${pid}" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
|
||||
local serve_pattern="musicdl.catalogsync.cli serve"
|
||||
local wrapper_pattern="${APP_HOME}/bin/serve_console.sh"
|
||||
for _ in $(seq 1 10); do
|
||||
local serve_count
|
||||
local wrapper_count
|
||||
serve_count="$(count_matching_processes "${serve_pattern}")"
|
||||
wrapper_count="$(count_matching_processes "${wrapper_pattern}")"
|
||||
if [[ "${serve_count}" == "0" && "${wrapper_count}" == "0" ]]; then
|
||||
break
|
||||
fi
|
||||
kill_matching_processes "TERM" "${serve_pattern}"
|
||||
kill_matching_processes "TERM" "${wrapper_pattern}"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
kill_matching_processes "KILL" "${serve_pattern}"
|
||||
kill_matching_processes "KILL" "${wrapper_pattern}"
|
||||
}
|
||||
|
||||
start_service() {
|
||||
local launch_log="${LOG_DIR}/serve_console_launch_$(date +%Y%m%d_%H%M%S).log"
|
||||
nohup bash "${APP_HOME}/bin/serve_console.sh" >"${launch_log}" 2>&1 &
|
||||
local launcher_pid=$!
|
||||
log "Started service launcher pid=${launcher_pid}, launch_log=${launch_log}"
|
||||
}
|
||||
|
||||
sync_catalogsync() {
|
||||
if [[ ! -d "${STAGING_DIR}" ]]; then
|
||||
fail "Staging directory not found: ${STAGING_DIR}"
|
||||
fi
|
||||
if [[ ! -f "${STAGING_DIR}/__init__.py" ]]; then
|
||||
fail "Invalid staging directory (missing __init__.py): ${STAGING_DIR}"
|
||||
fi
|
||||
|
||||
mkdir -p "${BACKUP_ROOT}" "$(dirname "${TARGET_DIR}")"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/catalogsync_$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
if [[ -d "${TARGET_DIR}" ]]; then
|
||||
mv "${TARGET_DIR}" "${BACKUP_DIR}"
|
||||
HAS_BACKUP=1
|
||||
log "Backed up current catalogsync to ${BACKUP_DIR}"
|
||||
fi
|
||||
|
||||
cp -a "${STAGING_DIR}" "${TARGET_DIR}"
|
||||
log "Synced new catalogsync from ${STAGING_DIR} -> ${TARGET_DIR}"
|
||||
}
|
||||
|
||||
wait_health() {
|
||||
if (( SKIP_HEALTH_CHECK == 1 )); then
|
||||
log "Health check skipped by --skip-health-check"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
fail "curl is required for health check"
|
||||
fi
|
||||
|
||||
log "Health checking: ${HEALTH_URL}"
|
||||
for _ in $(seq 1 "${HEALTH_RETRIES}"); do
|
||||
local code
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' "${HEALTH_URL}" || true)"
|
||||
if [[ "${code}" == "200" ]]; then
|
||||
log "Health check passed (HTTP 200)"
|
||||
return 0
|
||||
fi
|
||||
sleep "${HEALTH_INTERVAL_SECONDS}"
|
||||
done
|
||||
|
||||
log "Health check failed: ${HEALTH_URL}"
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_single_instance() {
|
||||
local serve_count
|
||||
serve_count="$(count_matching_processes 'musicdl.catalogsync.cli serve')"
|
||||
if [[ "${serve_count}" != "1" ]]; then
|
||||
log "Unexpected serve process count: ${serve_count}"
|
||||
return 1
|
||||
fi
|
||||
log "Single-instance check passed (serve_count=${serve_count})"
|
||||
return 0
|
||||
}
|
||||
|
||||
list_matching_processes() {
|
||||
local pattern="$1"
|
||||
ps -ef | grep -F "${pattern}" | grep -v grep | awk '{print $2}' || true
|
||||
}
|
||||
|
||||
count_matching_processes() {
|
||||
local pattern="$1"
|
||||
list_matching_processes "${pattern}" | awk 'NF {count++} END {print count+0}'
|
||||
}
|
||||
|
||||
kill_matching_processes() {
|
||||
local signal_name="$1"
|
||||
local pattern="$2"
|
||||
local pid
|
||||
while IFS= read -r pid; do
|
||||
if [[ -n "${pid}" ]]; then
|
||||
kill "-${signal_name}" "${pid}" 2>/dev/null || true
|
||||
fi
|
||||
done < <(list_matching_processes "${pattern}")
|
||||
}
|
||||
|
||||
rollback() {
|
||||
log "Starting rollback..."
|
||||
stop_service
|
||||
if (( HAS_BACKUP == 0 )) || [[ ! -d "${BACKUP_DIR}" ]]; then
|
||||
log "No backup available; rollback skipped"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -rf "${TARGET_DIR}"
|
||||
mv "${BACKUP_DIR}" "${TARGET_DIR}"
|
||||
HAS_BACKUP=0
|
||||
log "Restored backup to ${TARGET_DIR}"
|
||||
|
||||
start_service
|
||||
if ! wait_health; then
|
||||
log "Rollback service failed health check"
|
||||
return 1
|
||||
fi
|
||||
verify_single_instance
|
||||
}
|
||||
|
||||
prune_backups() {
|
||||
if (( KEEP_BACKUPS < 1 )); then
|
||||
return 0
|
||||
fi
|
||||
if [[ ! -d "${BACKUP_ROOT}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
mapfile -t backups < <(ls -1dt "${BACKUP_ROOT}"/catalogsync_* 2>/dev/null || true)
|
||||
if (( ${#backups[@]} <= KEEP_BACKUPS )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for old_backup in "${backups[@]:KEEP_BACKUPS}"; do
|
||||
rm -rf "${old_backup}"
|
||||
log "Pruned old backup: ${old_backup}"
|
||||
done
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--staging-dir)
|
||||
STAGING_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-url)
|
||||
HEALTH_URL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-retries)
|
||||
HEALTH_RETRIES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--health-interval-sec)
|
||||
HEALTH_INTERVAL_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-backups)
|
||||
KEEP_BACKUPS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--skip-health-check)
|
||||
SKIP_HEALTH_CHECK=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
fail "Unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_positive_integer "${HEALTH_RETRIES}" "HEALTH_RETRIES"
|
||||
validate_positive_integer "${HEALTH_INTERVAL_SECONDS}" "HEALTH_INTERVAL_SECONDS"
|
||||
validate_positive_integer "${KEEP_BACKUPS}" "KEEP_BACKUPS"
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in WEB_PORT LOG_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -z "${HEALTH_URL}" ]]; then
|
||||
HEALTH_URL="http://127.0.0.1:${WEB_PORT}/dashboard"
|
||||
fi
|
||||
|
||||
mkdir -p "${DEPLOY_DIR}" "${RUN_DIR}" "${LOG_DIR}" "${BACKUP_ROOT}" "${APP_HOME}/app/musicdl"
|
||||
acquire_deploy_lock
|
||||
trap cleanup_lock EXIT INT TERM
|
||||
|
||||
LOG_FILE="${LOG_DIR}/deploy_and_restart_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
log "Starting deploy. staging=${STAGING_DIR}"
|
||||
log "Deploy log: ${LOG_FILE}"
|
||||
|
||||
if ! sync_catalogsync; then
|
||||
fail "Sync step failed"
|
||||
fi
|
||||
|
||||
stop_service
|
||||
start_service
|
||||
|
||||
if ! wait_health; then
|
||||
log "New version failed health check; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed; rollback failed."
|
||||
fi
|
||||
|
||||
if ! verify_single_instance; then
|
||||
log "Single-instance check failed; attempting rollback."
|
||||
if rollback; then
|
||||
fail "Deploy failed by single-instance check; rollback succeeded."
|
||||
fi
|
||||
fail "Deploy failed by single-instance check; rollback failed."
|
||||
fi
|
||||
|
||||
prune_backups
|
||||
log "Deploy succeeded."
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[download_all.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/download_all_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[download_all.sh] logging to ${LOG_FILE}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
|
||||
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
|
||||
--db "${DB_PATH}" \
|
||||
--library-root "${LIBRARY_DIR}" \
|
||||
--workers "${DOWNLOAD_WORKERS:-10}" \
|
||||
"${EXTRA_ARGS[@]}" \
|
||||
"$@"
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
fail() {
|
||||
echo "[download_from_file.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
fail "usage: $0 <playlist-file> [extra args...]"
|
||||
fi
|
||||
|
||||
PLAYLIST_FILE="$1"
|
||||
shift
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
if [[ ! -f "${PLAYLIST_FILE}" ]]; then
|
||||
fail "playlist file not found: ${PLAYLIST_FILE}"
|
||||
fi
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in LIBRARY_DIR DB_PATH INPUT_DIR LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${LIBRARY_DIR}" "${APP_HOME}/data" "${INPUT_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/download_from_file_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[download_from_file.sh] logging to ${LOG_FILE}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "${DOWNLOAD_SOURCES:-}" ]]; then
|
||||
EXTRA_ARGS+=(--download-sources "${DOWNLOAD_SOURCES}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli run \
|
||||
--db "${DB_PATH}" \
|
||||
--library-root "${LIBRARY_DIR}" \
|
||||
--playlist-file "${PLAYLIST_FILE}" \
|
||||
"${EXTRA_ARGS[@]}" \
|
||||
"$@"
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[install_runtime.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
require_requirement() {
|
||||
local requirement_name="$1"
|
||||
if ! grep -Eq "^${requirement_name}([<>=!~].*)?$" "${NAS_REQUIREMENTS_FILE}"; then
|
||||
fail "Missing required runtime dependency in requirements.txt: ${requirement_name}"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in APP_HOME DB_PATH LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
APP_DIR="${APP_HOME}/app"
|
||||
REQUIREMENTS_FILE="${APP_DIR}/requirements.txt"
|
||||
SETUP_FILE="${APP_DIR}/setup.py"
|
||||
if [[ ! -f "${REQUIREMENTS_FILE}" ]]; then
|
||||
fail "requirements.txt not found: ${REQUIREMENTS_FILE}"
|
||||
fi
|
||||
if [[ ! -f "${SETUP_FILE}" ]]; then
|
||||
fail "setup.py not found: ${SETUP_FILE}"
|
||||
fi
|
||||
|
||||
mkdir -p "${APP_HOME}" "${APP_DIR}" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/install_runtime_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[install_runtime.sh] logging to ${LOG_FILE}"
|
||||
|
||||
if [[ ! -d "${VENV_DIR}" ]]; then
|
||||
"${PYTHON_BIN}" -m venv "${VENV_DIR}"
|
||||
fi
|
||||
|
||||
RUNTIME_PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
if [[ ! -x "${RUNTIME_PYTHON_BIN}" ]]; then
|
||||
fail "Virtualenv python not found after setup: ${RUNTIME_PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install --upgrade pip setuptools wheel
|
||||
|
||||
NAS_REQUIREMENTS_FILE="${APP_DIR}/requirements.nas.txt"
|
||||
grep -v '^nodejs-wheel$' "${REQUIREMENTS_FILE}" > "${NAS_REQUIREMENTS_FILE}"
|
||||
for runtime_requirement in fastapi uvicorn python-multipart; do
|
||||
require_requirement "${runtime_requirement}"
|
||||
done
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install -r "${NAS_REQUIREMENTS_FILE}"
|
||||
"${RUNTIME_PYTHON_BIN}" -m pip install --no-deps -e "${APP_DIR}"
|
||||
|
||||
echo "[install_runtime.sh] runtime ready: ${RUNTIME_PYTHON_BIN}"
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
trim_env_whitespace() {
|
||||
local value="$1"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
load_env_file() {
|
||||
local env_file="$1"
|
||||
local raw_line=""
|
||||
local normalized=""
|
||||
local key=""
|
||||
local value=""
|
||||
local trimmed_value=""
|
||||
local quote_char=""
|
||||
|
||||
[[ -f "${env_file}" ]] || return 1
|
||||
|
||||
while IFS= read -r raw_line || [[ -n "${raw_line}" ]]; do
|
||||
raw_line="${raw_line%$'\r'}"
|
||||
normalized="$(trim_env_whitespace "${raw_line}")"
|
||||
if [[ -z "${normalized}" || "${normalized:0:1}" == "#" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ "${normalized}" == export\ * ]]; then
|
||||
normalized="${normalized#export }"
|
||||
fi
|
||||
if [[ "${normalized}" != *=* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
key="$(trim_env_whitespace "${normalized%%=*}")"
|
||||
if [[ -z "${key}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
value="${normalized#*=}"
|
||||
trimmed_value="$(trim_env_whitespace "${value}")"
|
||||
if [[ ${#trimmed_value} -ge 2 ]]; then
|
||||
quote_char="${trimmed_value:0:1}"
|
||||
if [[ ( "${quote_char}" == "'" || "${quote_char}" == '"' ) && "${trimmed_value: -1}" == "${quote_char}" ]]; then
|
||||
value="${trimmed_value:1:${#trimmed_value}-2}"
|
||||
else
|
||||
value="${trimmed_value}"
|
||||
fi
|
||||
else
|
||||
value="${trimmed_value}"
|
||||
fi
|
||||
|
||||
printf -v "${key}" '%s' "${value}"
|
||||
export "${key}"
|
||||
done < "${env_file}"
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
RUN_DIR="${APP_HOME}/run"
|
||||
LOCK_DIR="${RUN_DIR}/serve.lock"
|
||||
PID_FILE="${RUN_DIR}/serve.pid"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[serve_console.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_serve_lock() {
|
||||
mkdir -p "${RUN_DIR}"
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local owner_pid=""
|
||||
if [[ -f "${LOCK_DIR}/owner_pid" ]]; then
|
||||
owner_pid="$(cat "${LOCK_DIR}/owner_pid" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "${owner_pid}" ]] && kill -0 "${owner_pid}" 2>/dev/null; then
|
||||
fail "Another serve_console instance is running (owner_pid=${owner_pid})"
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
if ! mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
fail "Cannot acquire serve lock: ${LOCK_DIR}"
|
||||
fi
|
||||
echo "$$" > "${LOCK_DIR}/owner_pid"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
kill -TERM "${SERVER_PID}" 2>/dev/null || true
|
||||
wait "${SERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
rm -rf "${LOCK_DIR}"
|
||||
return "${exit_code}"
|
||||
}
|
||||
|
||||
validate_port() {
|
||||
local port_value="$1"
|
||||
if ! [[ "${port_value}" =~ ^[0-9]+$ ]] || (( port_value < 1 || port_value > 65535 )); then
|
||||
fail "WEB_PORT must be an integer in range 1..65535: ${port_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in DB_PATH ENV_FILE WEB_HOST WEB_PORT LOG_DIR PYTHON_BIN VENV_DIR; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
validate_port "${WEB_PORT}"
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")" "$(dirname "${ENV_FILE}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
acquire_serve_lock
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
if [[ -f "${PID_FILE}" ]]; then
|
||||
existing_pid="$(cat "${PID_FILE}" 2>/dev/null || true)"
|
||||
if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
|
||||
fail "Server process already running (pid=${existing_pid})"
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
fi
|
||||
|
||||
LOG_FILE="${LOG_DIR}/serve_console_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[serve_console.sh] logging to ${LOG_FILE}"
|
||||
|
||||
# Run from the deployed app directory so `python -m musicdl...` does not
|
||||
# accidentally import an older checkout from the current working directory.
|
||||
cd "${APP_HOME}/app"
|
||||
|
||||
"${PYTHON_BIN}" -m musicdl.catalogsync.cli serve \
|
||||
--db "${DB_PATH}" \
|
||||
--env-file "${ENV_FILE}" \
|
||||
--host "${WEB_HOST}" \
|
||||
--port "${WEB_PORT}" \
|
||||
"$@" &
|
||||
SERVER_PID=$!
|
||||
echo "${SERVER_PID}" > "${PID_FILE}"
|
||||
echo "[serve_console.sh] server pid=${SERVER_PID}"
|
||||
wait "${SERVER_PID}"
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
APP_HOME="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
CONFIG_FILE="${APP_HOME}/config/catalogsync.env"
|
||||
# shellcheck source=./load_env.sh
|
||||
source "${SCRIPT_DIR}/load_env.sh"
|
||||
|
||||
fail() {
|
||||
echo "[upload_all.sh] ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_var() {
|
||||
local var_name="$1"
|
||||
if [[ -z "${!var_name:-}" ]]; then
|
||||
fail "Missing required config variable: ${var_name} (from ${CONFIG_FILE})"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -f "${CONFIG_FILE}" ]]; then
|
||||
load_env_file "${CONFIG_FILE}"
|
||||
else
|
||||
fail "Config file not found: ${CONFIG_FILE}. Copy catalogsync.env.example to catalogsync.env first."
|
||||
fi
|
||||
|
||||
for required_var in DB_PATH LOG_DIR PYTHON_BIN VENV_DIR OBJECT_BACKEND_NAME OBJECT_BUCKET OBJECT_ENDPOINT OBJECT_CREDENTIAL_ENV_PREFIX; do
|
||||
require_var "${required_var}"
|
||||
done
|
||||
|
||||
if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]; then
|
||||
PYTHON_BIN="${VENV_DIR}/bin/python"
|
||||
fi
|
||||
|
||||
if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
|
||||
fail "PYTHON_BIN is not executable or not found in PATH: ${PYTHON_BIN}"
|
||||
fi
|
||||
|
||||
ACCESS_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_ACCESS_KEY_ID"
|
||||
SECRET_KEY_VAR="${OBJECT_CREDENTIAL_ENV_PREFIX}_SECRET_ACCESS_KEY"
|
||||
require_var "${ACCESS_KEY_VAR}"
|
||||
require_var "${SECRET_KEY_VAR}"
|
||||
|
||||
mkdir -p "${APP_HOME}/data" "${LOG_DIR}" "$(dirname "${DB_PATH}")"
|
||||
export PYTHONPATH="${APP_HOME}/app${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
LOG_FILE="${LOG_DIR}/upload_all_$(date +%Y%m%d_%H%M%S).log"
|
||||
exec > >(tee -a "${LOG_FILE}") 2>&1
|
||||
echo "[upload_all.sh] logging to ${LOG_FILE}"
|
||||
|
||||
REGISTER_ARGS=(
|
||||
-m musicdl.catalogsync.cli register-object-backend
|
||||
--db "${DB_PATH}"
|
||||
--backend "${OBJECT_BACKEND_NAME}"
|
||||
--bucket "${OBJECT_BUCKET}"
|
||||
--endpoint "${OBJECT_ENDPOINT}"
|
||||
--credential-env-prefix "${OBJECT_CREDENTIAL_ENV_PREFIX}"
|
||||
)
|
||||
|
||||
if [[ -n "${OBJECT_REGION:-}" ]]; then
|
||||
REGISTER_ARGS+=(--region "${OBJECT_REGION}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_BASE_PREFIX:-}" ]]; then
|
||||
REGISTER_ARGS+=(--base-prefix "${OBJECT_BASE_PREFIX}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_ADDRESSING_STYLE:-}" ]]; then
|
||||
REGISTER_ARGS+=(--addressing-style "${OBJECT_ADDRESSING_STYLE}")
|
||||
fi
|
||||
if [[ -n "${OBJECT_PUBLIC_BASE_URL:-}" ]]; then
|
||||
REGISTER_ARGS+=(--public-base-url "${OBJECT_PUBLIC_BASE_URL}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" "${REGISTER_ARGS[@]}"
|
||||
|
||||
UPLOAD_ARGS=(
|
||||
-m musicdl.catalogsync.cli upload
|
||||
--db "${DB_PATH}"
|
||||
--backend "${OBJECT_BACKEND_NAME}"
|
||||
--workers "${UPLOAD_WORKERS:-4}"
|
||||
)
|
||||
|
||||
if [[ -n "${UPLOAD_SOURCES:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--sources "${UPLOAD_SOURCES}")
|
||||
fi
|
||||
if [[ -n "${UPLOAD_PLAYLIST_IDS:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--playlist-ids "${UPLOAD_PLAYLIST_IDS}")
|
||||
fi
|
||||
if [[ -n "${UPLOAD_LIMIT:-}" ]]; then
|
||||
UPLOAD_ARGS+=(--limit "${UPLOAD_LIMIT}")
|
||||
fi
|
||||
|
||||
"${PYTHON_BIN}" "${UPLOAD_ARGS[@]}" "$@"
|
||||
@@ -0,0 +1,32 @@
|
||||
'''
|
||||
Function:
|
||||
Implementation of removepycache
|
||||
Author:
|
||||
Zhenchao Jin
|
||||
WeChat Official Account (微信公众号):
|
||||
Charles的皮卡丘
|
||||
'''
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
'''removepycache'''
|
||||
def removepycache(root: str | os.PathLike = ".") -> int:
|
||||
root_path, removed = Path(root).resolve(), 0
|
||||
for p in root_path.rglob("__pycache__"):
|
||||
if p.is_dir():
|
||||
try:
|
||||
shutil.rmtree(p)
|
||||
removed += 1
|
||||
print(f"Removed: {p}")
|
||||
except Exception as e:
|
||||
print(f"Failed: {p} ({e})")
|
||||
print(f"\nDone. Removed {removed} __pycache__ directories under {root_path}")
|
||||
return removed
|
||||
|
||||
|
||||
'''run'''
|
||||
if __name__ == "__main__":
|
||||
removepycache(".")
|
||||
Reference in New Issue
Block a user