Initial import: Music_Server, MusicFree, catalog-sync
This commit is contained in:
@@ -0,0 +1,353 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
class RuntimeLayoutTests(unittest.TestCase):
|
||||
def test_runtime_config_builds_defaults_from_root_dir(self):
|
||||
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
||||
|
||||
config = CatalogSyncRuntimeConfig.from_mapping(
|
||||
{
|
||||
"ROOT_DIR": "/volume4/Music_Cloud",
|
||||
"PYTHON_BIN": "python3",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(Path("/volume4/Music_Cloud/catalogsync"), config.app_home)
|
||||
self.assertEqual(Path("/volume4/Music_Cloud/library"), config.library_dir)
|
||||
self.assertEqual(
|
||||
Path("/volume4/Music_Cloud/catalogsync/data/catalogsync.db"), config.db_path
|
||||
)
|
||||
self.assertEqual(
|
||||
Path("/volume4/Music_Cloud/catalogsync/config/catalogsync.env"),
|
||||
config.env_file,
|
||||
)
|
||||
self.assertEqual("127.0.0.1", config.web_host)
|
||||
self.assertEqual(18080, config.web_port)
|
||||
self.assertEqual("platform_first_artist", config.download_layout)
|
||||
|
||||
def test_catalogsync_modules_avoid_python310_only_dataclass_slots_for_nas_python38(self):
|
||||
for relative_path in (
|
||||
"musicdl/catalogsync/runtime.py",
|
||||
"musicdl/catalogsync/downloader.py",
|
||||
"musicdl/catalogsync/ops/executors.py",
|
||||
):
|
||||
with self.subTest(relative_path=relative_path):
|
||||
source = Path(relative_path).read_text(encoding="utf-8")
|
||||
self.assertNotIn("@dataclass(slots=True)", source)
|
||||
|
||||
def test_runtime_config_reads_web_fields_from_mapping(self):
|
||||
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
||||
|
||||
config = CatalogSyncRuntimeConfig.from_mapping(
|
||||
{
|
||||
"ROOT_DIR": "/volume4/Music_Cloud",
|
||||
"ENV_FILE": "/etc/catalogsync.env",
|
||||
"WEB_HOST": "0.0.0.0",
|
||||
"WEB_PORT": "19090",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(Path("/etc/catalogsync.env"), config.env_file)
|
||||
self.assertEqual("0.0.0.0", config.web_host)
|
||||
self.assertEqual(19090, config.web_port)
|
||||
|
||||
def test_runtime_config_falls_back_when_web_port_invalid_or_out_of_range(self):
|
||||
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
||||
|
||||
for raw_value in ("", "abc", "0", "-1", "70000"):
|
||||
config = CatalogSyncRuntimeConfig.from_mapping(
|
||||
{
|
||||
"ROOT_DIR": "/volume4/Music_Cloud",
|
||||
"WEB_PORT": raw_value,
|
||||
}
|
||||
)
|
||||
self.assertEqual(18080, config.web_port)
|
||||
|
||||
def test_runtime_config_ensure_directories_creates_expected_tree(self):
|
||||
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
root_dir = Path(tmpdir) / "Music_Cloud"
|
||||
config = CatalogSyncRuntimeConfig.from_mapping({"ROOT_DIR": str(root_dir)})
|
||||
|
||||
config.ensure_directories()
|
||||
|
||||
self.assertTrue((root_dir / "library").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "app").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "bin").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "config").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "data").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "inputs").is_dir())
|
||||
self.assertTrue((root_dir / "catalogsync" / "logs").is_dir())
|
||||
|
||||
def test_runtime_config_ensure_directories_respects_override_paths(self):
|
||||
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
||||
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
||||
root_dir = Path(tmpdir) / "Music_Cloud"
|
||||
db_path = root_dir / "state" / "db" / "catalogsync.db"
|
||||
input_dir = root_dir / "state" / "manual_inputs"
|
||||
log_dir = root_dir / "state" / "runtime_logs"
|
||||
config = CatalogSyncRuntimeConfig.from_mapping(
|
||||
{
|
||||
"ROOT_DIR": str(root_dir),
|
||||
"DB_PATH": str(db_path),
|
||||
"INPUT_DIR": str(input_dir),
|
||||
"LOG_DIR": str(log_dir),
|
||||
}
|
||||
)
|
||||
|
||||
config.ensure_directories()
|
||||
|
||||
self.assertTrue(db_path.parent.is_dir())
|
||||
self.assertTrue(input_dir.is_dir())
|
||||
self.assertTrue(log_dir.is_dir())
|
||||
|
||||
def test_build_download_relative_dir_uses_platform_and_first_artist(self):
|
||||
from musicdl.catalogsync.runtime import build_download_relative_dir
|
||||
|
||||
relative_dir = build_download_relative_dir(
|
||||
platform="qq",
|
||||
singers="Singer A / Singer B",
|
||||
)
|
||||
|
||||
self.assertEqual(Path("qq") / "Singer A", relative_dir)
|
||||
|
||||
def test_build_download_relative_dir_falls_back_to_unknown_artist(self):
|
||||
from musicdl.catalogsync.runtime import build_download_relative_dir
|
||||
|
||||
relative_dir = build_download_relative_dir(
|
||||
platform="netease",
|
||||
singers="",
|
||||
)
|
||||
|
||||
self.assertEqual(Path("netease") / "Unknown Artist", relative_dir)
|
||||
|
||||
def test_catalogsync_env_example_contains_required_keys(self):
|
||||
template = Path(
|
||||
"scripts/catalogsync/templates/catalogsync.env.example"
|
||||
).read_text(encoding="utf-8")
|
||||
self.assertIn("ROOT_DIR=", template)
|
||||
self.assertIn("APP_HOME=", template)
|
||||
self.assertIn("LIBRARY_DIR=", template)
|
||||
self.assertIn("DB_PATH=", template)
|
||||
self.assertIn("INPUT_DIR=", template)
|
||||
self.assertIn("LOG_DIR=", template)
|
||||
self.assertIn("ENV_FILE=", template)
|
||||
self.assertIn("WEB_HOST=", template)
|
||||
self.assertIn("WEB_PORT=", template)
|
||||
self.assertIn("PYTHON_BIN=", template)
|
||||
self.assertIn("VENV_DIR=", template)
|
||||
self.assertIn("DOWNLOAD_LAYOUT=platform_first_artist", template)
|
||||
self.assertIn("DOWNLOAD_SOURCES=", template)
|
||||
self.assertIn("OBJECT_BACKEND_NAME=", template)
|
||||
self.assertIn("OBJECT_BUCKET=", template)
|
||||
self.assertIn("OBJECT_ENDPOINT=", template)
|
||||
self.assertIn("OBJECT_CREDENTIAL_ENV_PREFIX=", template)
|
||||
self.assertIn("SYNC_WORKERS=", template)
|
||||
self.assertIn("UPLOAD_WORKERS=", template)
|
||||
|
||||
def test_requirements_include_eval_type_backport_for_python38_pydantic_web_models(self):
|
||||
requirements = Path("requirements.txt").read_text(encoding="utf-8")
|
||||
self.assertIn("eval_type_backport", requirements)
|
||||
|
||||
def test_requirements_include_jinja2_for_ops_web_templates(self):
|
||||
requirements = Path("requirements.txt").read_text(encoding="utf-8")
|
||||
self.assertIn("jinja2", requirements)
|
||||
|
||||
def test_bootstrap_script_uses_remote_host_parameter_name(self):
|
||||
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("[string]$RemoteHost", script)
|
||||
self.assertNotIn("[string]$Host,", script)
|
||||
|
||||
def test_bootstrap_script_uses_remote_path_quoting_helpers(self):
|
||||
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("function Quote-RemotePathForPosixShell", script)
|
||||
self.assertIn("function New-ScpRemoteTarget", script)
|
||||
self.assertIn("Quote-RemotePathForPosixShell -Path", script)
|
||||
self.assertIn("New-ScpRemoteTarget -Remote $Remote", script)
|
||||
self.assertIn("upload_all.sh", script)
|
||||
self.assertIn("install_runtime.sh", script)
|
||||
self.assertIn("serve_console.sh", script)
|
||||
self.assertIn("README.md", script)
|
||||
self.assertIn("LICENSE", script)
|
||||
self.assertNotIn("${Remote}:$AppHome/app/", script)
|
||||
self.assertNotIn("${Remote}:$AppHome/bin/", script)
|
||||
|
||||
def test_bootstrap_script_refreshes_env_example_when_template_changes(self):
|
||||
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertNotIn("if (($EnvCheck | Select-Object -Last 1).Trim() -eq \"missing\")", script)
|
||||
self.assertIn("catalogsync.env.example", script)
|
||||
|
||||
def test_bootstrap_script_is_parseable_by_powershell(self):
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
self.skipTest("PowerShell is unavailable in current environment.")
|
||||
|
||||
script_path = Path("scripts/catalogsync/bootstrap_to_linux.ps1").resolve()
|
||||
escaped_path = str(script_path).replace("'", "''")
|
||||
parse_command = (
|
||||
"$tokens=$null; $errors=$null; "
|
||||
f"[void][System.Management.Automation.Language.Parser]::ParseFile('{escaped_path}', [ref]$tokens, [ref]$errors); "
|
||||
"if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 }"
|
||||
)
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-Command", parse_command],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
0,
|
||||
result.returncode,
|
||||
msg=f"PowerShell parser errors:\n{result.stderr or result.stdout}",
|
||||
)
|
||||
|
||||
def test_runtime_script_templates_pass_bash_syntax_check(self):
|
||||
bash = shutil.which("bash")
|
||||
if bash is None:
|
||||
self.skipTest("bash is unavailable in current environment.")
|
||||
|
||||
for template_name in (
|
||||
"load_env.sh",
|
||||
"download_from_file.sh",
|
||||
"download_all.sh",
|
||||
"upload_all.sh",
|
||||
"install_runtime.sh",
|
||||
"serve_console.sh",
|
||||
):
|
||||
script_path = Path(
|
||||
f"scripts/catalogsync/templates/{template_name}"
|
||||
).resolve()
|
||||
result = subprocess.run(
|
||||
[bash, "-n", script_path.as_posix()],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
0,
|
||||
result.returncode,
|
||||
msg=f"bash -n failed for {template_name}:\n{result.stderr or result.stdout}",
|
||||
)
|
||||
|
||||
def test_runtime_script_template_uses_configured_library_dir(self):
|
||||
for template_name in ("download_from_file.sh", "download_all.sh"):
|
||||
script = Path(
|
||||
f"scripts/catalogsync/templates/{template_name}"
|
||||
).read_text(encoding="utf-8")
|
||||
self.assertIn("LIBRARY_DIR", script)
|
||||
self.assertIn("DB_PATH", script)
|
||||
self.assertIn("INPUT_DIR", script)
|
||||
self.assertIn("LOG_DIR", script)
|
||||
self.assertIn("PYTHON_BIN", script)
|
||||
self.assertIn("PYTHONPATH", script)
|
||||
self.assertIn("VENV_DIR", script)
|
||||
self.assertIn('if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]', script)
|
||||
self.assertIn("musicdl.catalogsync.cli run", script)
|
||||
|
||||
def test_upload_runtime_script_template_uses_upload_command_and_object_backend_vars(self):
|
||||
script = Path("scripts/catalogsync/templates/upload_all.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("DB_PATH", script)
|
||||
self.assertIn("LOG_DIR", script)
|
||||
self.assertIn("PYTHON_BIN", script)
|
||||
self.assertIn("PYTHONPATH", script)
|
||||
self.assertIn("VENV_DIR", script)
|
||||
self.assertIn('if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]', script)
|
||||
self.assertIn("OBJECT_BACKEND_NAME", script)
|
||||
self.assertIn("OBJECT_BUCKET", script)
|
||||
self.assertIn("OBJECT_ENDPOINT", script)
|
||||
self.assertIn("OBJECT_CREDENTIAL_ENV_PREFIX", script)
|
||||
self.assertIn("musicdl.catalogsync.cli register-object-backend", script)
|
||||
self.assertIn("musicdl.catalogsync.cli upload", script)
|
||||
|
||||
def test_install_runtime_script_template_sets_up_venv_and_nas_requirements(self):
|
||||
script = Path("scripts/catalogsync/templates/install_runtime.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("VENV_DIR", script)
|
||||
self.assertIn("PYTHON_BIN", script)
|
||||
self.assertIn("requirements.nas.txt", script)
|
||||
self.assertIn('APP_DIR="${APP_HOME}/app"', script)
|
||||
self.assertIn('REQUIREMENTS_FILE="${APP_DIR}/requirements.txt"', script)
|
||||
self.assertIn('SETUP_FILE="${APP_DIR}/setup.py"', script)
|
||||
self.assertIn("grep -v '^nodejs-wheel$'", script)
|
||||
self.assertIn('"${PYTHON_BIN}" -m venv "${VENV_DIR}"', script)
|
||||
self.assertIn('"${RUNTIME_PYTHON_BIN}" -m pip install -r "${NAS_REQUIREMENTS_FILE}"', script)
|
||||
self.assertIn('"${RUNTIME_PYTHON_BIN}" -m pip install --no-deps -e "${APP_DIR}"', script)
|
||||
load_config_index = script.index('load_env_file "${CONFIG_FILE}"')
|
||||
app_dir_index = script.index('APP_DIR="${APP_HOME}/app"')
|
||||
self.assertGreater(app_dir_index, load_config_index)
|
||||
|
||||
def test_runtime_script_templates_include_preflight_and_log_file(self):
|
||||
for template_name in (
|
||||
"download_from_file.sh",
|
||||
"download_all.sh",
|
||||
"upload_all.sh",
|
||||
"install_runtime.sh",
|
||||
"serve_console.sh",
|
||||
):
|
||||
script = Path(
|
||||
f"scripts/catalogsync/templates/{template_name}"
|
||||
).read_text(encoding="utf-8")
|
||||
self.assertIn('[[ -f "${CONFIG_FILE}" ]]', script)
|
||||
self.assertIn('command -v "${PYTHON_BIN}"', script)
|
||||
self.assertIn("for required_var in", script)
|
||||
self.assertIn("DB_PATH", script)
|
||||
self.assertIn("LOG_DIR", script)
|
||||
self.assertIn("PYTHON_BIN", script)
|
||||
self.assertIn('LOG_FILE="${LOG_DIR}/', script)
|
||||
self.assertIn('exec > >(tee -a "${LOG_FILE}") 2>&1', script)
|
||||
|
||||
def test_runtime_script_templates_use_safe_env_loader(self):
|
||||
helper = Path("scripts/catalogsync/templates/load_env.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("load_env_file()", helper)
|
||||
|
||||
for template_name in (
|
||||
"download_from_file.sh",
|
||||
"download_all.sh",
|
||||
"upload_all.sh",
|
||||
"install_runtime.sh",
|
||||
"serve_console.sh",
|
||||
"deploy_and_restart.sh",
|
||||
):
|
||||
script = Path(
|
||||
f"scripts/catalogsync/templates/{template_name}"
|
||||
).read_text(encoding="utf-8")
|
||||
self.assertIn('source "${SCRIPT_DIR}/load_env.sh"', script)
|
||||
self.assertIn('load_env_file "${CONFIG_FILE}"', script)
|
||||
self.assertNotIn('source "${CONFIG_FILE}"', script)
|
||||
|
||||
def test_serve_console_runtime_script_template_uses_serve_command_and_web_vars(self):
|
||||
script = Path("scripts/catalogsync/templates/serve_console.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("DB_PATH", script)
|
||||
self.assertIn("ENV_FILE", script)
|
||||
self.assertIn("WEB_HOST", script)
|
||||
self.assertIn("WEB_PORT", script)
|
||||
self.assertIn("LOG_DIR", script)
|
||||
self.assertIn("PYTHON_BIN", script)
|
||||
self.assertIn("PYTHONPATH", script)
|
||||
self.assertIn("VENV_DIR", script)
|
||||
self.assertIn("validate_port", script)
|
||||
self.assertIn("WEB_PORT must be an integer in range 1..65535", script)
|
||||
self.assertIn(
|
||||
'if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]',
|
||||
script,
|
||||
)
|
||||
self.assertIn("musicdl.catalogsync.cli serve", script)
|
||||
Reference in New Issue
Block a user