354 lines
15 KiB
Python
354 lines
15 KiB
Python
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
|
|
|
|
class RuntimeLayoutTests(unittest.TestCase):
|
|
def test_runtime_config_builds_defaults_from_root_dir(self):
|
|
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
|
|
|
config = CatalogSyncRuntimeConfig.from_mapping(
|
|
{
|
|
"ROOT_DIR": "/volume4/Music_Cloud",
|
|
"PYTHON_BIN": "python3",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(Path("/volume4/Music_Cloud/catalogsync"), config.app_home)
|
|
self.assertEqual(Path("/volume4/Music_Cloud/library"), config.library_dir)
|
|
self.assertEqual(
|
|
Path("/volume4/Music_Cloud/catalogsync/data/catalogsync.db"), config.db_path
|
|
)
|
|
self.assertEqual(
|
|
Path("/volume4/Music_Cloud/catalogsync/config/catalogsync.env"),
|
|
config.env_file,
|
|
)
|
|
self.assertEqual("127.0.0.1", config.web_host)
|
|
self.assertEqual(18080, config.web_port)
|
|
self.assertEqual("platform_first_artist", config.download_layout)
|
|
|
|
def test_catalogsync_modules_avoid_python310_only_dataclass_slots_for_nas_python38(self):
|
|
for relative_path in (
|
|
"musicdl/catalogsync/runtime.py",
|
|
"musicdl/catalogsync/downloader.py",
|
|
"musicdl/catalogsync/ops/executors.py",
|
|
):
|
|
with self.subTest(relative_path=relative_path):
|
|
source = Path(relative_path).read_text(encoding="utf-8")
|
|
self.assertNotIn("@dataclass(slots=True)", source)
|
|
|
|
def test_runtime_config_reads_web_fields_from_mapping(self):
|
|
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
|
|
|
config = CatalogSyncRuntimeConfig.from_mapping(
|
|
{
|
|
"ROOT_DIR": "/volume4/Music_Cloud",
|
|
"ENV_FILE": "/etc/catalogsync.env",
|
|
"WEB_HOST": "0.0.0.0",
|
|
"WEB_PORT": "19090",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(Path("/etc/catalogsync.env"), config.env_file)
|
|
self.assertEqual("0.0.0.0", config.web_host)
|
|
self.assertEqual(19090, config.web_port)
|
|
|
|
def test_runtime_config_falls_back_when_web_port_invalid_or_out_of_range(self):
|
|
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
|
|
|
for raw_value in ("", "abc", "0", "-1", "70000"):
|
|
config = CatalogSyncRuntimeConfig.from_mapping(
|
|
{
|
|
"ROOT_DIR": "/volume4/Music_Cloud",
|
|
"WEB_PORT": raw_value,
|
|
}
|
|
)
|
|
self.assertEqual(18080, config.web_port)
|
|
|
|
def test_runtime_config_ensure_directories_creates_expected_tree(self):
|
|
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
root_dir = Path(tmpdir) / "Music_Cloud"
|
|
config = CatalogSyncRuntimeConfig.from_mapping({"ROOT_DIR": str(root_dir)})
|
|
|
|
config.ensure_directories()
|
|
|
|
self.assertTrue((root_dir / "library").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "app").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "bin").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "config").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "data").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "inputs").is_dir())
|
|
self.assertTrue((root_dir / "catalogsync" / "logs").is_dir())
|
|
|
|
def test_runtime_config_ensure_directories_respects_override_paths(self):
|
|
from musicdl.catalogsync.runtime import CatalogSyncRuntimeConfig
|
|
|
|
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
|
|
root_dir = Path(tmpdir) / "Music_Cloud"
|
|
db_path = root_dir / "state" / "db" / "catalogsync.db"
|
|
input_dir = root_dir / "state" / "manual_inputs"
|
|
log_dir = root_dir / "state" / "runtime_logs"
|
|
config = CatalogSyncRuntimeConfig.from_mapping(
|
|
{
|
|
"ROOT_DIR": str(root_dir),
|
|
"DB_PATH": str(db_path),
|
|
"INPUT_DIR": str(input_dir),
|
|
"LOG_DIR": str(log_dir),
|
|
}
|
|
)
|
|
|
|
config.ensure_directories()
|
|
|
|
self.assertTrue(db_path.parent.is_dir())
|
|
self.assertTrue(input_dir.is_dir())
|
|
self.assertTrue(log_dir.is_dir())
|
|
|
|
def test_build_download_relative_dir_uses_platform_and_first_artist(self):
|
|
from musicdl.catalogsync.runtime import build_download_relative_dir
|
|
|
|
relative_dir = build_download_relative_dir(
|
|
platform="qq",
|
|
singers="Singer A / Singer B",
|
|
)
|
|
|
|
self.assertEqual(Path("qq") / "Singer A", relative_dir)
|
|
|
|
def test_build_download_relative_dir_falls_back_to_unknown_artist(self):
|
|
from musicdl.catalogsync.runtime import build_download_relative_dir
|
|
|
|
relative_dir = build_download_relative_dir(
|
|
platform="netease",
|
|
singers="",
|
|
)
|
|
|
|
self.assertEqual(Path("netease") / "Unknown Artist", relative_dir)
|
|
|
|
def test_catalogsync_env_example_contains_required_keys(self):
|
|
template = Path(
|
|
"scripts/catalogsync/templates/catalogsync.env.example"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn("ROOT_DIR=", template)
|
|
self.assertIn("APP_HOME=", template)
|
|
self.assertIn("LIBRARY_DIR=", template)
|
|
self.assertIn("DB_PATH=", template)
|
|
self.assertIn("INPUT_DIR=", template)
|
|
self.assertIn("LOG_DIR=", template)
|
|
self.assertIn("ENV_FILE=", template)
|
|
self.assertIn("WEB_HOST=", template)
|
|
self.assertIn("WEB_PORT=", template)
|
|
self.assertIn("PYTHON_BIN=", template)
|
|
self.assertIn("VENV_DIR=", template)
|
|
self.assertIn("DOWNLOAD_LAYOUT=platform_first_artist", template)
|
|
self.assertIn("DOWNLOAD_SOURCES=", template)
|
|
self.assertIn("OBJECT_BACKEND_NAME=", template)
|
|
self.assertIn("OBJECT_BUCKET=", template)
|
|
self.assertIn("OBJECT_ENDPOINT=", template)
|
|
self.assertIn("OBJECT_CREDENTIAL_ENV_PREFIX=", template)
|
|
self.assertIn("SYNC_WORKERS=", template)
|
|
self.assertIn("UPLOAD_WORKERS=", template)
|
|
|
|
def test_requirements_include_eval_type_backport_for_python38_pydantic_web_models(self):
|
|
requirements = Path("requirements.txt").read_text(encoding="utf-8")
|
|
self.assertIn("eval_type_backport", requirements)
|
|
|
|
def test_requirements_include_jinja2_for_ops_web_templates(self):
|
|
requirements = Path("requirements.txt").read_text(encoding="utf-8")
|
|
self.assertIn("jinja2", requirements)
|
|
|
|
def test_bootstrap_script_uses_remote_host_parameter_name(self):
|
|
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("[string]$RemoteHost", script)
|
|
self.assertNotIn("[string]$Host,", script)
|
|
|
|
def test_bootstrap_script_uses_remote_path_quoting_helpers(self):
|
|
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("function Quote-RemotePathForPosixShell", script)
|
|
self.assertIn("function New-ScpRemoteTarget", script)
|
|
self.assertIn("Quote-RemotePathForPosixShell -Path", script)
|
|
self.assertIn("New-ScpRemoteTarget -Remote $Remote", script)
|
|
self.assertIn("upload_all.sh", script)
|
|
self.assertIn("install_runtime.sh", script)
|
|
self.assertIn("serve_console.sh", script)
|
|
self.assertIn("README.md", script)
|
|
self.assertIn("LICENSE", script)
|
|
self.assertNotIn("${Remote}:$AppHome/app/", script)
|
|
self.assertNotIn("${Remote}:$AppHome/bin/", script)
|
|
|
|
def test_bootstrap_script_refreshes_env_example_when_template_changes(self):
|
|
script = Path("scripts/catalogsync/bootstrap_to_linux.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertNotIn("if (($EnvCheck | Select-Object -Last 1).Trim() -eq \"missing\")", script)
|
|
self.assertIn("catalogsync.env.example", script)
|
|
|
|
def test_bootstrap_script_is_parseable_by_powershell(self):
|
|
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
|
if powershell is None:
|
|
self.skipTest("PowerShell is unavailable in current environment.")
|
|
|
|
script_path = Path("scripts/catalogsync/bootstrap_to_linux.ps1").resolve()
|
|
escaped_path = str(script_path).replace("'", "''")
|
|
parse_command = (
|
|
"$tokens=$null; $errors=$null; "
|
|
f"[void][System.Management.Automation.Language.Parser]::ParseFile('{escaped_path}', [ref]$tokens, [ref]$errors); "
|
|
"if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 }"
|
|
)
|
|
result = subprocess.run(
|
|
[powershell, "-NoProfile", "-Command", parse_command],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
result.returncode,
|
|
msg=f"PowerShell parser errors:\n{result.stderr or result.stdout}",
|
|
)
|
|
|
|
def test_runtime_script_templates_pass_bash_syntax_check(self):
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
self.skipTest("bash is unavailable in current environment.")
|
|
|
|
for template_name in (
|
|
"load_env.sh",
|
|
"download_from_file.sh",
|
|
"download_all.sh",
|
|
"upload_all.sh",
|
|
"install_runtime.sh",
|
|
"serve_console.sh",
|
|
):
|
|
script_path = Path(
|
|
f"scripts/catalogsync/templates/{template_name}"
|
|
).resolve()
|
|
result = subprocess.run(
|
|
[bash, "-n", script_path.as_posix()],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
result.returncode,
|
|
msg=f"bash -n failed for {template_name}:\n{result.stderr or result.stdout}",
|
|
)
|
|
|
|
def test_runtime_script_template_uses_configured_library_dir(self):
|
|
for template_name in ("download_from_file.sh", "download_all.sh"):
|
|
script = Path(
|
|
f"scripts/catalogsync/templates/{template_name}"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn("LIBRARY_DIR", script)
|
|
self.assertIn("DB_PATH", script)
|
|
self.assertIn("INPUT_DIR", script)
|
|
self.assertIn("LOG_DIR", script)
|
|
self.assertIn("PYTHON_BIN", script)
|
|
self.assertIn("PYTHONPATH", script)
|
|
self.assertIn("VENV_DIR", script)
|
|
self.assertIn('if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]', script)
|
|
self.assertIn("musicdl.catalogsync.cli run", script)
|
|
|
|
def test_upload_runtime_script_template_uses_upload_command_and_object_backend_vars(self):
|
|
script = Path("scripts/catalogsync/templates/upload_all.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("DB_PATH", script)
|
|
self.assertIn("LOG_DIR", script)
|
|
self.assertIn("PYTHON_BIN", script)
|
|
self.assertIn("PYTHONPATH", script)
|
|
self.assertIn("VENV_DIR", script)
|
|
self.assertIn('if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]', script)
|
|
self.assertIn("OBJECT_BACKEND_NAME", script)
|
|
self.assertIn("OBJECT_BUCKET", script)
|
|
self.assertIn("OBJECT_ENDPOINT", script)
|
|
self.assertIn("OBJECT_CREDENTIAL_ENV_PREFIX", script)
|
|
self.assertIn("musicdl.catalogsync.cli register-object-backend", script)
|
|
self.assertIn("musicdl.catalogsync.cli upload", script)
|
|
|
|
def test_install_runtime_script_template_sets_up_venv_and_nas_requirements(self):
|
|
script = Path("scripts/catalogsync/templates/install_runtime.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("VENV_DIR", script)
|
|
self.assertIn("PYTHON_BIN", script)
|
|
self.assertIn("requirements.nas.txt", script)
|
|
self.assertIn('APP_DIR="${APP_HOME}/app"', script)
|
|
self.assertIn('REQUIREMENTS_FILE="${APP_DIR}/requirements.txt"', script)
|
|
self.assertIn('SETUP_FILE="${APP_DIR}/setup.py"', script)
|
|
self.assertIn("grep -v '^nodejs-wheel$'", script)
|
|
self.assertIn('"${PYTHON_BIN}" -m venv "${VENV_DIR}"', script)
|
|
self.assertIn('"${RUNTIME_PYTHON_BIN}" -m pip install -r "${NAS_REQUIREMENTS_FILE}"', script)
|
|
self.assertIn('"${RUNTIME_PYTHON_BIN}" -m pip install --no-deps -e "${APP_DIR}"', script)
|
|
load_config_index = script.index('load_env_file "${CONFIG_FILE}"')
|
|
app_dir_index = script.index('APP_DIR="${APP_HOME}/app"')
|
|
self.assertGreater(app_dir_index, load_config_index)
|
|
|
|
def test_runtime_script_templates_include_preflight_and_log_file(self):
|
|
for template_name in (
|
|
"download_from_file.sh",
|
|
"download_all.sh",
|
|
"upload_all.sh",
|
|
"install_runtime.sh",
|
|
"serve_console.sh",
|
|
):
|
|
script = Path(
|
|
f"scripts/catalogsync/templates/{template_name}"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn('[[ -f "${CONFIG_FILE}" ]]', script)
|
|
self.assertIn('command -v "${PYTHON_BIN}"', script)
|
|
self.assertIn("for required_var in", script)
|
|
self.assertIn("DB_PATH", script)
|
|
self.assertIn("LOG_DIR", script)
|
|
self.assertIn("PYTHON_BIN", script)
|
|
self.assertIn('LOG_FILE="${LOG_DIR}/', script)
|
|
self.assertIn('exec > >(tee -a "${LOG_FILE}") 2>&1', script)
|
|
|
|
def test_runtime_script_templates_use_safe_env_loader(self):
|
|
helper = Path("scripts/catalogsync/templates/load_env.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("load_env_file()", helper)
|
|
|
|
for template_name in (
|
|
"download_from_file.sh",
|
|
"download_all.sh",
|
|
"upload_all.sh",
|
|
"install_runtime.sh",
|
|
"serve_console.sh",
|
|
"deploy_and_restart.sh",
|
|
):
|
|
script = Path(
|
|
f"scripts/catalogsync/templates/{template_name}"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn('source "${SCRIPT_DIR}/load_env.sh"', script)
|
|
self.assertIn('load_env_file "${CONFIG_FILE}"', script)
|
|
self.assertNotIn('source "${CONFIG_FILE}"', script)
|
|
|
|
def test_serve_console_runtime_script_template_uses_serve_command_and_web_vars(self):
|
|
script = Path("scripts/catalogsync/templates/serve_console.sh").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("DB_PATH", script)
|
|
self.assertIn("ENV_FILE", script)
|
|
self.assertIn("WEB_HOST", script)
|
|
self.assertIn("WEB_PORT", script)
|
|
self.assertIn("LOG_DIR", script)
|
|
self.assertIn("PYTHON_BIN", script)
|
|
self.assertIn("PYTHONPATH", script)
|
|
self.assertIn("VENV_DIR", script)
|
|
self.assertIn("validate_port", script)
|
|
self.assertIn("WEB_PORT must be an integer in range 1..65535", script)
|
|
self.assertIn(
|
|
'if [[ -n "${VENV_DIR:-}" && -x "${VENV_DIR}/bin/python" ]]',
|
|
script,
|
|
)
|
|
self.assertIn("musicdl.catalogsync.cli serve", script)
|