xiaoming/musicdl-catalog-sync-suite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: Music_Server, MusicFree, catalog-sync

Browse Source
This commit is contained in:
Codex
2026-05-23 16:51:14 +08:00
commit 069af30dba
847 changed files with 179878 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalog-sync/musicdl/modules/utils/__init__.py
+17
View File
@@ -0,0 +1,17 @@
'''initialize'''
from .data import SongInfo
from .hls import HLSDownloader
from .ip import RandomIPGenerator
from .quarkparser import QuarkParser
from .lanzouyparser import LanZouYParser
from .songinfoutils import SongInfoUtils
from .modulebuilder import BaseModuleBuilder
from .hosts import obtainhostname, hostmatchessuffix
from .importutils import optionalimport, optionalimportfrom
from .lyric import WhisperLRC, LyricSearchClient, extractdurationsecondsfromlrc, cleanlrc
from .logger import LoggerHandle, colorize, printtable, printfullline, smarttrunctable, cursorpickintable
from .misc import (
AudioLinkTester, legalizestring, touchdir, seconds2hms, byte2mb, cachecookies, resp2json, isvalidresp, safeextractfromdict, replacefile,
usedownloadheaderscookies, useparseheaderscookies, usesearchheaderscookies, cookies2dict, cookies2string, estimatedurationwithfilesizebr,
estimatedurationwithfilelink, searchdictbykey, shortenpathsinsonginfos, naiveguessextfromaudiobytes,
)
catalog-sync/musicdl/modules/utils/appleutils.py
+849
View File
@@ -0,0 +1,849 @@
'''
Function:
Implementation of AppleMusicClient Utils (Refer To https://github.com/glomatico/gamdl)
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import re
import os
import io
import m3u8
import uuid
import json
import base64
import shutil
import datetime
import requests
import subprocess
from enum import Enum
from typing import Any
from pathlib import Path
from xml.dom import minidom
from mutagen.mp4 import MP4
from xml.etree import ElementTree
from dataclasses import dataclass
from platformdirs import user_log_dir
from pywidevine import PSSH, Cdm, Device
from urllib.parse import parse_qs, urlparse
from .misc import safeextractfromdict, resp2json
from pywidevine.license_protocol_pb2 import WidevinePsshData
'''settings'''
FOURCC_MAP = {"h264": "avc1", "h265": "hvc1"}
MEDIA_TYPE_STR_MAP = {1: "Song", 6: "Music Video"}
LEGACY_SONG_CODECS = {"aac-legacy", "aac-he-legacy"}
IMAGE_FILE_EXTENSION_MAP = {"jpeg": ".jpg", "tiff": ".tif"}
MEDIA_RATING_STR_MAP = {0: "None", 1: "Explicit", 2: "Clean"}
MP4_FORMAT_CODECS = ["ec-3", "hvc1", "audio-atmos", "audio-ec3"]
SONG_MEDIA_TYPE = {"song", "songs", "library-songs"}
ALBUM_MEDIA_TYPE = {"album", "albums", "library-albums"}
MUSIC_VIDEO_MEDIA_TYPE = {"music-video", "music-videos", "library-music-videos"}
ARTIST_MEDIA_TYPE = {"artist", "artists", "library-artists"}
UPLOADED_VIDEO_MEDIA_TYPE = {"post", "uploaded-videos"}
PLAYLIST_MEDIA_TYPE = {"playlist", "playlists", "library-playlists"}
UPLOADED_VIDEO_QUALITY_RANK = ["1080pHdVideo", "720pHdVideo", "sdVideoWithPlusAudio", "sdVideo", "sd480pVideo", "provisionalUploadVideo"]
SONG_CODEC_REGEX_MAP = {
"aac": r"audio-stereo-\d+", "aac-he": r"audio-HE-stereo-\d+", "aac-binaural": r"audio-stereo-\d+-binaural", "aac-downmix": r"audio-stereo-\d+-downmix", "aac-he-binaural": r"audio-HE-stereo-\d+-binaural",
"aac-he-downmix": r"audio-HE-stereo-\d+-downmix", "atmos": r"audio-atmos-.*", "ac3": r"audio-ac3-.*", "alac": r"audio-alac-.*",
}
DRM_DEFAULT_KEY_MAPPING = {
"urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed": ("data:text/plain;base64,AAAAOHBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAABgSEAAAAAAAAAAAczEvZTEgICBI88aJmwY="),
"com.microsoft.playready": ("data:text/plain;charset=UTF-16;base64,vgEAAAEAAQC0ATwAVwBSAE0ASABFAEEARABFAFIAIAB4AG0AbABuAHMAPQAiAGgAdAB0AHAAOgAvAC8AcwBjAGgAZQBtAGEAcwAuAG0AaQBjAHIAbwBzAG8AZgB0AC4AYwBvAG0ALwBEAFIATQAvADIAMAAwADcALwAwADMALwBQAGwAYQB5AFIAZQBhAGQAeQBIAGUAYQBkAGUAcgAiACAAdgBlAHIAcwBpAG8AbgA9ACIANAAuADMALgAwAC4AMAAiAD4APABEAEEAVABBAD4APABQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsASQBEAFMAPgA8AEsASQBEACAAQQBMAEcASQBEAD0AIgBBAEUAUwBDAEIAQwAiACAAVgBBAEwAVQBFAD0AIgBBAEEAQQBBAEEAQQBBAEEAQQBBAEIAegBNAFMAOQBsAE0AUwBBAGcASQBBAD0APQAiAD4APAAvAEsASQBEAD4APAAvAEsASQBEAFMAPgA8AC8AUABSAE8AVABFAEMAVABJAE4ARgBPAD4APAAvAEQAQQBUAEEAPgA8AC8AVwBSAE0ASABFAEEARABFAFIAPgA="),
"com.apple.streamingkeydelivery": "skd://itunes.apple.com/P000000000/s1/e1",
}
DEFAULT_SONG_DECRYPTION_KEY = "32b8ade1769e26b1ffb8986352793fc6"
HARDCODED_WVD = """V1ZEAgIDAASoMIIEpAIBAAKCAQEAwnCFAPXy4U1J7p1NohAS+xl040f5FBaE/59bPp301bGz0UGFT9VoEtY3vaeakKh/d319xTNvCSWsEDRaMmp/wSnMiEZUkkl04872jx2uHuR4k6KYuuJoqhsIo1TwUBueFZynHBUJzXQeW8Eb1tYAROGwp8W7r+b0RIjHC89RFnfVXpYlF5I6McktyzJNSOwlQbMqlVihfSUkv3WRd3HFmA0Oxay51CEIkoTlNTHVlzVyhov5eHCDSp7QENRgaaQ03jC/CcgFOoQymhsBtRCM0CQmfuAHjA9e77R6m/GJPy75G9fqoZM1RMzVDHKbKZPd3sFd0c0+77gLzW8cWEaaHwIDAQABAoIBAQCB2pN46MikHvHZIcTPDt0eRQoDH/YArGl2Lf7J+sOgU2U7wv49KtCug9IGHwDiyyUVsAFmycrF2RroV45FTUq0vi2SdSXV7Kjb20Ren/vBNeQw9M37QWmU8Sj7q6YyWb9hv5T69DHvvDTqIjVtbM4RMojAAxYti5hmjNIh2PrWfVYWhXxCQ/WqAjWLtZBM6Oww1byfr5I/wFogAKkgHi8wYXZ4LnIC8V7jLAhujlToOvMMC9qwcBiPKDP2FO+CPSXaqVhH+LPSEgLggnU3EirihgxovbLNAuDEeEbRTyR70B0lW19tLHixso4ZQa7KxlVUwOmrHSZf7nVuWqPpxd+BAoGBAPQLyJ1IeRavmaU8XXxfMdYDoc8+xB7v2WaxkGXb6ToX1IWPkbMz4yyVGdB5PciIP3rLZ6s1+ruuRRV0IZ98i1OuN5TSR56ShCGg3zkd5C4L/xSMAz+NDfYSDBdO8BVvBsw21KqSRUi1ctL7QiIvfedrtGb5XrE4zhH0gjXlU5qZAoGBAMv2segn0Jx6az4rqRa2Y7zRx4iZ77JUqYDBI8WMnFeR54uiioTQ+rOs3zK2fGIWlrn4ohco/STHQSUTB8oCOFLMx1BkOqiR+UyebO28DJY7+V9ZmxB2Guyi7W8VScJcIdpSOPyJFOWZQKXdQFW3YICD2/toUx/pDAJh1sEVQsV3AoGBANyyp1rthmvoo5cVbymhYQ08vaERDwU3PLCtFXu4E0Ow90VNn6Ki4ueXcv/gFOp7pISk2/yuVTBTGjCblCiJ1en4HFWekJwrvgg3Vodtq8Okn6pyMCHRqvWEPqD5hw6rGEensk0K+FMXnF6GULlfn4mgEkYpb+PvDhSYvQSGfkPJAoGAF/bAKFqlM/1eJEvU7go35bNwEiij9Pvlfm8y2L8Qj2lhHxLV240CJ6IkBz1Rl+S3iNohkT8LnwqaKNT3kVB5daEBufxMuAmOlOX4PmZdxDj/r6hDg8ecmjj6VJbXt7JDd/c5ItKoVeGPqu035dpJyE+1xPAY9CLZel4scTsiQTkCgYBt3buRcZMwnc4qqpOOQcXK+DWD6QvpkcJ55ygHYw97iP/lF4euwdHd+I5b+11pJBAao7G0fHX3eSjqOmzReSKboSe5L8ZLB2cAI8AsKTBfKHWmCa8kDtgQuI86fUfirCGdhdA9AVP2QXN2eNCuPnFWi0WHm4fYuUB5be2c18ucxAb9CAESmgsK3QMIAhIQ071yBlsbLoO2CSB9Ds0cmRif6uevBiKOAjCCAQoCggEBAMJwhQD18uFNSe6dTaIQEvsZdONH+RQWhP+fWz6d9NWxs9FBhU/VaBLWN72nmpCof3d9fcUzbwklrBA0WjJqf8EpzIhGVJJJdOPO9o8drh7keJOimLriaKobCKNU8FAbnhWcpxwVCc10HlvBG9bWAEThsKfFu6/m9ESIxwvPURZ31V6WJReSOjHJLcsyTUjsJUGzKpVYoX0lJL91kXdxxZgNDsWsudQhCJKE5TUx1Zc1coaL+Xhwg0qe0BDUYGmkNN4wvwnIBTqEMpobAbUQjNAkJn7gB4wPXu+0epvxiT8u+RvX6qGTNUTM1QxymymT3d7BXdHNPu+4C81vHFhGmh8CAwEAASjwIkgBUqoBCAEQABqBAQQlRbfiBNDb6eU6aKrsH5WJaYszTioXjPLrWN9dqyW0vwfT11kgF0BbCGkAXew2tLJJqIuD95cjJvyGUSN6VyhL6dp44fWEGDSBIPR0mvRq7bMP+m7Y/RLKf83+OyVJu/BpxivQGC5YDL9f1/A8eLhTDNKXs4Ia5DrmTWdPTPBL8SIgyfUtg3ofI+/I9Tf7it7xXpT0AbQBJfNkcNXGpO3JcBMSgAIL5xsXK5of1mMwAl6ygN1Gsj4aZ052otnwN7kXk12SMsXheWTZ/PYh2KRzmt9RPS1T8hyFx/Kp5VkBV2vTAqqWrGw/dh4URqiHATZJUlhO7PN5m2Kq1LVFdXjWSzP5XBF2S83UMe+YruNHpE5GQrSyZcBqHO0QrdPcU35GBT7S7+IJr2AAXvnjqnb8yrtpPWN2ZW/IWUJN2z4vZ7/HV4aj3OZhkxC1DIMNyvsusUKoQQuf8gwKiEe8cFwbwFSicywlFk9la2IPe8oFShcxAzHLCCn/TIYUAvEL3/4LgaZvqWm80qCPYbgIP5HT8hPYkKWJ4WYknEWK+3InbnkzteFfGrQFCq4CCAESEGnj6Ji7LD+4o7MoHYT4jBQYjtW+kQUijgIwggEKAoIBAQDY9um1ifBRIOmkPtDZTqH+CZUBbb0eK0Cn3NHFf8MFUDzPEz+emK/OTub/hNxCJCao//pP5L8tRNUPFDrrvCBMo7Rn+iUb+mA/2yXiJ6ivqcN9Cu9i5qOU1ygon9SWZRsujFFB8nxVreY5Lzeq0283zn1Cg1stcX4tOHT7utPzFG/ReDFQt0O/GLlzVwB0d1sn3SKMO4XLjhZdncrtF9jljpg7xjMIlnWJUqxDo7TQkTytJmUl0kcM7bndBLerAdJFGaXc6oSY4eNy/IGDluLCQR3KZEQsy/mLeV1ggQ44MFr7XOM+rd+4/314q/deQbjHqjWFuVr8iIaKbq+R63ShAgMBAAEo8CISgAMii2Mw6z+Qs1bvvxGStie9tpcgoO2uAt5Zvv0CDXvrFlwnSbo+qR71Ru2IlZWVSbN5XYSIDwcwBzHjY8rNr3fgsXtSJty425djNQtF5+J2jrAhf3Q2m7EI5aohZGpD2E0cr+dVj9o8x0uJR2NWR8FVoVQSXZpad3M/4QzBLNto/tz+UKyZwa7Sc/eTQc2+ZcDS3ZEO3lGRsH864Kf/cEGvJRBBqcpJXKfG+ItqEW1AAPptjuggzmZEzRq5xTGf6or+bXrKjCpBS9G1SOyvCNF1k5z6lG8KsXhgQxL6ADHMoulxvUIihyPY5MpimdXfUdEQ5HA2EqNiNVNIO4qP007jW51yAeThOry4J22xs8RdkIClOGAauLIl0lLA4flMzW+VfQl5xYxP0E5tuhn0h+844DslU8ZF7U1dU2QprIApffXD9wgAACk26Rggy8e96z8i86/+YYyZQkc9hIdCAERrgEYCEbByzONrdRDs1MrS/ch1moV5pJv63BIKvQHGvLkaFwoMY29tcGFueV9uYW1lEgd1bmtub3duGioKCm1vZGVsX25hbWUSHEFuZHJvaWQgU0RLIGJ1aWx0IGZvciB4ODZfNjQaGwoRYXJjaGl0ZWN0dXJlX25hbWUSBng4Nl82NBodCgtkZXZpY2VfbmFtZRIOZ2VuZXJpY194ODZfNjQaIAoMcHJvZHVjdF9uYW1lEhBzZGtfcGhvbmVfeDg2XzY0GmMKCmJ1aWxkX2luZm8SVUFuZHJvaWQvc2RrX3Bob25lX3g4Nl82NC9nZW5lcmljX3g4Nl82NDo5L1BTUjEuMTgwNzIwLjAxMi80OTIzMjE0OnVzZXJkZWJ1Zy90ZXN0LWtleXMaHgoUd2lkZXZpbmVfY2RtX3ZlcnNpb24SBjE0LjAuMBokCh9vZW1fY3J5cHRvX3NlY3VyaXR5X3BhdGNoX2xldmVsEgEwMg4QASAAKA0wAEAASABQAA=="""
APPLE_MUSIC_COOKIE_DOMAIN = ".music.apple.com"
AMP_API_URL = "https://amp-api.music.apple.com"
ITUNES_PAGE_API_URL = "https://music.apple.com"
APPLE_MUSIC_HOMEPAGE_URL = "https://music.apple.com"
ITUNES_LOOKUP_API_URL = "https://itunes.apple.com/lookup"
WEBPLAYBACK_API_URL = "https://play.itunes.apple.com/WebObjects/MZPlay.woa/wa/webPlayback"
LICENSE_API_URL = "https://play.itunes.apple.com/WebObjects/MZPlay.woa/wa/acquireWebPlaybackLicense"
STOREFRONT_IDS = {
"AE": "143481-2,32", "AG": "143540-2,32", "AI": "143538-2,32", "AL": "143575-2,32", "AM": "143524-2,32", "AO": "143564-2,32", "AR": "143505-28,32", "AT": "143445-4,32",
"AU": "143460-27,32", "AZ": "143568-2,32", "BB": "143541-2,32", "BE": "143446-2,32", "BF": "143578-2,32", "BG": "143526-2,32", "BH": "143559-2,32", "BJ": "143576-2,32",
"BM": "143542-2,32", "BN": "143560-2,32", "BO": "143556-28,32", "BR": "143503-15,32", "BS": "143539-2,32", "BT": "143577-2,32", "BW": "143525-2,32", "BY": "143565-2,32",
"BZ": "143555-2,32", "CA": "143455-6,32", "CG": "143582-2,32", "CH": "143459-57,32", "CM": "143574-2,32", "CL": "143483-28,32", "CN": "143465-19,32", "CO": "143501-28,32",
"CR": "143495-28,32", "CV": "143580-2,32", "CY": "143557-2,32", "CZ": "143489-2,32", "DE": "143443-4,32", "DK": "143458-2,32", "DM": "143545-2,32", "DO": "143508-28,32",
"DZ": "143563-2,32", "EC": "143509-28,32", "EE": "143518-2,32", "EG": "143516-2,32", "ES": "143454-8,32", "FI": "143447-2,32", "FJ": "143583-2,32", "FM": "143591-2,32",
"FR": "143442-3,32", "GB": "143444-2,32", "GD": "143546-2,32", "GH": "143573-2,32", "GM": "143584-2,32", "GR": "143448-2,32", "GT": "143504-28,32", "GW": "143585-2,32",
"GY": "143553-2,32", "HK": "143463-45,32", "HN": "143510-28,32", "HR": "143494-2,32", "HU": "143482-2,32", "ID": "143476-2,32", "IE": "143449-2,32", "IL": "143491-2,32",
"IN": "143467-2,32", "IS": "143558-2,32", "IT": "143450-7,32", "JM": "143511-2,32", "JO": "143528-2,32", "JP": "143462-9,32", "KE": "143529-2,32", "KG": "143586-2,32",
"KH": "143579-2,32", "KN": "143548-2,32", "KR": "143466-13,32", "KW": "143493-2,32", "KY": "143544-2,32", "KZ": "143517-2,32", "LA": "143587-2,32", "LB": "143497-2,32",
"LC": "143549-2,32", "LK": "143486-2,32", "LR": "143588-2,32", "LT": "143520-2,32", "LU": "143451-2,32", "LV": "143519-2,32", "MD": "143523-2,32", "MG": "143531-2,32",
"MK": "143530-2,32", "ML": "143532-2,32", "MN": "143592-2,32", "MO": "143515-45,32", "MR": "143590-2,32", "MS": "143547-2,32", "MT": "143521-2,32", "MU": "143533-2,32",
"MW": "143589-2,32", "MX": "143468-28,32", "MY": "143473-2,32", "MZ": "143593-2,32", "NA": "143594-2,32", "NE": "143534-2,32", "NG": "143561-2,32", "NI": "143512-28,32",
"NL": "143452-10,32", "NO": "143457-2,32", "NP": "143484-2,32", "NZ": "143461-27,32", "OM": "143562-2,32", "PA": "143485-28,32", "PE": "143507-28,32", "PG": "143597-2,32",
"PH": "143474-2,32", "PK": "143477-2,32", "PL": "143478-2,32", "PT": "143453-24,32", "PW": "143595-2,32", "PY": "143513-28,32", "QA": "143498-2,32", "RO": "143487-2,32",
"RU": "143469-16,32", "SA": "143479-2,32", "SB": "143601-2,32", "SC": "143599-2,32", "SE": "143456-17,32", "SG": "143464-19,32", "SI": "143499-2,32", "SK": "143496-2,32",
"SL": "143600-2,32", "SN": "143535-2,32", "SR": "143554-2,32", "ST": "143598-2,32", "SV": "143506-28,32", "SZ": "143602-2,32", "TC": "143552-2,32", "TD": "143581-2,32",
"TH": "143475-2,32", "TJ": "143603-2,32", "TM": "143604-2,32", "TN": "143536-2,32", "TR": "143480-2,32", "TT": "143551-2,32", "TW": "143470-18,32", "TZ": "143572-2,32",
"UA": "143492-2,32", "UG": "143537-2,32", "US": "143441-1,32", "UY": "143514-2,32", "UZ": "143566-2,32", "VC": "143550-2,32", "VE": "143502-28,32", "VG": "143543-2,32",
"VN": "143471-2,32", "YE": "143571-2,32", "ZA": "143472-2,32", "ZW": "143605-2,32",
}
'''CoverFormat'''
class CoverFormat(Enum):
JPG = "jpg"
PNG = "png"
RAW = "raw"
'''RemuxFormatMusicVideo'''
class RemuxFormatMusicVideo(Enum):
M4V = "m4v"
MP4 = "mp4"
'''SyncedLyricsFormat'''
class SyncedLyricsFormat(Enum):
LRC = "lrc"
SRT = "srt"
TTML = "ttml"
'''MediaType'''
class MediaType(Enum):
SONG = 1
MUSIC_VIDEO = 6
def __str__(self): return MEDIA_TYPE_STR_MAP[self.value]
def __int__(self): return self.value
'''MediaRating'''
class MediaRating(Enum):
NONE = 0
EXPLICIT = 1
CLEAN = 2
def __str__(self): return MEDIA_RATING_STR_MAP[self.value]
def __int__(self): return self.value
'''MediaFileFormat'''
class MediaFileFormat(Enum):
MP4 = "mp4"
M4V = "m4v"
M4A = "m4a"
'''DownloadMode'''
class DownloadMode(Enum):
NM3U8DLRE = "nm3u8dlre"
'''RemuxMode'''
class RemuxMode(Enum):
FFMPEG = "ffmpeg"
MP4BOX = "mp4box"
'''SongCodec'''
class SongCodec(Enum):
AAC_LEGACY = "aac-legacy"
AAC_HE_LEGACY = "aac-he-legacy"
AAC = "aac"
AAC_HE = "aac-he"
AAC_BINAURAL = "aac-binaural"
AAC_DOWNMIX = "aac-downmix"
AAC_HE_BINAURAL = "aac-he-binaural"
AAC_HE_DOWNMIX = "aac-he-downmix"
ATMOS = "atmos"
AC3 = "ac3"
ALAC = "alac"
ASK = "ask"
def islegacy(self): return self.value in LEGACY_SONG_CODECS
'''MusicVideoCodec'''
class MusicVideoCodec(Enum):
H264 = "h264"
H265 = "h265"
ASK = "ask"
def fourcc(self): return FOURCC_MAP[self.value]
'''MusicVideoResolution'''
class MusicVideoResolution(Enum):
R240P = "240p"
R360P = "360p"
R480P = "480p"
R540P = "540p"
R720P = "720p"
R1080P = "1080p"
R1440P = "1440p"
R2160P = "2160p"
def __int__(self): return int(self.value[:-1])
'''Lyrics'''
@dataclass
class Lyrics:
synced: str = None
unsynced: str = None
'''MediaTags'''
@dataclass
class MediaTags:
album: str = None
album_artist: str = None
album_id: int = None
album_sort: str = None
artist: str = None
artist_id: int = None
artist_sort: str = None
comment: str = None
compilation: bool = None
composer: str = None
composer_id: int = None
composer_sort: str = None
copyright: str = None
date: datetime.date | str = None
disc: int = None
disc_total: int = None
gapless: bool = None
genre: str = None
genre_id: int = None
lyrics: str = None
media_type: MediaType = None
rating: MediaRating = None
storefront: str = None
title: str = None
title_id: int = None
title_sort: str = None
track: int = None
track_total: int = None
xid: str = None
'''asmp4tags'''
def asmp4tags(self, date_format: str = None):
disc_mp4 = [self.disc if self.disc is not None else 0, self.disc_total if self.disc_total is not None else 0]
if disc_mp4[0] == 0 and disc_mp4[1] == 0: disc_mp4 = None
track_mp4 = [self.track if self.track is not None else 0, self.track_total if self.track_total is not None else 0]
if track_mp4[0] == 0 and track_mp4[1] == 0: track_mp4 = None
if isinstance(self.date, datetime.date):
if date_format is None: date_mp4 = self.date.isoformat()
else: date_mp4 = self.date.strftime(date_format)
elif isinstance(self.date, str):
date_mp4 = self.date
else:
date_mp4 = None
mp4_tags = {
"\xa9alb": self.album, "aART": self.album_artist, "plID": self.album_id, "soal": self.album_sort, "\xa9ART": self.artist, "atID": self.artist_id, "soar": self.artist_sort, "\xa9cmt": self.comment, "xid": self.xid,
"cpil": bool(self.compilation) if self.compilation is not None else None, "\xa9wrt": self.composer, "cmID": self.composer_id, "soco": self.composer_sort, "cprt": self.copyright, "\xa9day": date_mp4, "trkn": track_mp4,
"disk": disc_mp4, "pgap": bool(self.gapless) if self.gapless is not None else None, "\xa9lyr": self.lyrics, "geID": self.genre_id, "stik": int(self.media_type) if self.media_type is not None else None, "\xa9nam": self.title,
"\xa9gen": self.genre, "rtng": int(self.rating) if self.rating is not None else None, "sfID": self.storefront, "cnID": self.title_id, "sonm": self.title_sort,
}
return {k: ([v] if not isinstance(v, bool) else v) for k, v in mp4_tags.items() if v is not None}
'''PlaylistTags'''
@dataclass
class PlaylistTags:
playlist_artist: str = None
playlist_id: int = None
playlist_title: str = None
playlist_track: int = None
'''StreamInfo'''
@dataclass
class StreamInfo:
stream_url: str = None
widevine_pssh: str = None
playready_pssh: str = None
fairplay_key: str = None
codec: str = None
width: int = None
height: int = None
'''StreamInfoAv'''
@dataclass
class StreamInfoAv:
media_id: str = None
video_track: StreamInfo = None
audio_track: StreamInfo = None
file_format: MediaFileFormat = None
'''DecryptionKey'''
@dataclass
class DecryptionKey:
kid: str = None
key: str = None
'''DecryptionKeyAv'''
@dataclass
class DecryptionKeyAv:
video_track: DecryptionKey = None
audio_track: DecryptionKey = None
'''DownloadItem'''
@dataclass
class DownloadItem:
media_metadata: dict = None
playlist_metadata: dict = None
random_uuid: str = None
lyrics: Lyrics = None
media_tags: MediaTags = None
extra_tags: dict = None
playlist_tags: PlaylistTags = None
stream_info: StreamInfoAv = None
decryption_key: DecryptionKeyAv = None
cover_url_template: str = None
cover_url: str = None
staged_path: str = None
final_path: str = None
playlist_file_path: str = None
synced_lyrics_path: str = None
cover_path: str = None
flat_filter_result: Any = None
error: Exception = None
'''UrlInfo'''
@dataclass
class UrlInfo:
storefront: str = None
type: str = None
slug: str = None
id: str = None
sub_id: str = None
library_storefront: str = None
library_type: str = None
library_id: str = None
'''AppleMusicClientAPIUtils'''
class AppleMusicClientAPIUtils:
def __init__(self, storefront: str = "us", language: str = "en-US", media_user_token: str | None = None, developer_token: str | None = None) -> None:
self.storefront = storefront
self.language = language
self.media_user_token = media_user_token
self.token = developer_token
@property
def active_subscription(self) -> bool: return safeextractfromdict(getattr(self, "account_info", {}), ['meta', 'subscription', 'active'], False)
@property
def account_restrictions(self) -> dict | None: return safeextractfromdict(getattr(self, "account_info", {}), ['data', 0, 'attributes', 'restrictions'], None)
'''createfromnetscapecookies'''
@classmethod
def createfromnetscapecookies(cls, cookies: dict, request_overrides: dict = None, *args, **kwargs) -> "AppleMusicClientAPIUtils":
request_overrides = request_overrides or {}
media_user_token = cookies.get('media-user-token')
if not media_user_token: raise ValueError('"media-user-token" is not configured in cookies.')
return cls.create(storefront=None, media_user_token=media_user_token, developer_token=None, request_overrides=request_overrides, *args, **kwargs)
'''createfromwrapper'''
@classmethod
def createfromwrapper(cls, wrapper_account_url: str = "http://127.0.0.1:30020/", request_overrides: dict = None, *args, **kwargs) -> "AppleMusicClientAPIUtils":
request_overrides = request_overrides or {}
wrapper_account_response = requests.get(wrapper_account_url)
wrapper_account_response.raise_for_status()
wrapper_account_info = wrapper_account_response.json()
return cls.create(storefront=None, media_user_token=wrapper_account_info["music_token"], developer_token=wrapper_account_info["dev_token"], request_overrides=request_overrides, *args, **kwargs)
'''create'''
@classmethod
def create(cls, storefront: str | None = "us", language: str = "en-US", media_user_token: str | None = None, developer_token: str | None = None, request_overrides: dict = None) -> "AppleMusicClientAPIUtils":
request_overrides = request_overrides or {}
api = cls(storefront=storefront, language=language, media_user_token=media_user_token, developer_token=developer_token)
api.initialize(request_overrides=request_overrides)
return api
'''initialize'''
def initialize(self, request_overrides: dict = None) -> None:
request_overrides = request_overrides or {}
self.initializeclient(); self.initializetoken(request_overrides=request_overrides); self.initializeaccountinfo(request_overrides=request_overrides)
'''initializeclient'''
def initializeclient(self) -> None:
self.client = requests.Session()
self.client.headers.update({
"accept": "*/*", "accept-language": "en-US", "origin": APPLE_MUSIC_HOMEPAGE_URL, "priority": "u=1, i", "referer": APPLE_MUSIC_HOMEPAGE_URL, "sec-ch-ua": '"Google Chrome";v="137", "Chromium";v="137", "Not/A)Brand";v="24"', "sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"', "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
})
'''gettoken'''
def gettoken(self, request_overrides: dict = None) -> str:
request_overrides = request_overrides or {}
(resp := self.client.get(APPLE_MUSIC_HOMEPAGE_URL, params={"l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
index_js_uri_match = re.search(r"/(assets/index-legacy[~-][^/\"]+\.js)", resp.text)
if not index_js_uri_match: raise Exception("index.js URI not found in Apple Music homepage")
index_js_uri = index_js_uri_match.group(1)
(resp := self.client.get(f"{APPLE_MUSIC_HOMEPAGE_URL}/{index_js_uri}", params={"l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
token_match = re.search('(?=eyJh)(.*?)(?=")', resp.text)
if not token_match: raise Exception("Token not found in index.js page")
token = token_match.group(1)
return token
'''initializetoken'''
def initializetoken(self, request_overrides: dict = None) -> None:
request_overrides = request_overrides or {}
self.token = self.token or self.gettoken(request_overrides=request_overrides)
self.client.headers.update({"authorization": f"Bearer {self.token}"})
'''initializeaccountinfo'''
def initializeaccountinfo(self, request_overrides: dict = None) -> None:
request_overrides = request_overrides or {}
if not self.media_user_token: return
self.client.cookies.update({"media-user-token": self.media_user_token})
self.account_info = self.getaccountinfo(request_overrides=request_overrides)
self.storefront = safeextractfromdict(self.account_info, ['meta', 'subscription', 'storefront'], 'us')
'''getaccountinfo'''
def getaccountinfo(self, meta: str | None = "subscription", request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/me/account", params={**({"meta": meta} if meta else {}), **{"l": self.language}}, allow_redirects=True, **request_overrides)).raise_for_status()
account_info = resp2json(resp=resp)
if not "data" in account_info or (meta and "meta" not in account_info): raise Exception("Error getting account info: ", resp.text)
return account_info
'''getsong'''
def getsong(self, song_id: str, extend: str = "extendedAssetUrls", include: str = "lyrics,albums", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/songs/{song_id}", params={"extend": extend, "include": include, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
song = resp2json(resp=resp)
if not ("data" in song): raise Exception("Error getting song: ", resp.text)
return song
'''getmusicvideo'''
def getmusicvideo(self, music_video_id: str, include: str = "albums", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/music-videos/{music_video_id}", params={"include": include, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
music_video = resp2json(resp=resp)
if not ("data" in music_video): raise Exception("Error getting music video: ", resp.text)
return music_video
'''getuploadedvideo'''
def getuploadedvideo(self, post_id: str, request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/uploaded-videos/{post_id}", params={"l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
uploaded_video = resp2json(resp=resp)
if not ("data" in uploaded_video): raise Exception("Error getting uploaded video: ", resp.text)
return uploaded_video
'''getalbum'''
def getalbum(self, album_id: str, extend: str = "extendedAssetUrls", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/albums/{album_id}", params={"extend": extend, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
album = resp2json(resp=resp)
if not ("data" in album): raise Exception("Error getting album: ", resp.text)
return album
'''getplaylist'''
def getplaylist(self, playlist_id: str, limit_tracks: int = 300, extend: str = "extendedAssetUrls", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/playlists/{playlist_id}", params={"limit[tracks]": limit_tracks, "extend": extend, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
playlist = resp2json(resp=resp)
if not ("data" in playlist): raise Exception("Error getting playlist: ", resp.text)
return playlist
'''getartist'''
def getartist(self, artist_id: str, include: str = "albums,music-videos", limit: int = 100, request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/artists/{artist_id}", params={"include": include, "l": self.language, **{f"limit[{_include}]": limit for _include in include.split(",")}}, allow_redirects=True, **request_overrides)).raise_for_status()
artist = resp2json(resp=resp)
if not ("data" in artist): raise Exception("Error getting artist:", resp.text)
return artist
'''getlibraryalbum'''
def getlibraryalbum(self, album_id: str, extend: str = "extendedAssetUrls", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/me/library/albums/{album_id}", params={"extend": extend, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
album = resp2json(resp=resp)
if not ("data" in album): raise Exception("Error getting library album: ", resp.text)
return album
'''getlibraryplaylist'''
def getlibraryplaylist(self, playlist_id: str, include: str = "tracks", limit: int = 100, extend: str = "extendedAssetUrls", request_overrides: dict = None) -> dict | None:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/me/library/playlists/{playlist_id}", params={"include": include, **{f"limit[{_include}]": limit for _include in include.split(",")}, "extend": extend, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
playlist = resp2json(resp=resp)
if not ("data" in playlist): raise Exception("Error getting library playlist: ", resp.text)
return playlist
'''getsearchresults'''
def getsearchresults(self, term: str, types: str = "songs,music-videos,albums,playlists,artists", limit: int = 50, offset: int = 0, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{AMP_API_URL}/v1/catalog/{self.storefront}/search", params={"term": term, "types": types, "limit": limit, "offset": offset, "l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
search_results = resp2json(resp=resp)
if not ("results" in search_results): raise Exception("Error searching: ", resp.text)
return search_results
'''extendapidata'''
def extendapidata(self, api_response: dict, extend: str = "extendedAssetUrls", request_overrides: dict = None):
request_overrides = request_overrides or {}
next_uri: str = api_response.get("next")
if not next_uri: return
next_uri_params = parse_qs(urlparse(next_uri).query)
limit = int(next_uri_params["offset"][0])
while next_uri:
extended_api_data = self.getextendedapidata(next_uri, limit, extend, request_overrides)
yield extended_api_data
next_uri = extended_api_data.get("next")
'''getextendedapidata'''
def getextendedapidata(self, next_uri: str, limit: int, extend: str, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.get(AMP_API_URL + next_uri, params={"limit": limit, "extend": extend, "l": self.language, **parse_qs(urlparse(next_uri).query)}, allow_redirects=True, **request_overrides)).raise_for_status()
extended_api_data = resp2json(resp=resp)
if not ("data" in extended_api_data): raise Exception("Error getting extended API data: ", resp.text)
return extended_api_data
'''getwebplayback'''
def getwebplayback(self, track_id: str, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.post(WEBPLAYBACK_API_URL, json={"salableAdamId": track_id, "language": self.language}, params={"l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
webplayback = resp2json(resp=resp)
if not ("songList" in webplayback): raise Exception("Error getting webplayback: ", resp.text)
return webplayback
'''getlicenseexchange'''
def getlicenseexchange(self, track_id: str, track_uri: str, challenge: str, key_system: str = "com.widevine.alpha", request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.post(LICENSE_API_URL, json={"challenge": challenge, "key-system": key_system, "uri": track_uri, "adamId": track_id, "isLibrary": False, "user-initiated": True}, params={"l": self.language}, allow_redirects=True, **request_overrides)).raise_for_status()
license_exchange = resp2json(resp=resp)
if not ("license" in license_exchange): raise Exception("Error getting license exchange: ", resp.text)
return license_exchange
'''AppleMusicClientItunesApiUtils'''
class AppleMusicClientItunesApiUtils:
def __init__(self, storefront: str = "us", language: str = "en-US") -> None:
self.storefront = storefront
self.language = language
self.initialize()
'''initialize'''
def initialize(self) -> None:
self.initializestorefrontid()
self.initializeclient()
'''initializestorefrontid'''
def initializestorefrontid(self) -> None:
try: self.storefront_id = STOREFRONT_IDS[self.storefront.upper()]
except KeyError: raise Exception(f"No storefront id for {self.storefront}")
'''initializeclient'''
def initializeclient(self) -> None:
self.client = requests.Session()
self.client.headers.update({"X-Apple-Store-Front": f"{self.storefront_id} t:music31"})
'''getlookupresult'''
def getlookupresult(self, media_id: str, entity: str = "album", request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.get(ITUNES_LOOKUP_API_URL, params={"id": media_id, "entity": entity, "country": self.storefront, "lang": self.language}, **request_overrides)).raise_for_status()
lookup_result = resp2json(resp)
if ("results" not in lookup_result): raise Exception("Error getting lookup result: ", resp.text)
return lookup_result
'''getitunespage'''
def getitunespage(self, media_type: str, media_id: str, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := self.client.get(f"{ITUNES_PAGE_API_URL}/{media_type}/{media_id}", params={"country": self.storefront, "lang": self.language}, **request_overrides)).raise_for_status()
itunes_page = resp2json(resp)
if ("storePlatformData" not in itunes_page): raise Exception("Error getting iTunes page: ", resp.text)
return itunes_page
'''AppleMusicClientDownloadSongUtils'''
class AppleMusicClientDownloadSongUtils:
cdm = Cdm.from_device(Device.loads(HARDCODED_WVD))
cdm.MAX_NUM_OF_SESSIONS = float("inf")
'''getrandomuuid4'''
@staticmethod
def getrandomuuid4() -> str: return uuid.uuid4().hex
'''parsedate'''
@staticmethod
def parsedate(date: str) -> datetime.datetime: return datetime.datetime.fromisoformat(date.split("Z")[0])
'''fixkeyid'''
@staticmethod
def fixkeyid(input_path: str):
count = 0
with open(input_path, "rb+") as f:
while (data := f.read(4096)):
pos, i = f.tell(), 0
while (tenc := max(0, data.find(b"tenc", i))):
kid = tenc + 12; f.seek(max(0, pos - 4096) + kid, 0); f.write(bytes.fromhex(f"{count:032}")); count += 1; i = kid + 1
f.seek(pos, 0)
'''getmediaidoflibrarymedia'''
@staticmethod
def getmediaidoflibrarymedia(library_media_metadata: dict) -> str:
play_params = safeextractfromdict(library_media_metadata, ['attributes', 'playParams'], {})
return play_params.get("catalogId", library_media_metadata["id"])
'''getlyrics'''
@staticmethod
def getlyrics(song_metadata: dict, synced_lyrics_format: SyncedLyricsFormat = SyncedLyricsFormat.LRC, apple_music_api: AppleMusicClientAPIUtils = None, request_overrides: dict = None) -> Lyrics | None:
# no lyrics
if not safeextractfromdict(song_metadata, ['attributes', 'hasLyrics'], None): return None
# init
parse_ttml_timestamp_func = lambda ts: datetime.datetime.fromtimestamp((lambda parts: (int(parts[-2]) * 60 + int(parts[-1])) if (len(parts) == 2 and ":" in ts) else (int(parts[-1]) / 1000) if (len(parts) == 1) else ((int(parts[-3]) * 60) if (len(parts) > 2) else 0) + float(f"{parts[-2]}.{parts[-1]}"))(re.findall(r"\d+", ts)), tz=datetime.timezone.utc)
get_lyrics_line_srt_func = lambda index, element: (f"{index}\n" f"{parse_ttml_timestamp_func(element.attrib.get('begin')).strftime('%H:%M:%S,%f')[:-3]} --> " f"{parse_ttml_timestamp_func(element.attrib.get('end')).strftime('%H:%M:%S,%f')[:-3]}\n" f"{element.text}\n")
get_lyrics_line_lrc_func = lambda element: ((lambda ts, text: (lambda ms_new: f"[{((ts + (datetime.timedelta(milliseconds=((int(ms_new[:2]) + 1) * 10))) - datetime.timedelta(microseconds=ts.microsecond)) if int(ms_new[-1]) >= 5 else ts).strftime('%M:%S.%f')[:-4]}]{text}")(ts.strftime("%f")[:-3]))(parse_ttml_timestamp_func(element.attrib.get("begin")), element.text))
# re-fetch lyrics if need
if ("relationships" not in song_metadata or "lyrics" not in song_metadata["relationships"]): song_metadata = (apple_music_api.getsong(AppleMusicClientDownloadSongUtils.getmediaidoflibrarymedia(song_metadata), request_overrides=request_overrides))["data"][0]
lyrics_ttml = safeextractfromdict(song_metadata, ['relationships', 'lyrics', 'data', 0, 'attributes', 'ttml'], None)
if not lyrics_ttml: return None
# refactor lyrics
lyrics_ttml_et, unsynced_lyrics, synced_lyrics, index = ElementTree.fromstring(lyrics_ttml), [], [], 1
for div in lyrics_ttml_et.iter("{http://www.w3.org/ns/ttml}div"):
stanza = []; unsynced_lyrics.append(stanza)
for p in div.iter("{http://www.w3.org/ns/ttml}p"):
if p.text is not None: stanza.append(p.text)
if p.attrib.get("begin"):
if synced_lyrics_format == SyncedLyricsFormat.LRC: synced_lyrics.append(get_lyrics_line_lrc_func(p))
if synced_lyrics_format == SyncedLyricsFormat.SRT: synced_lyrics.append(get_lyrics_line_srt_func(index, p))
if synced_lyrics_format == SyncedLyricsFormat.TTML:
if not synced_lyrics: synced_lyrics.append(minidom.parseString(lyrics_ttml).toprettyxml())
continue
index += 1
# return
return Lyrics(synced="\n".join(synced_lyrics + ["\n"]) if synced_lyrics else None, unsynced=("\n\n".join(["\n".join(lyric_group) for lyric_group in unsynced_lyrics]) if unsynced_lyrics else None))
'''getmediadate'''
@staticmethod
def getmediadate(media_id: str, itunes_api: AppleMusicClientItunesApiUtils, request_overrides: dict = None) -> datetime.datetime | None:
lookup_result = itunes_api.getlookupresult(media_id, request_overrides=request_overrides)
if not lookup_result["results"]: return None
release_date = safeextractfromdict(lookup_result, ['results', 0, 'releaseDate'], None)
if not release_date: return None
parsed_date = AppleMusicClientDownloadSongUtils.parsedate(release_date)
return parsed_date
'''gettags'''
@staticmethod
def gettags(webplayback: dict, lyrics: str | None = None, use_album_date: bool = False, itunes_api: AppleMusicClientItunesApiUtils = None, request_overrides: dict = None) -> MediaTags:
webplayback_metadata = safeextractfromdict(webplayback, ['songList', 0, 'assets', 0, 'metadata'], {})
tags = MediaTags(
album=webplayback_metadata["playlistName"], album_artist=webplayback_metadata["playlistArtistName"], album_id=int(webplayback_metadata["playlistId"]), album_sort=webplayback_metadata["sort-album"], disc=webplayback_metadata["discNumber"], track_total=webplayback_metadata["trackCount"],
artist=webplayback_metadata["artistName"], artist_id=int(webplayback_metadata["artistId"]), artist_sort=webplayback_metadata["sort-artist"], comment=webplayback_metadata.get("comments"), rating=MediaRating(webplayback_metadata["explicit"]), lyrics=lyrics if lyrics else None,
compilation=webplayback_metadata["compilation"], composer=webplayback_metadata.get("composerName"), composer_id=(int(webplayback_metadata.get("composerId")) if webplayback_metadata.get("composerId") else None), genre=webplayback_metadata.get("genre"), media_type=MediaType.SONG,
composer_sort=webplayback_metadata.get("sort-composer"), copyright=webplayback_metadata.get("copyright"), disc_total=webplayback_metadata["discCount"], gapless=webplayback_metadata["gapless"], genre_id=int(webplayback_metadata["genreId"]), xid=webplayback_metadata.get("xid"),
date=(AppleMusicClientDownloadSongUtils.getmediadate(webplayback_metadata["playlistId"], itunes_api, request_overrides) if use_album_date else (AppleMusicClientDownloadSongUtils.parsedate(webplayback_metadata["releaseDate"]) if webplayback_metadata.get("releaseDate") else None)),
track=webplayback_metadata["trackNumber"], storefront=webplayback_metadata["s"], title=webplayback_metadata["itemName"], title_id=int(webplayback_metadata["itemId"]), title_sort=webplayback_metadata["sort-name"],
)
return tags
'''getextratags'''
@staticmethod
def getextratags(song_metadata: dict, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
previews = safeextractfromdict(song_metadata, ['attributes', 'previews'], []) or []
if not previews: return {}
preview_url = previews[0]["url"]
preview_response = requests.get(preview_url, **request_overrides)
preview_bytes = preview_response.content
preview_tags = dict(MP4(io.BytesIO(preview_bytes)).tags)
return preview_tags
'''getplaylisttags'''
@staticmethod
def getplaylisttags(playlist_metadata: dict, media_metadata: dict) -> PlaylistTags:
playlist_track = (safeextractfromdict(playlist_metadata, ['relationships', 'tracks', 'data'], '').index(media_metadata) + 1)
return PlaylistTags(
playlist_artist=safeextractfromdict(playlist_metadata, ['attributes', 'curatorName'], 'Unknown'), playlist_id=playlist_metadata["attributes"]["playParams"]["id"],
playlist_title=playlist_metadata["attributes"]["name"], playlist_track=playlist_track,
)
'''getstreaminfolegacy'''
@staticmethod
def getstreaminfolegacy(webplayback: dict, codec: SongCodec, request_overrides: dict = None) -> StreamInfoAv:
request_overrides = request_overrides or {}
flavor = "32:ctrp64" if codec == SongCodec.AAC_HE_LEGACY else "28:ctrp256"
stream_info = StreamInfo()
stream_info.stream_url = next(i for i in webplayback["songList"][0]["assets"] if i["flavor"] == flavor)["URL"]
m3u8_obj = m3u8.loads(requests.get(stream_info.stream_url, **request_overrides).text)
stream_info.widevine_pssh = m3u8_obj.keys[0].uri
stream_info_av = StreamInfoAv(media_id=webplayback["songList"][0]["songId"], audio_track=stream_info, file_format=MediaFileFormat.M4A)
return stream_info_av
'''getdecryptionkeylegacy'''
@staticmethod
def getdecryptionkeylegacy(stream_info: StreamInfoAv, cdm: Cdm, apple_music_api: AppleMusicClientAPIUtils = None, request_overrides: dict = None) -> DecryptionKeyAv:
stream_info_audio, request_overrides = stream_info.audio_track, request_overrides or {}
try:
cdm_session = cdm.open(); widevine_pssh_data = WidevinePsshData()
widevine_pssh_data.algorithm = 1
widevine_pssh_data.key_ids.append(base64.b64decode(stream_info_audio.widevine_pssh.split(",")[1]))
pssh_obj = PSSH(widevine_pssh_data.SerializeToString())
challenge = base64.b64encode(cdm.get_license_challenge(cdm_session, pssh_obj)).decode()
license_resp = apple_music_api.getlicenseexchange(stream_info.media_id, stream_info.audio_track.widevine_pssh, challenge, request_overrides=request_overrides)
cdm.parse_license(cdm_session, license_resp["license"])
decryption_key = next(i for i in cdm.get_keys(cdm_session) if i.type == "CONTENT")
finally:
cdm.close(cdm_session)
decryption_key = DecryptionKeyAv(audio_track=DecryptionKey(kid=decryption_key.kid.hex, key=decryption_key.key.hex()))
return decryption_key
'''getdecryptionkey'''
@staticmethod
def getdecryptionkey(stream_info: StreamInfoAv, cdm: Cdm, apple_music_api: AppleMusicClientAPIUtils, request_overrides: dict = None) -> DecryptionKeyAv:
track_uri, track_id = stream_info.audio_track.widevine_pssh, stream_info.media_id
try:
cdm_session = cdm.open(); pssh_obj = PSSH(track_uri.split(",")[-1])
challenge = base64.b64encode(cdm.get_license_challenge(cdm_session, pssh_obj)).decode()
license = apple_music_api.getlicenseexchange(track_id, track_uri, challenge, request_overrides=request_overrides)
cdm.parse_license(cdm_session, license["license"])
decryption_key_info = next(i for i in cdm.get_keys(cdm_session) if i.type == "CONTENT")
finally:
cdm.close(cdm_session)
decryption_key = DecryptionKey(key=decryption_key_info.key.hex(), kid=decryption_key_info.kid.hex)
return DecryptionKeyAv(audio_track=decryption_key)
'''getplaylistfromcodec'''
@staticmethod
def getplaylistfromcodec(m3u8_data: dict, codec: SongCodec) -> dict | None:
matching_playlists = [playlist for playlist in m3u8_data["playlists"] if re.fullmatch(SONG_CODEC_REGEX_MAP[codec.value], playlist["stream_info"]["audio"])]
if not matching_playlists: return None
return max(matching_playlists, key=lambda x: x["stream_info"]["average_bandwidth"])
'''getm3u8metadata'''
@staticmethod
def getm3u8metadata(m3u8_data: dict, data_id: str):
for session_data in m3u8_data.get("session_data", []):
if session_data["data_id"] == data_id:
return json.loads(base64.b64decode(session_data["value"]).decode("utf-8"))
return None
'''getaudiosessionkeymetadata'''
@staticmethod
def getaudiosessionkeymetadata(m3u8_data: dict):
return AppleMusicClientDownloadSongUtils.getm3u8metadata(m3u8_data, "com.apple.hls.AudioSessionKeyInfo")
'''getassetmetadata'''
@staticmethod
def getassetmetadata(m3u8_data: dict):
return AppleMusicClientDownloadSongUtils.getm3u8metadata(m3u8_data, "com.apple.hls.audioAssetMetadata")
'''getdrmurifromsessionkey'''
@staticmethod
def getdrmurifromsessionkey(drm_infos: dict, drm_ids: list, drm_key: str) -> str | None:
for drm_id in drm_ids:
if drm_id != "1" and drm_key in drm_infos.get(drm_id, {}):
return drm_infos[drm_id][drm_key]["URI"]
return None
'''getdrmurifromm3u8keys'''
@staticmethod
def getdrmurifromm3u8keys(m3u8_obj: m3u8.M3U8, drm_key: str) -> str | None:
default_uri = DRM_DEFAULT_KEY_MAPPING[drm_key]
for key in m3u8_obj.keys:
if key.keyformat == drm_key and key.uri != default_uri: return key.uri
return None
'''getstreaminfo'''
@staticmethod
def getstreaminfo(song_metadata: dict, codec: SongCodec, request_overrides: dict = None) -> StreamInfoAv | None:
request_overrides = request_overrides or {}
m3u8_master_url: str = safeextractfromdict(song_metadata, ['attributes', 'extendedAssetUrls', 'enhancedHls'], None)
if not m3u8_master_url: return None
m3u8_master_obj = m3u8.loads(requests.get(m3u8_master_url, **request_overrides).text)
m3u8_master_data = m3u8_master_obj.data
playlist = AppleMusicClientDownloadSongUtils.getplaylistfromcodec(m3u8_master_data, codec)
if playlist is None: return None
stream_info = StreamInfo()
stream_info.stream_url = (f"{m3u8_master_url.rpartition('/')[0]}/{playlist['uri']}")
stream_info.codec = playlist["stream_info"]["codecs"]
is_mp4 = any(stream_info.codec.startswith(codec) for codec in MP4_FORMAT_CODECS)
session_key_metadata = AppleMusicClientDownloadSongUtils.getaudiosessionkeymetadata(m3u8_master_data)
if session_key_metadata:
asset_metadata = AppleMusicClientDownloadSongUtils.getassetmetadata(m3u8_master_data)
drm_ids = asset_metadata[playlist["stream_info"]["stable_variant_id"]]["AUDIO-SESSION-KEY-IDS"]
stream_info.widevine_pssh = AppleMusicClientDownloadSongUtils.getdrmurifromsessionkey(session_key_metadata, drm_ids, "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed")
stream_info.playready_pssh = AppleMusicClientDownloadSongUtils.getdrmurifromsessionkey(session_key_metadata, drm_ids, "com.microsoft.playready")
stream_info.fairplay_key = AppleMusicClientDownloadSongUtils.getdrmurifromsessionkey(session_key_metadata, drm_ids, "com.apple.streamingkeydelivery")
else:
m3u8_obj = m3u8.loads(requests.get(stream_info.stream_url, **request_overrides).text)
stream_info.widevine_pssh = AppleMusicClientDownloadSongUtils.getdrmurifromm3u8keys(m3u8_obj, "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed")
stream_info.playready_pssh = AppleMusicClientDownloadSongUtils.getdrmurifromm3u8keys(m3u8_obj, "com.microsoft.playready")
stream_info.fairplay_key = AppleMusicClientDownloadSongUtils.getdrmurifromm3u8keys(m3u8_obj, "com.apple.streamingkeydelivery")
stream_info_av = StreamInfoAv(audio_track=stream_info, file_format=MediaFileFormat.MP4 if is_mp4 else MediaFileFormat.M4A)
return stream_info_av
'''getrawcoverurl'''
@staticmethod
def getrawcoverurl(cover_url_template: str) -> str:
return re.sub(r"image/thumb/", "", re.sub(r"is1-ssl", "a1", cover_url_template))
'''getcoverurltemplate'''
@staticmethod
def getcoverurltemplate(metadata: dict, cover_format: CoverFormat) -> str:
if cover_format == CoverFormat.RAW:
cover_url_template = AppleMusicClientDownloadSongUtils.getrawcoverurl(metadata["attributes"]["artwork"]["url"])
cover_url_template = metadata["attributes"]["artwork"]["url"]
return cover_url_template
'''getcoverurl'''
@staticmethod
def getcoverurl(cover_url_template: str, cover_size: int, cover_format: CoverFormat) -> str:
cover_url = re.sub(r"\{w\}x\{h\}([a-z]{2})\.jpg", (f"{cover_size}x{cover_size}bb.{cover_format.value}" if cover_format != CoverFormat.RAW else ""), cover_url_template)
return cover_url
'''getdownloaditem'''
@staticmethod
def getdownloaditem(song_metadata: dict, playlist_metadata: dict, synced_lyrics_format: SyncedLyricsFormat = SyncedLyricsFormat.LRC, codec: SongCodec = SongCodec.AAC_LEGACY, apple_music_api: AppleMusicClientAPIUtils = None, itunes_api: AppleMusicClientItunesApiUtils = None, use_album_date: bool = False, fetch_extra_tags: bool = False, use_wrapper: bool = False, cover_format: CoverFormat = CoverFormat.JPG, cover_size: int = 1200, request_overrides: dict = None):
# init
request_overrides = request_overrides or {}
download_item = DownloadItem()
download_item.media_metadata, download_item.playlist_metadata = song_metadata, playlist_metadata
# lyrics
song_id = AppleMusicClientDownloadSongUtils.getmediaidoflibrarymedia(song_metadata)
download_item.lyrics = AppleMusicClientDownloadSongUtils.getlyrics(song_metadata, synced_lyrics_format=synced_lyrics_format, apple_music_api=apple_music_api, request_overrides=request_overrides)
# get media tags
webplayback = apple_music_api.getwebplayback(song_id, request_overrides=request_overrides)
download_item.media_tags = AppleMusicClientDownloadSongUtils.gettags(webplayback, download_item.lyrics.unsynced if download_item.lyrics else None, use_album_date, itunes_api, request_overrides)
if fetch_extra_tags: download_item.extra_tags = AppleMusicClientDownloadSongUtils.getextratags(song_metadata, request_overrides)
if playlist_metadata: download_item.playlist_tags = AppleMusicClientDownloadSongUtils.getplaylisttags(playlist_metadata, song_metadata)
# None for all paths as default value, auto set after searching
download_item.final_path = None; download_item.synced_lyrics_path = None; download_item.staged_path = None; download_item.playlist_file_path = None
# stream info and decryption key
if codec.islegacy():
download_item.stream_info = AppleMusicClientDownloadSongUtils.getstreaminfolegacy(webplayback, codec, request_overrides)
download_item.decryption_key = AppleMusicClientDownloadSongUtils.getdecryptionkeylegacy(download_item.stream_info, AppleMusicClientDownloadSongUtils.cdm, apple_music_api=apple_music_api, request_overrides=request_overrides)
else:
download_item.stream_info = AppleMusicClientDownloadSongUtils.getstreaminfo(song_metadata, codec, request_overrides=request_overrides)
if (not use_wrapper and download_item.stream_info and download_item.stream_info.audio_track.widevine_pssh):
download_item.decryption_key = AppleMusicClientDownloadSongUtils.getdecryptionkey(download_item.stream_info, AppleMusicClientDownloadSongUtils.cdm, apple_music_api=apple_music_api, request_overrides=request_overrides)
else:
download_item.decryption_key = None
# cover url
download_item.cover_url_template = AppleMusicClientDownloadSongUtils.getcoverurltemplate(song_metadata, cover_format)
download_item.cover_url = AppleMusicClientDownloadSongUtils.getcoverurl(download_item.cover_url_template, cover_size, cover_format)
# uuid for tmp results saving
download_item.random_uuid = AppleMusicClientDownloadSongUtils.getrandomuuid4()
# return
return download_item
'''remuxmp4box'''
@staticmethod
def remuxmp4box(input_path: str, output_path: str, silent: bool = False, artist: str = ''):
cmd = ["MP4Box", "-quiet", "-add", input_path, "-itags", f"artist={artist}", "-keep-utc", "-new", output_path]
capture_output = True if silent else False
ret = subprocess.run(cmd, check=True, capture_output=capture_output, text=True, encoding='utf-8', errors='ignore')
return (ret.returncode == 0)
'''remuxffmpeg'''
@staticmethod
def remuxffmpeg(input_path: str, output_path: str, decryption_key: str = None, silent: bool = False):
key = ["-decryption_key", decryption_key] if decryption_key else []
cmd = ['ffmpeg', "-loglevel", "error", "-y", *key, "-i", input_path, "-c", "copy", "-movflags", "+faststart", output_path]
capture_output = True if silent else False
ret = subprocess.run(cmd, check=True, capture_output=capture_output, text=True, encoding='utf-8', errors='ignore')
return (ret.returncode == 0)
'''decryptmp4decrypt'''
@staticmethod
def decryptmp4decrypt(input_path: str, output_path: str, decryption_key: str, legacy: bool, silent: bool = False):
if legacy: keys = ["--key", f"1:{decryption_key}"]
else: AppleMusicClientDownloadSongUtils.fixkeyid(input_path); keys = ["--key", "0" * 31 + "1" + f":{decryption_key}", "--key", "0" * 32 + f":{DEFAULT_SONG_DECRYPTION_KEY}"]
cmd = ["mp4decrypt", *keys, input_path, output_path]
capture_output = True if silent else False
ret = subprocess.run(cmd, check=True, capture_output=capture_output, text=True, encoding='utf-8', errors='ignore')
return (ret.returncode == 0)
'''decryptamdecrypt'''
@staticmethod
def decryptamdecrypt(input_path: str, output_path: str, media_id: str, fairplay_key: str, wrapper_decrypt_ip: str = "127.0.0.1:10020", silent: bool = False):
cmd = ['amdecrypt', wrapper_decrypt_ip, shutil.which('mp4decrypt'), media_id, fairplay_key, input_path, output_path]
capture_output = True if silent else False
ret = subprocess.run(cmd, check=True, capture_output=capture_output, text=True, encoding='utf-8', errors='ignore')
return (ret.returncode == 0)
'''stage'''
@staticmethod
def stage(encrypted_path: str, decrypted_path: str, staged_path: str, decryption_key: DecryptionKeyAv, codec: SongCodec, media_id: str, fairplay_key: str, remux_mode: RemuxMode = RemuxMode.MP4BOX, silent: bool = False, wrapper_decrypt_ip: str = "127.0.0.1:10020", artist: str = "", use_wrapper: bool = False):
if codec.islegacy() and remux_mode == RemuxMode.FFMPEG:
AppleMusicClientDownloadSongUtils.remuxffmpeg(encrypted_path, staged_path, decryption_key.audio_track.key, silent=silent)
elif codec.islegacy() or not use_wrapper:
AppleMusicClientDownloadSongUtils.decryptmp4decrypt(encrypted_path, decrypted_path, decryption_key.audio_track.key, codec.islegacy(), silent)
if remux_mode == RemuxMode.FFMPEG: AppleMusicClientDownloadSongUtils.remuxffmpeg(decrypted_path, staged_path, silent=silent)
else: AppleMusicClientDownloadSongUtils.remuxmp4box(decrypted_path, staged_path, silent=silent, artist=artist)
else:
AppleMusicClientDownloadSongUtils.decryptamdecrypt(encrypted_path, staged_path, media_id, fairplay_key, wrapper_decrypt_ip=wrapper_decrypt_ip, silent=silent)
'''downloadstreamwithnm3u8dlre'''
@staticmethod
def downloadstreamwithnm3u8dlre(stream_url: str, download_path: str, silent: bool = False, random_uuid: str = ''):
download_path_obj = Path(download_path)
download_path_obj.parent.mkdir(parents=True, exist_ok=True)
log_file_path = os.path.join(user_log_dir(appname='musicdl', appauthor='zcjin'), f"musicdl_{random_uuid}.log")
cmd = [
"N_m3u8DL-RE", stream_url, "--binary-merge", "--ffmpeg-binary-path", shutil.which('ffmpeg'), "--save-name", download_path_obj.stem,
"--save-dir", download_path_obj.parent, "--tmp-dir", download_path_obj.parent, "--log-file-path", log_file_path,
]
capture_output = True if silent else False
ret = subprocess.run(cmd, check=True, capture_output=capture_output, text=True, encoding='utf-8', errors='ignore')
return (ret.returncode == 0)
'''download'''
@staticmethod
def download(download_item: DownloadItem, work_dir: str = './', silent: bool = False, codec: SongCodec = SongCodec.AAC_LEGACY, wrapper_decrypt_ip: str = "127.0.0.1:10020", remux_mode: RemuxMode = RemuxMode.MP4BOX, artist: str = "", use_wrapper: bool = False):
ext = download_item.stream_info.file_format.value
encrypted_path = os.path.join(work_dir, f"{download_item.random_uuid}_encrypted.m4a")
is_success = AppleMusicClientDownloadSongUtils.downloadstreamwithnm3u8dlre(download_item.stream_info.audio_track.stream_url, encrypted_path, silent=silent, random_uuid=download_item.random_uuid)
decrypted_path = os.path.join(work_dir, f"{download_item.random_uuid}_decrypted.m4a")
download_item.staged_path = os.path.join(work_dir, f"{download_item.random_uuid}_staged.{ext}")
is_success = AppleMusicClientDownloadSongUtils.stage(
encrypted_path=encrypted_path, decrypted_path=decrypted_path, staged_path=download_item.staged_path, decryption_key=download_item.decryption_key,
codec=codec, media_id=download_item.media_metadata["id"], fairplay_key=download_item.stream_info.audio_track.fairplay_key, remux_mode=remux_mode,
silent=silent, wrapper_decrypt_ip=wrapper_decrypt_ip, artist=artist, use_wrapper=use_wrapper,
)
return is_success
catalog-sync/musicdl/modules/utils/data.py
+141
View File
@@ -0,0 +1,141 @@
'''
Function:
Implementation of SongInfo
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import os
from typing import Any, Dict, Optional
from dataclasses import dataclass, field, fields
from .misc import sanitize_filepath, safeextractfromdict, AudioLinkTester
def remove_prefix(value: str, prefix: str) -> str:
if prefix and value.startswith(prefix):
return value[len(prefix):]
return value
def remove_suffix(value: str, suffix: str) -> str:
if suffix and value.endswith(suffix):
return value[: -len(suffix)]
return value
'''SongInfo'''
@dataclass
class SongInfo:
# raw data replied by requested APIs
raw_data: Dict[str, Any] = field(default_factory=dict)
# from which music client
source: Optional[str] = None
root_source: Optional[str] = None
# song information
song_name: Optional[str] = None
singers: Optional[str] = None
album: Optional[str] = None
ext: Optional[str] = None
file_size_bytes: Optional[int] = None
file_size: Optional[str] = None
duration_s: Optional[int] = None
duration: Optional[str] = None
bitrate: Optional[int] = None
codec: Optional[str] = None
samplerate: Optional[int] = None
channels: Optional[int] = None
# lyric
lyric: Optional[str] = None
# cover
cover_url: Optional[str] = None
# episodes, each item in episodes is SongInfo object, used by FM site like XimalayaMusicClient
episodes: Optional[list[SongInfo]] = None
# download url related variables
download_url: Optional[Any] = None
download_url_status: Optional[Any] = None
default_download_headers: Dict[str, Any] = field(default_factory=dict)
downloaded_contents: Optional[Any] = None
chunk_size: Optional[int] = 1024 * 1024
protocol: Optional[str] = 'HTTP' # should be in {'HTTP', 'HLS'}
@property
def with_valid_download_url(self) -> bool:
if self.episodes: return all([eps.with_valid_download_url for eps in self.episodes])
if isinstance(self.download_url, str): is_valid_download_url_format = self.download_url and self.download_url.startswith('http')
else: is_valid_download_url_format = bool(self.download_url)
with_downloaded_contents = bool(self.downloaded_contents)
is_downloadable = isinstance(self.download_url_status, dict) and self.download_url_status.get('ok')
if not is_downloadable and (safeextractfromdict(self.download_url_status, ['probe_status', 'ext'], None) in AudioLinkTester.VALID_AUDIO_EXTS): is_downloadable = True
return bool((is_valid_download_url_format or with_downloaded_contents) and is_downloadable)
# save info
work_dir: Optional[str] = './'
_save_path: Optional[str] = None
@property
def save_path(self) -> str:
if self._save_path is not None: return self._save_path
ext = remove_prefix(str(self.ext or ""), ".")
sp, same_name_file_idx = os.path.join(self.work_dir, f"{self.song_name} - {self.identifier}.{ext}"), 1
while os.path.exists(sp):
sp = os.path.join(self.work_dir, f"{self.song_name} - {self.identifier} ({same_name_file_idx}).{ext}")
same_name_file_idx += 1
sp = sanitize_filepath(sp)
self._save_path = sp
return sp
# identifier
identifier: Optional[str] = None
'''fieldnames'''
@classmethod
def fieldnames(cls) -> set[str]:
return {f.name for f in fields(cls)}
'''fromdict'''
@classmethod
def fromdict(cls, data: Dict[str, Any]) -> "SongInfo":
field_names = cls.fieldnames()
filtered = {k: v for k, v in data.items() if k in field_names}
if "episodes" in filtered and filtered["episodes"] and isinstance(filtered["episodes"], list):
episodes = [cls.fromdict(e) if isinstance(e, dict) else e for e in filtered["episodes"]]
filtered["episodes"] = episodes
return cls(**filtered)
'''todict'''
def todict(self) -> Dict[str, Any]:
converted_dict = {f.name: getattr(self, f.name) for f in fields(self)}
if self.episodes and isinstance(self.episodes, list): converted_dict['episodes'] = [e.todict() for e in self.episodes]
return converted_dict
'''update'''
def update(self, data: Dict[str, Any] = None, **kwargs: Any) -> "SongInfo":
if data is None: data = {}
merged: Dict[str, Any] = {**data, **kwargs}
field_names = self.fieldnames()
for k, v in merged.items():
if k in field_names: setattr(self, k, v)
return self
'''getitem'''
def __getitem__(self, key: str) -> Any:
field_names = self.fieldnames()
if key not in field_names: raise KeyError(key)
return getattr(self, key)
'''setitem'''
def __setitem__(self, key: str, value: Any) -> None:
field_names = self.fieldnames()
if key not in field_names: raise KeyError(key)
setattr(self, key, value)
'''contains'''
def __contains__(self, key: object) -> bool:
return isinstance(key, str) and key in self.fieldnames()
'''get'''
def get(self, key: str, default: Any = None) -> Any:
if key in self.fieldnames(): return getattr(self, key)
return default
'''largerthan'''
def largerthan(self, song_info: SongInfo):
# file_size_a
try: file_size_a = float(remove_suffix(str(self.file_size), 'MB').strip())
except Exception: file_size_a = 0.0
if not isinstance(file_size_a, (int, float)): file_size_a = 0.0
# file_size_b
try: file_size_b = float(remove_suffix(str(song_info.file_size), 'MB').strip())
except Exception: file_size_b = 0.0
if not isinstance(file_size_b, (int, float)): file_size_b = 0.0
# compare
return bool(file_size_a > file_size_b)
catalog-sync/musicdl/modules/utils/deezerutils.py
+64
View File
@@ -0,0 +1,64 @@
'''
Function:
Implementation of DeezerMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import re
import base64
import hashlib
import binascii
import functools
from Cryptodome.Cipher import AES, Blowfish
'''DeezerMusicClientUtils'''
class DeezerMusicClientUtils():
BLOWFISH_SECRET = "g4el58wc0zvf9na1"
MUSIC_QUALITIES = ('FLAC', 'MP3_320', 'MP3_128')
IS_ENCRYPTED_RPATTERN = re.compile("/m(?:obile|edia)/")
SHARED_TOKENS = ['ZjI4N2JkNzRjM2Q1NGY5YmJmOTc5OTdjNzhkZWJkMzdiMTU4NjRjZDdhM2MwZjk0MjUxNWNjOWIwNGE1MWM1N2RhYmZiOTQ4YWYyNjM0MDFhOTRkZTUxOGI3MjRlZDdmNDBmMjcyMmNlZGMwMTgxZTEwYmZmNDk5MmVjNzc4NzU3MmU1MDUzZjk0Nzc1NjFiZjhkMjcwNDc0NzRiNzMxMTcxNjUyZWQxNzg0YzlmNTdhMTUxZDMxOTk2NmVjY2Ex']
token_decrypt_func = lambda t: base64.b64decode(str(t).encode('utf-8')).decode('utf-8')
'''decryptchunk'''
@staticmethod
def decryptchunk(key, data):
return Blowfish.new(key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(data)
'''generateblowfishkey'''
@staticmethod
def generateblowfishkey(track_id: str) -> bytes:
md5_hash = hashlib.md5(str(track_id).encode()).hexdigest()
return "".join(chr(functools.reduce(lambda x, y: x ^ y, map(ord, t))) for t in zip(md5_hash[:16], md5_hash[16:], DeezerMusicClientUtils.BLOWFISH_SECRET)).encode()
'''getencryptedfileurl'''
@staticmethod
def getencryptedfileurl(meta_id: str, track_hash: str, media_version: str, format_number: int = 1):
url_bytes = b"\xa4".join((track_hash.encode(), str(format_number).encode(), str(meta_id).encode(), str(media_version).encode()))
info_bytes = bytearray(hashlib.md5(url_bytes).hexdigest().encode())
info_bytes.extend(b"\xa4"); info_bytes.extend(url_bytes); info_bytes.extend(b"\xa4")
padding_len = 16 - (len(info_bytes) % 16); info_bytes.extend(b"." * padding_len)
path = binascii.hexlify(AES.new(b"jo6aey6haid2Teih", AES.MODE_ECB).encrypt(info_bytes)).decode("utf-8")
return f"https://e-cdns-proxy-{track_hash[0]}.dzcdn.net/mobile/1/{path}"
'''getcoverurl'''
@staticmethod
def getcoverurl(pic_id: str):
if not pic_id: return None
return f"https://e-cdns-images.dzcdn.net/images/cover/{pic_id}/1200x1200.jpg"
'''covert2lrclyrics'''
@staticmethod
def covert2lrclyrics(lyrics_node: dict):
lrc_lines = []; lyrics_node.get("writers") and lrc_lines.append(f"[ar:{lyrics_node['writers']}]")
if (sync_lines := lyrics_node.get("synchronizedLines")):
for item in sync_lines: lrc_lines.append(f"{item.get('lrcTimestamp', '')}{item.get('line', '')}") if item.get("lrcTimestamp", "") else (lrc_lines.append(f"[{int(item['milliseconds']) // 60000:02d}:{(int(item['milliseconds']) % 60000) / 1000:05.2f}]{item.get('line', '')}") if "milliseconds" in item else None)
return "\n".join(lrc_lines)
else:
return lyrics_node.get("text")
'''decryptdownloadedaudiofile'''
@staticmethod
def decryptdownloadedaudiofile(src_path: str, dst_path: str, blowfish_key: str):
encrypt_chunk_size = 3 * 2048
with open(src_path, "rb") as src, open(dst_path, "wb") as dst:
while True:
if not (data := src.read(encrypt_chunk_size)): break
decrypted_chunk = DeezerMusicClientUtils.decryptchunk(blowfish_key, data[:2048]) + data[2048:] if len(data) >= 2048 else data
dst.write(decrypted_chunk)
catalog-sync/musicdl/modules/utils/hls.py
+383
View File
@@ -0,0 +1,383 @@
'''
Function:
Implementation of HLSDownloader
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import os
import re
import copy
import time
import math
import m3u8
import base64
import shutil
import hashlib
import requests
import threading
import concurrent.futures as cf
from pathlib import Path
from .misc import touchdir
from .logger import LoggerHandle
from urllib.parse import urljoin
from dataclasses import dataclass
from rich.progress import Progress
from typing import Optional, Dict, Any, Tuple, List, Union, Callable
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
'''SegmentJob'''
@dataclass(frozen=True)
class SegmentJob:
index: int
uri: str
byterange: Optional[str]
key_method: Optional[str]
key_uri: Optional[str]
key_iv: Optional[str]
keyformat: Optional[str]
media_sequence: int
map_uri: Optional[str]
map_byterange: Optional[str]
'''HLSDownloader'''
class HLSDownloader:
def __init__(self, output_dir: str = "downloads", proxies: Optional[Dict[str, str]] = None, headers: Optional[Dict[str, str]] = None, cookies: Optional[Dict[str, str]] = None, timeout: Tuple[float, float] = (10.0, 30.0), logger_handle: LoggerHandle = None,
verify_tls: bool = True, concurrency: int = 16, max_retries: int = 8, backoff_base: float = 0.6, backoff_cap: float = 10.0, chunk_size: int = 1024 * 256, strict_key_length: bool = False, disable_print: bool = False, request_overrides: dict = None):
# work dir
self.output_dir = output_dir
touchdir(self.output_dir)
# logger
self.logger_handle = logger_handle
self.disable_print = disable_print
# http requests
self.proxies = proxies or {}
self.headers = headers or {}
self.cookies = cookies or {}
self.timeout = timeout
self.verify_tls = verify_tls
self.chunk_size = int(chunk_size)
self.backoff_cap = float(backoff_cap)
self.backoff_base = float(backoff_base)
self.concurrency = max(1, int(concurrency))
self.max_retries = max(1, int(max_retries))
self.strict_key_length = bool(strict_key_length)
self.request_overrides = request_overrides or {}
# threading
self._tls = threading.local()
self._key_cache: Dict[str, bytes] = {}
self._key_cache_lock = threading.Lock()
'''download'''
def download(self, m3u8_url: str, output_path: str, quality: Union[str, int, Callable[[List[Dict[str, Any]]], int]] = "best", keep_segments: bool = False, temp_subdir: Optional[str] = None, progress: Progress = None, progress_id: int = 0) -> str:
master_or_media = self._loadm3u8(m3u8_url)
if master_or_media.is_variant:
variant_url = self._selectvariant(master_or_media, quality)
self.logger_handle.info(f"Selected variant: {variant_url}", disable_print=self.disable_print)
playlist = self._loadm3u8(variant_url)
else:
playlist = master_or_media
jobs, global_init_map = self._buildjobs(playlist)
temp_folder, global_init_path = os.path.join(self.output_dir, temp_subdir or f".hls_tmp_{self._safenamefromurl(m3u8_url)}"), None
touchdir(temp_folder)
if global_init_map:
global_init_path = os.path.join(temp_folder, "_global_init.bin")
if not self._fileok(global_init_path): self._atomicwrite(global_init_path, self._fetchbytes(global_init_map["uri"], global_init_map.get("byterange")))
seg_paths = self._downloadallsegments(jobs, temp_folder, progress=progress, progress_id=progress_id)
touchdir(os.path.dirname(os.path.abspath(output_path)) or ".")
self._mergefiles(global_init_path, seg_paths, output_path)
if not keep_segments: shutil.rmtree(temp_folder, ignore_errors=True)
return output_path
'''_getsession'''
def _getsession(self) -> requests.Session:
sess = getattr(self._tls, "session", None)
if sess is None:
sess = requests.Session()
sess.headers.update(self.headers)
if self.cookies: sess.cookies.update(self.cookies)
self._tls.session = sess
return sess
'''_request'''
def _request(self, url: str, method: str = "GET", headers: Optional[Dict[str, str]] = None, stream: bool = False, **kwargs) -> requests.Response:
kwargs.update(copy.deepcopy(self.request_overrides))
sess, last_exc = self._getsession(), None
hdrs = dict(self.headers)
if headers: hdrs.update(headers)
for attempt in range(1, self.max_retries + 1):
try:
resp = sess.request(method=method, url=url, headers=hdrs, proxies=self.proxies, timeout=self.timeout, verify=self.verify_tls, stream=stream, **kwargs)
if resp.status_code in (429, 500, 502, 503, 504): resp.close(); raise requests.HTTPError(f"HTTP {resp.status_code} for {url}")
resp.raise_for_status()
return resp
except Exception as e:
last_exc = e
t = min(self.backoff_cap, self.backoff_base * (2 ** (attempt - 1)))
t = t + (0.1 * t * (0.5 - (time.time() % 1)))
time.sleep(max(0.0, t))
raise RuntimeError(f"Request failed after retries: {url}\nLast error: {last_exc}")
'''_gettext'''
def _gettext(self, url: str) -> str:
resp = self._request(url, stream=False)
return resp.text
'''_getbytes'''
def _getbytes(self, url: str, headers: Optional[Dict[str, str]] = None) -> bytes:
resp = self._request(url, headers=headers, stream=True)
chunks = []
for c in resp.iter_content(chunk_size=self.chunk_size):
if c: chunks.append(c)
resp.close()
return b"".join(chunks)
'''_fetchbytes'''
def _fetchbytes(self, url: str, byterange: Optional[str]) -> bytes:
headers = {}
if byterange:
length, offset = self._parsebyterange(byterange)
headers["Range"] = f"bytes={offset}-{offset + length - 1}"
return self._getbytes(url, headers=headers)
'''_loadm3u8'''
def _loadm3u8(self, url: str) -> m3u8.M3U8:
text = self._gettext(url)
return m3u8.loads(text, uri=url)
'''_selectvariant'''
def _selectvariant(self, master: m3u8.M3U8, quality: Union[str, int, Callable[[List[Dict[str, Any]]], int]]) -> str:
variants, bw_func = [], lambda v: int(v.get("average_bandwidth") or v.get("bandwidth") or 0)
for i, p in enumerate(master.playlists or []):
si = getattr(p, "stream_info", None)
variants.append({
"index": i, "absolute_uri": getattr(p, "absolute_uri", None) or urljoin(master.base_uri or master.uri, p.uri), "uri": p.uri, "bandwidth": getattr(si, "bandwidth", None) if si else None,
"average_bandwidth": getattr(si, "average_bandwidth", None) if si else None, "resolution": getattr(si, "resolution", None) if si else None, "codecs": getattr(si, "codecs", None) if si else None,
"frame_rate": getattr(si, "frame_rate", None) if si else None,
})
if not variants: raise ValueError("Master playlist has no variants.")
if callable(quality): idx = int(quality(variants)); idx = max(0, min(idx, len(variants) - 1)); return variants[idx]["absolute_uri"]
if isinstance(quality, str):
q = quality.lower().strip()
if q == "best":
chosen = max(variants, key=bw_func)
elif q == "lowest":
chosen = min(variants, key=bw_func)
else:
m = re.search(r"(\d+)", q)
if m: target = int(m.group(1)); chosen = min(variants, key=lambda v: abs(bw_func(v) - target))
else: chosen = max(variants, key=bw_func)
else:
target = int(quality)
chosen = min(variants, key=lambda v: abs(bw_func(v) - target))
return chosen["absolute_uri"]
'''_buildjobs'''
def _buildjobs(self, playlist: m3u8.M3U8) -> Tuple[List[SegmentJob], Optional[Dict[str, Any]]]:
media_seq = int(getattr(playlist, "media_sequence", 0) or 0)
global_init, seg_map = None, getattr(playlist, "segment_map", None)
if seg_map:
try: sm0 = seg_map[0]; global_init = {"uri": getattr(sm0, "absolute_uri", None) or urljoin(playlist.base_uri, sm0.uri), "byterange": getattr(sm0, "byterange", None)}
except Exception: global_init = None
jobs: List[SegmentJob] = []
session_keys = getattr(playlist, "session_keys", None) or []
fallback_session_key, last_key_obj = session_keys[-1] if session_keys else None, None
for i, seg in enumerate(playlist.segments or []):
seg_uri, key_obj = getattr(seg, "absolute_uri", None) or urljoin(playlist.base_uri, seg.uri), getattr(seg, "key", None) or last_key_obj or fallback_session_key
if getattr(seg, "key", None) is not None: last_key_obj = getattr(seg, "key", None)
key_method, key_uri, key_iv, keyformat = (getattr(key_obj, k, None) for k in ("method", "uri", "iv", "keyformat")) if key_obj else (None, None, None, None)
key_uri_abs = (key_uri if key_uri and (key_uri.startswith("data:") or key_uri.startswith("skd://")) else (urljoin(playlist.base_uri, key_uri) if key_uri else None))
init_section = getattr(seg, "init_section", None)
map_uri, map_byterange = ((getattr(init_section, "absolute_uri", None) or (urljoin(playlist.base_uri, getattr(init_section, "uri", "")) if getattr(init_section, "uri", None) else None)), getattr(init_section, "byterange", None)) if init_section is not None else (None, None)
jobs.append(SegmentJob(index=i, uri=seg_uri, byterange=getattr(seg, "byterange", None), key_method=key_method, key_uri=key_uri_abs, key_iv=key_iv, keyformat=keyformat, media_sequence=media_seq, map_uri=map_uri, map_byterange=map_byterange))
return jobs, global_init
'''_downloadallsegments'''
def _downloadallsegments(self, jobs: List[SegmentJob], temp_folder: str, progress: Progress, progress_id: int) -> List[str]:
progress.update(progress_id, description=f"HLSDownloader._downloadallsegments >>> completed (0/{len(jobs)})", total=len(jobs), kind='hls')
byterange_cursor: Dict[str, int] = {}; seg_paths: List[Optional[str]] = [None] * len(jobs)
init_cache: Dict[str, str] = {}; init_inflight: Dict[str, threading.Event] = {}; init_cache_lock = threading.Lock()
def ensureinitsection_func(map_uri: str, map_byterange: Optional[str]) -> bytes:
key = f"{map_uri}|{map_byterange or ''}"
with init_cache_lock:
cached = init_cache.get(key)
if cached and self._fileok(cached): return Path(cached).read_bytes()
leader = (evt := init_inflight.get(key)) is None; evt = init_inflight[key] = threading.Event() if leader else evt
if not leader:
evt.wait()
with init_cache_lock: cached = init_cache.get(key)
return Path(cached).read_bytes() if cached and self._fileok(cached) else (_ for _ in ()).throw(RuntimeError(f"init_section download failed: {key}"))
try:
data = self._fetchbytes(map_uri, map_byterange)
path = os.path.join(temp_folder, f"_initsec_{abs(hash(key)) & 0xffffffff:08x}.bin")
self._atomicwrite(path, data)
with init_cache_lock: init_cache[key] = path
return data
finally:
with init_cache_lock: (evt := init_inflight.pop(key, None)) and evt.set()
def worker_func(job: SegmentJob) -> Tuple[int, str]:
seg_path = os.path.join(temp_folder, f"seg_{job.index:06d}.bin")
if self._fileok(seg_path): return job.index, seg_path
prepend = ensureinitsection_func(job.map_uri, job.map_byterange) if job.map_uri else b""
eff_byterange = self._normalizebyterange(job.uri, job.byterange, byterange_cursor) if job.byterange else job.byterange
data = self._fetchandmaybedecrypt(job, eff_byterange)
self._atomicwrite(seg_path, prepend + data)
return job.index, seg_path
exceptions: List[Exception] = []
with cf.ThreadPoolExecutor(max_workers=self.concurrency) as ex:
futures = [ex.submit(worker_func, j) for j in jobs]
for fut in cf.as_completed(futures):
try:
idx, path = fut.result()
seg_paths[idx] = path
except Exception as e:
exceptions.append(e)
finally:
progress.advance(progress_id, 1)
num_downloaded_segs = int(progress.tasks[progress_id].completed)
progress.update(progress_id, description=f"HLSDownloader._downloadallsegments >>> completed ({num_downloaded_segs}/{len(jobs)})")
if exceptions: raise exceptions[0]
return [p for p in seg_paths if p is not None]
'''_fetchandmaybedecrypt'''
def _fetchandmaybedecrypt(self, job: SegmentJob, eff_byterange: Optional[str]) -> bytes:
method_raw, keyformat = (job.key_method or "").strip(), (job.keyformat or "").strip().lower()
if not method_raw or method_raw.upper() == "NONE": return self._fetchbytes(job.uri, eff_byterange)
if keyformat and keyformat not in ("identity",): raise NotImplementedError(f"Unsupported KEYFORMAT={job.keyformat} (likely DRM).")
method = method_raw.upper().replace("_", "-")
dec_mode = self._classifyencryptionmethod(method)
if dec_mode in ("DRM", "UNSUPPORTED"): raise NotImplementedError(f"Unsupported encryption method: {method_raw}")
if not job.key_uri: raise RuntimeError(f"Encrypted segment missing key URI at seg {job.index}")
key, base_iv = self._prepareaeskey(method, self._getkeybytes(job.key_uri)), self._deriveiv(job.key_iv, job.media_sequence + job.index)
if not eff_byterange: ciphertext = self._fetchbytes(job.uri, None); return self._decryptwhole(ciphertext, dec_mode, key, base_iv)
length, offset = self._parsebyterange(eff_byterange)
block, end = 16, offset + length
aligned_start, aligned_end = (offset // block) * block, int(math.ceil(end / block) * block)
if dec_mode == "CBC":
fetch_start, drop = ((aligned_start - block, offset - aligned_start + block) if aligned_start > 0 else (aligned_start, offset - aligned_start)); fetch_len = aligned_end - fetch_start; fetch_range = f"{fetch_len}@{fetch_start}"
ciphertext = self._fetchbytes(job.uri, fetch_range)
iv = (b"\x00" * 16) if fetch_start > 0 else base_iv
plaintext = self._aescbcdecrypt(ciphertext, key, iv)
return plaintext[drop: drop+length]
else:
fetch_start, drop, fetch_len, fetch_range = aligned_start, offset - aligned_start, aligned_end - aligned_start, f"{aligned_end - aligned_start}@{aligned_start}"
ciphertext = self._fetchbytes(job.uri, fetch_range)
block_index = fetch_start // block
iv_int = int.from_bytes(base_iv, "big")
adj_iv = ((iv_int + block_index) % (1 << 128)).to_bytes(16, "big")
plaintext = self._aesctrcrypt(ciphertext, key, adj_iv)
return plaintext[drop: drop+length]
'''_decryptwhole'''
def _decryptwhole(self, ciphertext: bytes, dec_mode: str, key: bytes, iv: bytes) -> bytes:
if dec_mode == "CBC": return self._aescbcdecrypt(ciphertext, key, iv)
if dec_mode == "CTR": return self._aesctrcrypt(ciphertext, key, iv)
raise NotImplementedError(f"decrypt mode {dec_mode} not supported")
'''_classifyencryptionmethod'''
def _classifyencryptionmethod(self, method: str) -> str:
m = method.strip().upper()
if m in ("AES-128", "AES-128-CBC", "AES-CBC", "CBC"): return "CBC"
if m in ("AES-CTR", "AES-128-CTR", "AES-192-CTR", "AES-256-CTR"): return "CTR"
if m.startswith("SAMPLE-AES") or "SKD" in m: return "DRM"
return "UNSUPPORTED"
'''_getkeybytes'''
def _getkeybytes(self, key_uri: str) -> bytes:
if key_uri.startswith("data:"):
if "base64," in key_uri: b64 = key_uri.split("base64,", 1)[1]; return base64.b64decode(b64)
if "," in key_uri: raw = key_uri.split(",", 1)[1]; return raw.encode("utf-8", errors="ignore")
raise ValueError("Unsupported data: key URI")
if key_uri.startswith("skd://"): raise NotImplementedError("skd:// indicates DRM (FairPlay). Not supported.")
with self._key_cache_lock:
if key_uri in self._key_cache: return self._key_cache[key_uri]
b = self._getbytes(key_uri)
with self._key_cache_lock: self._key_cache[key_uri] = b
return b
'''_decodekeyguess'''
def _decodekeyguess(self, key_bytes: bytes) -> bytes:
b = key_bytes.strip()
if b"\x00" in b: return b
b2 = b
if b2.lower().startswith(b"0x"): b2 = b2[2:]
if re.fullmatch(rb"[0-9a-fA-F]+", b2) and len(b2) in (32, 48, 64):
try: return bytes.fromhex(b2.decode("ascii"))
except Exception: pass
if re.fullmatch(rb"[A-Za-z0-9+/=\r\n]+", b) and (len(b) % 4 == 0):
try:
dec = base64.b64decode(b, validate=False)
if len(dec) in (16, 24, 32): return dec
except Exception:
pass
return b
'''_expectedkeylen'''
def _expectedkeylen(self, method: str) -> int:
m = method.upper()
if "256" in m: return 32
if "192" in m: return 24
return 16
'''_prepareaeskey'''
def _prepareaeskey(self, method: str, key_bytes: bytes) -> bytes:
k = self._decodekeyguess(key_bytes)
want = self._expectedkeylen(method)
if len(k) == want: return k
if self.strict_key_length: raise ValueError(f"Bad key length for {method}: got {len(k)} bytes, expected {want}")
self.logger_handle.warning(f"Key length mismatch for {method}: got {len(k)}, expected {want}. Best-effort fix.", disable_print=self.disable_print)
if len(k) > want: return k[:want]
return (k + b"\x00" * want)[:want]
'''_deriveiv'''
def _deriveiv(self, iv_str: Optional[str], seq_num: int) -> bytes:
if not iv_str: return seq_num.to_bytes(16, byteorder="big", signed=False)
s = str(iv_str).strip().lower()
if s.startswith("0x"): s = s[2:]
try: iv = bytes.fromhex(s)
except Exception: iv = s.encode("utf-8", errors="ignore")
if len(iv) < 16: iv = (b"\x00" * (16 - len(iv))) + iv
if len(iv) > 16: iv = iv[-16:]
return iv
'''_aescbcdecrypt'''
def _aescbcdecrypt(self, ciphertext: bytes, key: bytes, iv: bytes) -> bytes:
if len(ciphertext) % 16 != 0: raise ValueError(f"CBC ciphertext length not multiple of 16: {len(ciphertext)} bytes")
cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
dec = cipher.decryptor()
return dec.update(ciphertext) + dec.finalize()
'''_aesctrcrypt'''
def _aesctrcrypt(self, data: bytes, key: bytes, iv: bytes) -> bytes:
cipher = Cipher(algorithms.AES(key), modes.CTR(iv))
dec = cipher.decryptor()
return dec.update(data) + dec.finalize()
'''_parsebyterange'''
def _parsebyterange(self, s: str) -> Tuple[int, int]:
s = s.strip()
if "@" in s: a, b = s.split("@", 1); return int(a), int(b)
raise ValueError(f"BYTERANGE missing offset: {s}")
'''_normalizebyterange'''
def _normalizebyterange(self, uri: str, byterange: str, cursor: Dict[str, int]) -> str:
s = byterange.strip()
if "@" in s: length, offset = s.split("@", 1); length_i, offset_i = int(length), int(offset); cursor[uri] = offset_i + length_i; return f"{length_i}@{offset_i}"
length_i = int(s)
prev = cursor.get(uri, 0)
cursor[uri] = prev + length_i
return f"{length_i}@{prev}"
'''_mergefiles'''
def _mergefiles(self, global_init_path: Optional[str], seg_paths: List[str], output_path: str) -> None:
tmp_out = output_path + ".part"
with open(tmp_out, "wb") as out:
if global_init_path and self._fileok(global_init_path):
with open(global_init_path, "rb") as fp: shutil.copyfileobj(fp, out, length=1024 * 1024)
for p in seg_paths:
with open(p, "rb") as fp: shutil.copyfileobj(fp, out, length=1024 * 1024)
os.replace(tmp_out, output_path)
'''_safenamefromurl'''
def _safenamefromurl(self, url: str, max_len: int = 20) -> str:
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:max_len]
'''_fileok'''
def _fileok(self, path: str) -> bool:
return os.path.exists(path) and os.path.getsize(path) > 0
'''_atomicwrite'''
def _atomicwrite(self, path: str, data: bytes) -> None:
touchdir(os.path.dirname(os.path.abspath(path)) or ".")
pid, tid = os.getpid(), threading.get_ident()
tmp, last = f"{path}.tmp.{pid}.{tid}.{time.time_ns()}", None
with open(tmp, "wb") as fp:
fp.write(data)
try: fp.flush(); os.fsync(fp.fileno())
except Exception: pass
for i in range(12):
try: os.replace(tmp, path); return
except PermissionError as e: last = e; time.sleep(min(0.5, 0.03 * (2 ** i)))
except OSError as e: last = e; time.sleep(min(0.5, 0.03 * (2 ** i)))
try:
if os.path.exists(tmp): os.remove(tmp)
except Exception:
pass
raise last
catalog-sync/musicdl/modules/utils/hosts.py
+53
View File
@@ -0,0 +1,53 @@
'''
Function:
Implementation of URL Domain Related Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
from functools import lru_cache
from urllib.parse import urlsplit
'''settings'''
APPLE_MUSIC_HOSTS = {"music.apple.com", "geo.music.apple.com", "embed.music.apple.com", "itunes.apple.com", "geo.itunes.apple.com", "apple.com"}
DEEZER_MUSIC_HOSTS = {"deezer.com", "www.deezer.com", "deezer.page.link",}
FIVESING_MUSIC_HOSTS = {"5sing.kugou.com",}
JOOX_MUSIC_HOSTS = {"joox.com",}
JAMENDO_MUSIC_HOSTS = {"jamendo.com",}
KUWO_MUSIC_HOSTS = {"kuwo.cn", "www.kuwo.cn", "m.kuwo.cn", "mobile.kuwo.cn",}
KUGOU_MUSIC_HOSTS = {"www.kugou.com", "m.kugou.com", "kugou.com", "h5.kugou.com",}
MIGU_MUSIC_HOSTS = {"music.migu.cn", "m.music.migu.cn", "h5.nf.migu.cn", "c.migu.cn", "migu.cn"}
NETEASE_MUSIC_HOSTS = {"music.163.com", "y.music.163.com", "m.music.163.com", "3g.music.163.com", "163cn.tv",}
QQ_MUSIC_HOSTS = {"y.qq.com", "i.y.qq.com", "m.y.qq.com", "c.y.qq.com", "c6.y.qq.com", "music.qq.com",}
QIANQIAN_MUSIC_HOSTS = {"music.91q.com", "music.taihe.com", "music.baidu.com"}
QOBUZ_MUSIC_HOSTS = {"open.qobuz.com", "play.qobuz.com", "www.qobuz.com", "qobuz.com"}
STREETVOICE_MUSIC_HOSTS = {"streetvoice.cn"}
SOUNDCLOUD_MUSIC_HOSTS = {"soundcloud.com"}
SODA_MUSIC_HOSTS = {"qishui.douyin.com", "music.douyin.com", "www.qishui.com", "www.douyin.com", "z-qishui.douyin.com", "lf-luna-release.qishui.com", "luna-web.douyin.com", "bubble.qishui.com", "qishui.com", "douyin.com"}
SPOTIFY_MUSIC_HOSTS = {"open.spotify.com", "spotify.link", "play.spotify.com", "spotify.com"}
TIDAL_MUSIC_HOSTS = {"tidal.com", "listen.tidal.com", "embed.tidal.com",}
'''obtainhostname'''
@lru_cache(maxsize=200_000)
def obtainhostname(url: str) -> str | None:
if not url: return None
u = url.strip()
if "://" not in u: u = "https://" + u
try: host = urlsplit(u).hostname
except Exception: return None
return host.lower().strip(".") if host else None
'''hostmatchessuffix'''
def hostmatchessuffix(host: str | None, suffixes: set[str]) -> bool:
if not host: return False
h = host.lower().strip(".")
for s in suffixes:
s = s.lower().strip(".")
if h == s or h.endswith("." + s): return True
return False
catalog-sync/musicdl/modules/utils/importutils.py
+38
View File
@@ -0,0 +1,38 @@
'''
Function:
Implementation of Optional Import Related Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import sys
import warnings
import importlib
'''optionalimport'''
def optionalimport(name: str, show_warning: bool = False):
if name in sys.modules: return sys.modules[name]
try:
return importlib.import_module(name)
except ModuleNotFoundError:
missing = getattr(optionalimport, "_missing", set())
if (name not in missing) and show_warning: warnings.warn(f'Optional dependency "{name}" is not installed; skipping import.', category=ImportWarning, stacklevel=2)
missing.add(name)
optionalimport._missing = missing
return None
'''optionalimportfrom'''
def optionalimportfrom(module: str, attr: str, show_warning: bool = False):
try:
mod = sys.modules.get(module) or importlib.import_module(module)
return (getattr(mod, attr) if hasattr(mod, attr) else importlib.import_module(f"{module}.{attr}"))
except (ModuleNotFoundError, AttributeError):
key = (module, attr)
missing = getattr(optionalimportfrom, "_missing", set())
if (key not in missing) and show_warning: warnings.warn(f"Optional import failed: from {module} import {attr}", ImportWarning, stacklevel=2)
missing.add(key)
optionalimportfrom._missing = missing
return None
catalog-sync/musicdl/modules/utils/ip.py
+103
View File
@@ -0,0 +1,103 @@
'''
Function:
Implementation of RandomIPGenerator
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import random
import requests
import ipaddress
from bisect import bisect
from typing import List, Optional, Sequence
'''RandomIPGenerator'''
class RandomIPGenerator:
def __init__(self, default_ipv4_prefixes: Optional[Sequence[str]] = None, default_ipv6_prefixes: Optional[Sequence[str]] = None, max_attempts: int = 10000):
self.max_attempts = max_attempts
self.default_ipv4_prefixes: List[str] = list(default_ipv4_prefixes or [])
self.default_ipv6_prefixes: List[str] = list(default_ipv6_prefixes or [])
'''ipv4'''
def ipv4(self, prefix: Optional[str] = None) -> str:
if prefix is None and self.default_ipv4_prefixes: prefix = random.choice(self.default_ipv4_prefixes)
if prefix is not None: return self._randomipv4inprefix(prefix)
else: return self._randomglobalipv4()
'''ipv6'''
def ipv6(self, prefix: Optional[str] = None) -> str:
if prefix is None and self.default_ipv6_prefixes: prefix = random.choice(self.default_ipv6_prefixes)
if prefix is not None: return self._randomipv6inprefix(prefix)
else: return self._randomglobalipv6()
'''randomipv4scn'''
def randomipv4scn(self, num_samples: int = 1) -> List[str]:
def buildsampler_func(blocks):
cum, s = [], 0
for _, c in blocks: s += c; cum.append(s)
total = s
def sample_func(n=10): out=[]; [out.append(str(ipaddress.IPv4Address((lambda bc: bc[0]+random.randrange(bc[1]))(blocks[bisect(cum, random.randrange(total))])))) for _ in range(n)]; return out
return sample_func
blocks = self._loadcnipv4blocks()
sampler = buildsampler_func(blocks)
return sampler(num_samples)
'''addrandomipv4toheaders'''
def addrandomipv4toheaders(self, headers: dict = None, prefix: Optional[str] = None) -> dict:
assert isinstance(headers, dict), f'input "headers" should be "dict", but get {type(headers)}'
random_ip = self.ipv4(prefix=prefix)
headers.update({"X-Forwarded-For": random_ip, "X-Real-IP": random_ip, "Forwarded": f"for={random_ip};proto=https"})
return headers
'''_loadcnipv4blocks'''
def _loadcnipv4blocks(self):
text = requests.get("https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-latest", timeout=30).text.splitlines()
blocks = []
for line in text:
if not line or line.startswith("#"): continue
parts = line.strip().split("|")
if len(parts) < 7: continue
_, cc, rtype, start, value, _, status = parts[:7]
if cc != "CN" or rtype != "ipv4": continue
if status not in ("allocated", "assigned"): continue
base = int(ipaddress.IPv4Address(start))
count = int(value)
if count > 0: blocks.append((base, count))
if not blocks: raise RuntimeError("No CN IPv4 blocks found. Check APNIC file format/URL.")
return blocks
'''_randomipv4inprefix'''
def _randomipv4inprefix(self, prefix: str) -> str:
net = ipaddress.IPv4Network(prefix, strict=False)
if net.prefixlen <= 30:
network_int = int(net.network_address)
broadcast_int = int(net.broadcast_address)
if broadcast_int - network_int <= 2: candidate_int = random.randint(network_int, broadcast_int)
else: candidate_int = random.randint(network_int + 1, broadcast_int - 1)
else:
offset = random.randrange(net.num_addresses)
candidate_int = int(net.network_address) + offset
addr = ipaddress.IPv4Address(candidate_int)
return str(addr)
'''_randomglobalipv4'''
def _randomglobalipv4(self) -> str:
attempts = 0
while attempts < self.max_attempts:
attempts += 1
candidate_int = random.getrandbits(32)
addr = ipaddress.IPv4Address(candidate_int)
if addr.is_global: return str(addr)
return str(addr)
'''_randomipv6inprefix'''
def _randomipv6inprefix(self, prefix: str) -> str:
net = ipaddress.IPv6Network(prefix, strict=False)
host_bits = 128 - net.prefixlen
rand_host = random.getrandbits(host_bits)
addr_int = int(net.network_address) + rand_host
addr = ipaddress.IPv6Address(addr_int)
return str(addr)
'''_randomglobalipv6'''
def _randomglobalipv6(self) -> str:
attempts = 0
while attempts < self.max_attempts:
attempts += 1
candidate_int = random.getrandbits(128)
addr = ipaddress.IPv6Address(candidate_int)
if addr.is_global: return str(addr)
return str(addr)
catalog-sync/musicdl/modules/utils/kugouutils.py
+181
View File
@@ -0,0 +1,181 @@
'''
Function:
Implementation of KugouMusicClient Utils
>>> old api: https://trackercdn.kugou.com/i/?cmd=4&pid=1&forceDown=0&vip=1&hash={file_hash}&key={MD5(file_hash+kgcloud)}
>>> webv2 play: https://trackercdnbj.kugou.com/i/v2/?cmd=23&pid=1&behavior=play&hash={file_hash}&key={MD5(file_hash+kgcloudv2)}
>>> appv2 play: https://trackercdn.kugou.com/i/v2/?appid=1005&pid=2&cmd=25&behavior=play&hash={file_hash}&key={MD5(file_hash+kgcloudv2)}
>>> appv2 download: https://trackercdn.kugou.com/i/v2/?cdnBackup=1&behavior=download&pid=1&cmd=21&appid=1001&hash={file_hash}&key={MD5(file_hash+kgcloudv2)}
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import json
import uuid
import time
import random
import base64
import hashlib
import requests
from Crypto.PublicKey import RSA
from .misc import safeextractfromdict
from typing import Any, Dict, Optional
from Crypto.Cipher import AES, PKCS1_v1_5
'''settings'''
IS_LITE = True
APPID = 3116 if IS_LITE else 1005
CLIENTVER = 11440 if IS_LITE else 20489
MUSIC_QUALITIES = ('viper_tape', 'viper_clear', 'viper_atmos', 'flac', 'high', '320', '128')
SIGNATURE_WEB_SECRET = "NVPh5oo715z5DIWAeQlhMDsWXXQV4hwt"
SIGN_KEY_SECRET = "185672dd44712f60bb1736df5a377e82" if IS_LITE else "57ae12eb6890223e355ccfcb74edf70d"
SIGNATURE_ANDROID_SECRET = "LnT6xpN3khm36zse0QzvmgTZ3waWdRSA" if IS_LITE else "OIlwieks28dk2k092lksi2UIkp"
PUBLIC_RSA_KEY = """-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDECi0Np2UR87scwrvTr72L6oO01rBbbBPriSDFPxr3Z5syug0O24QyQO8bg27+0+4kBzTBTBOZ/WWU0WryL1JSXRTXLgFVxtzIY41Pe7lPOgsfTCn5kZcvKhYKJesKnnJDNr5/abvTGf+rHG3YRwsCHcQ08/q6ifSioBszvb3QiwIDAQAB
-----END PUBLIC KEY-----""" if IS_LITE else """-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDIAG7QOELSYoIJvTFJhMpe1s/gbjDJX51HBNnEl5HXqTW6lQ7LC8jr9fWZTwusknp+sVGzwd40MwP6U5yDE27M/X1+UR4tvOGOqp94TJtQ1EPnWGWXngpeIW5GxoQGao1rmYWAu6oi1z9XkChrsUdC6DJE5E221wf/4WLFxwAtRQIDAQAB
-----END PUBLIC KEY-----"""
'''KugouMusicClientUtils'''
class KugouMusicClientUtils:
'''md5hex'''
@staticmethod
def md5hex(data: Any) -> str:
if isinstance(data, (dict, list)): data = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
if isinstance(data, str): data = data.encode("utf-8")
return hashlib.md5(data).hexdigest()
'''randomstring'''
@staticmethod
def randomstring(length=16) -> str:
chars = "1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ"
return "".join(random.choice(chars) for _ in range(length))
'''calculatemid'''
@staticmethod
def calculatemid(seed: str) -> str:
return str(int(hashlib.md5(seed.encode("utf-8")).hexdigest(), 16))
'''pad'''
@staticmethod
def pad(data: bytes, block_size: int = 16) -> bytes:
pad_len = block_size - len(data) % block_size
return data + bytes([pad_len]) * pad_len
'''unpad'''
@staticmethod
def unpad(data: bytes) -> bytes:
pad_len = data[-1]
return data[:-pad_len]
'''rsaencryptpkcs1'''
@staticmethod
def rsaencryptpkcs1(data: Any, public_key_pem: str = PUBLIC_RSA_KEY) -> str:
if isinstance(data, (dict, list)): data = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
if isinstance(data, str): data = data.encode("utf-8")
rsa_key = RSA.import_key(public_key_pem)
cipher = PKCS1_v1_5.new(rsa_key)
enc = cipher.encrypt(data)
return enc.hex()
'''signatureandroid'''
@staticmethod
def signatureandroid(params: Dict[str, Any], data: str = "") -> str:
params_string = "".join(f"{k}={json.dumps(params[k], separators=(',', ':'), ensure_ascii=False) if isinstance(params[k], (dict, list)) else params[k]}" for k in sorted(params.keys()))
return KugouMusicClientUtils.md5hex(f"{SIGNATURE_ANDROID_SECRET}{params_string}{data}{SIGNATURE_ANDROID_SECRET}")
'''signatureandroidwithsecret'''
@staticmethod
def signatureandroidwithsecret(params: Dict[str, Any], data: str, secret: str = "OIlwieks28dk2k092lksi2UIkp") -> str:
params_string = "".join(f"{k}={json.dumps(params[k], separators=(',', ':'), ensure_ascii=False) if isinstance(params[k], (dict, list)) else params[k]}" for k in sorted(params.keys()))
return KugouMusicClientUtils.md5hex(f"{secret}{params_string}{data}{secret}")
'''signatureweb'''
@staticmethod
def signatureweb(params: Dict[str, Any]) -> str:
params_string = "".join(f"{k}={params[k]}" for k in sorted(params.keys()))
return KugouMusicClientUtils.md5hex(f"{SIGNATURE_WEB_SECRET}{params_string}{SIGNATURE_WEB_SECRET}")
'''signkey'''
@staticmethod
def signkey(hash_value: str, mid: str, userid: str, appid: str) -> str:
return KugouMusicClientUtils.md5hex(f"{hash_value}{SIGN_KEY_SECRET}{appid}{mid}{userid or 0}")
'''initdevice'''
@staticmethod
def initdevice(cookies: dict = None):
cookies = cookies or {}
guid = str(uuid.uuid4())
mid = KugouMusicClientUtils.calculatemid(guid)
cookies["KUGOU_API_GUID"] = guid
cookies["KUGOU_API_MID"] = mid
cookies["KUGOU_API_MAC"] = KugouMusicClientUtils.randomstring(12)
cookies["KUGOU_API_DEV"] = KugouMusicClientUtils.randomstring(16)
return cookies
'''updatecookies'''
@staticmethod
def updatecookies(resp: requests.Response, cookies: dict):
for k, v in resp.cookies.items(): cookies[k] = v
return cookies
'''sendrequest'''
@staticmethod
def sendrequest(session: requests.Session, method: str, url: str, params: Optional[Dict[str, Any]] = None, data: Optional[Any] = None, headers: Optional[Dict[str, str]] = None, encrypt_type: str = "android", base_url: str = "https://gateway.kugou.com", encrypt_key: bool = False, not_sign: bool = False, response_type: Optional[str] = None, cookies: Optional[Dict[str, str]] = None, cookies_override: Optional[Dict[str, str]] = None, request_overrides: dict = None):
# init
clienttime = int(time.time())
params, headers, used_cookies, request_overrides = params or {}, headers or {}, dict(cookies), request_overrides or {}
if cookies_override: used_cookies.update(cookies_override)
token, dfid, userid, mid = used_cookies.get("token", ""), used_cookies.get("dfid", "-"), used_cookies.get("userid", 0), used_cookies.get("KUGOU_API_MID", "-")
# construct params
default_params = {"dfid": dfid, "mid": mid, "uuid": "-", "appid": APPID, "clientver": CLIENTVER, "clienttime": clienttime}
if token: default_params["token"] = token
if userid: default_params["userid"] = userid
params = {**default_params, **params}
# encrypt key
if encrypt_key: params["key"] = KugouMusicClientUtils.signkey(params["hash"], params["mid"], params.get("userid"), params["appid"])
# signature
data_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False) if isinstance(data, (dict, list)) else (data or "")
if not_sign:
if "signature" in params: params.pop("signature", None)
else:
if "signature" not in params: params["signature"] = KugouMusicClientUtils.signatureweb(params) if encrypt_type == "web" else KugouMusicClientUtils.signatureandroid(params, data_str)
# construct headers
base_headers = {"User-Agent": "Android15-1070-11083-46-0-DiscoveryDRADProtocol-wifi", "dfid": dfid, "clienttime": str(params["clienttime"]), "mid": mid, "kg-rc": "1", "kg-thash": "5d816a0", "kg-rec": "1", "kg-rf": "B9EDA08A64250DEFFBCADDEE00F8F25F"}
final_headers = {**base_headers, **headers}
# send request
resp = session.request(method, f"{base_url}{url}", params=params, json=data, headers=final_headers, **request_overrides) if isinstance(data, (dict, list)) else session.request(method, f"{base_url}{url}", params=params, data=data, headers=final_headers, **request_overrides)
resp.raise_for_status()
KugouMusicClientUtils.updatecookies(resp, cookies)
# return
if response_type == "arraybuffer": return resp.content
try: return resp.json()
except Exception: return resp.text
'''registerdevice'''
@staticmethod
def registerdevice(session: requests.Session, cookies: dict, request_overrides: dict = None):
# construct
data_map = {
"availableRamSize": 4983533568, "availableRomSize": 48114719, "availableSDSize": 48114717, "basebandVer": "", "batteryLevel": 100, "batteryStatus": 3, "brand": "Redmi", "buildSerial": "unknown",
"device": "marble", "imei": cookies.get("KUGOU_API_GUID"), "imsi": "", "manufacturer": "Xiaomi", "uuid": cookies.get("KUGOU_API_GUID"), "accelerometer": False, "accelerometerValue": "",
"gravity": False, "gravityValue": "", "gyroscope": False, "gyroscopeValue": "", "light": False, "lightValue": "", "magnetic": False, "magneticValue": "", "orientation": False, "orientationValue": "",
"pressure": False, "pressureValue": "", "step_counter": False, "step_counterValue": "", "temperature": False, "temperatureValue": "",
}
# aes
aes_key = KugouMusicClientUtils.randomstring(6).lower(); encrypt_key = KugouMusicClientUtils.md5hex(aes_key)[:16]; encrypt_iv = KugouMusicClientUtils.md5hex(aes_key)[16: 32]
cipher = AES.new(encrypt_key.encode("utf-8"), AES.MODE_CBC, encrypt_iv.encode("utf-8"))
raw = json.dumps(data_map, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
enc = cipher.encrypt(KugouMusicClientUtils.pad(raw))
aes_body = base64.b64encode(enc).decode("utf-8")
p = KugouMusicClientUtils.rsaencryptpkcs1({"aes": aes_key, "uid": cookies.get("userid", 0), "token": cookies.get("token", "")})
# send request and return result
resp_raw: bytes = KugouMusicClientUtils.sendrequest(session, "POST", "/risk/v2/r_register_dev", params={"part": 1, "platid": 1, "p": p}, data=aes_body, base_url="https://userservice.kugou.com", encrypt_type="android", response_type="arraybuffer", cookies=cookies, request_overrides=request_overrides)
try:
text: str = resp_raw.decode("utf-8"); result = json.loads(text) if text.startswith("{") else None
if result: return result
except Exception:
pass
dec_cipher = AES.new(encrypt_key.encode("utf-8"), AES.MODE_CBC, encrypt_iv.encode("utf-8"))
decrypted = KugouMusicClientUtils.unpad(dec_cipher.decrypt(resp_raw)).decode("utf-8")
result: dict = json.loads(decrypted)
if result.get("status") == 1 and safeextractfromdict(result, ['data', 'dfid'], None): cookies["dfid"] = result["data"]["dfid"]
return result
'''getsongurl'''
@staticmethod
def getsongurl(session: requests.Session, hash_value: str, album_id: int = 0, album_audio_id: int = 0, quality: str = "128", free_part: bool = False, cookies: dict = None, request_overrides: dict = None):
params = {
"album_id": int(album_id), "area_code": 1, "hash": hash_value.lower(), "ssa_flag": "is_fromtrack", "version": 11436, "page_id": 151369488 if not IS_LITE else 967177915,
"quality": quality, "album_audio_id": int(album_audio_id), "behavior": "play", "pid": 2 if not IS_LITE else 411, "cmd": 26, "pidversion": 3001, "IsFreePart": 1 if free_part else 0,
"ppage_id": "463467626,350369493,788954147" if not IS_LITE else "356753938,823673182,967485191", "cdnBackup": 1, "kcard": 0, "module": "",
}
return KugouMusicClientUtils.sendrequest(session, "GET", "/v5/url", params=params, headers={"x-router": "trackercdn.kugou.com"}, encrypt_type="android", encrypt_key=True, cookies=cookies, cookies_override={'dfid': KugouMusicClientUtils.randomstring(24)}, request_overrides=request_overrides)
catalog-sync/musicdl/modules/utils/kuwoutils.py
+193
View File
@@ -0,0 +1,193 @@
'''
Function:
Implementation of KuwoMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import re
import math
import zlib
import base64
'''settings'''
MASK32 = (1 << 32) - 1
MASK64 = (1 << 64) - 1
'''HelperFunctions'''
class HelperFunctions():
@staticmethod
def u64(x: int) -> int: return x & MASK64
@staticmethod
def u32(x: int) -> int: return x & MASK32
@staticmethod
def rangen(n: int): return range(n)
@staticmethod
def power2(n: int) -> int: return 1 << n
@staticmethod
def longarray(*arr): return list(arr)
'''settings'''
SECRET_KEY_SONG, SECRET_KEY_LYRIC = b"ylzsxkwm", b'yeelion'
ARRAYLS = [1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1]
ARRAYLSMASK = HelperFunctions.longarray(0, 0x100001, 0x300003)
ARRAYE = HelperFunctions.longarray(31, 0, 1, 2, 3, 4, -1, -1, 3, 4, 5, 6, 7, 8, -1, -1, 7, 8, 9, 10, 11, 12, -1, -1, 11, 12, 13, 14, 15, 16, -1, -1, 15, 16, 17, 18, 19, 20, -1, -1, 19, 20, 21, 22, 23, 24, -1, -1, 23, 24, 25, 26, 27, 28, -1, -1, 27, 28, 29, 30, 31, 30, -1, -1)
ARRAYIP1 = HelperFunctions.longarray(39, 7, 47, 15, 55, 23, 63, 31, 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25, 32, 0, 40, 8, 48, 16, 56, 24)
ARRAYIP2 = HelperFunctions.longarray(57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, 56, 48, 40, 32, 24, 16, 8, 0, 58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, 62, 54, 46, 38, 30, 22, 14, 6)
ARRAYMASK = [HelperFunctions.power2(n) for n in HelperFunctions.rangen(64)]
ARRAYMASK[-1] = -ARRAYMASK[-1]
ARRAYP = HelperFunctions.longarray(15, 6, 19, 20, 28, 11, 27, 16, 0, 14, 22, 25, 4, 17, 30, 9, 1, 7, 23, 13, 31, 26, 2, 8, 18, 12, 29, 5, 21, 10, 3, 24)
ARRAYPC1 = HelperFunctions.longarray(56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3)
ARRAYPC2 = HelperFunctions.longarray(13, 16, 10, 23, 0, 4, -1, -1, 2, 27, 14, 5, 20, 9, -1, -1, 22, 18, 11, 3, 25, 7, -1, -1, 15, 6, 26, 19, 12, 1, -1, -1, 40, 51, 30, 36, 46, 54, -1, -1, 29, 39, 50, 44, 32, 47, -1, -1, 43, 48, 38, 55, 33, 52, -1, -1, 45, 41, 49, 35, 28, 31, -1, -1)
MATRIXNSBOX = [
[14,4,3,15,2,13,5,3,13,14,6,9,11,2,0,5,4,1,10,12,15,6,9,10,1,8,12,7,8,11,7,0,0,15,10,5,14,4,9,10,7,8,12,3,13,1,3,6,15,12,6,11,2,9,5,0,4,2,11,14,1,7,8,13],
[15,0,9,5,6,10,12,9,8,7,2,12,3,13,5,2,1,14,7,8,11,4,0,3,14,11,13,6,4,1,10,15,3,13,12,11,15,3,6,0,4,10,1,7,8,4,11,14,13,8,0,6,2,15,9,5,7,1,10,12,14,2,5,9],
[10,13,1,11,6,8,11,5,9,4,12,2,15,3,2,14,0,6,13,1,3,15,4,10,14,9,7,12,5,0,8,7,13,1,2,4,3,6,12,11,0,13,5,14,6,8,15,2,7,10,8,15,4,9,11,5,9,0,14,3,10,7,1,12],
[7,10,1,15,0,12,11,5,14,9,8,3,9,7,4,8,13,6,2,1,6,11,12,2,3,0,5,14,10,13,15,4,13,3,4,9,6,10,1,12,11,0,2,5,0,13,14,2,8,15,7,4,15,1,10,7,5,6,12,11,3,8,9,14],
[2,4,8,15,7,10,13,6,4,1,3,12,11,7,14,0,12,2,5,9,10,13,0,3,1,11,15,5,6,8,9,14,14,11,5,6,4,1,3,10,2,12,15,0,13,2,8,5,11,8,0,15,7,14,9,4,12,7,10,9,1,13,6,3],
[12,9,0,7,9,2,14,1,10,15,3,4,6,12,5,11,1,14,13,0,2,8,7,13,15,5,4,10,8,3,11,6,10,4,6,11,7,9,0,6,4,2,13,1,9,15,3,8,15,3,1,14,12,5,11,0,2,12,14,7,5,10,8,13],
[4,1,3,10,15,12,5,0,2,11,9,6,8,7,6,9,11,4,12,15,0,3,10,5,14,13,7,8,13,14,1,2,13,6,14,9,4,1,2,14,11,13,5,0,1,10,8,3,0,11,3,5,9,4,15,2,7,8,12,15,10,7,6,12],
[13,7,10,0,6,9,5,15,8,4,3,10,11,14,12,5,2,11,9,6,15,12,0,3,4,1,14,13,1,2,7,8,1,2,12,15,10,4,0,3,13,14,6,9,7,8,9,6,15,1,5,12,3,10,14,5,8,7,11,0,4,13,2,11],
]
'''KuwoMusicClientUtils'''
class KuwoMusicClientUtils:
'''bittransform'''
@staticmethod
def bittransform(arr_int, n, l):
l2 = 0
for i in HelperFunctions.rangen(n):
idx = arr_int[i]
if idx < 0: continue
if (l & ARRAYMASK[idx]) == 0: continue
l2 |= ARRAYMASK[i]
return HelperFunctions.u64(l2)
'''des64'''
@staticmethod
def des64(longs, l):
p_r, p_source, out = [0] * 8, [0, 0], KuwoMusicClientUtils.bittransform(ARRAYIP2, 64, l)
p_source[0], p_source[1] = HelperFunctions.u32(out), HelperFunctions.u32((out & 0xFFFFFFFF00000000) >> 32)
for i in HelperFunctions.rangen(16):
s_out, R = 0, KuwoMusicClientUtils.bittransform(ARRAYE, 64, p_source[1])
R ^= longs[i]
for j in HelperFunctions.rangen(8): p_r[j] = (R >> (j * 8)) & 0xFF
for sbi in reversed(HelperFunctions.rangen(8)): s_out = (s_out << 4) | (MATRIXNSBOX[sbi][p_r[sbi]] & 0xF)
R, L = KuwoMusicClientUtils.bittransform(ARRAYP, 32, s_out), p_source[0]
p_source[0] = p_source[1]
p_source[1] = HelperFunctions.u32(L ^ R)
p_source.reverse()
out = ((p_source[1] << 32) & 0xFFFFFFFF00000000) | (p_source[0] & 0xFFFFFFFF)
out = KuwoMusicClientUtils.bittransform(ARRAYIP1, 64, out)
return HelperFunctions.u64(out)
'''subkeys'''
@staticmethod
def subkeys(l, longs, mode):
l2 = KuwoMusicClientUtils.bittransform(ARRAYPC1, 56, l)
for i in HelperFunctions.rangen(16):
r = ARRAYLS[i]
mask = ARRAYLSMASK[r]
not_mask = HelperFunctions.u64(~mask)
part1, part2 = HelperFunctions.u64((l2 & mask) << (28 - r)), (l2 & not_mask) >> r
l2 = HelperFunctions.u64(part1 | part2)
longs[i] = KuwoMusicClientUtils.bittransform(ARRAYPC2, 64, l2)
if mode == 1:
for j in HelperFunctions.rangen(8): longs[j], longs[15 - j] = longs[15 - j], longs[j]
'''crypt'''
@staticmethod
def crypt(msg: bytes, key: bytes, mode: int) -> bytes:
l = 0
for i in HelperFunctions.rangen(8): l |= (key[i] & 0xFF) << (i * 8)
l, j, arr_long1 = HelperFunctions.u64(l), len(msg) // 8, [0] * 16
KuwoMusicClientUtils.subkeys(l, arr_long1, mode)
arr_long2 = [0] * j
for m in HelperFunctions.rangen(j):
v = 0
for n in HelperFunctions.rangen(8): v |= (msg[n + m * 8] & 0xFF) << (n * 8)
arr_long2[m] = HelperFunctions.u64(v)
arr_long3 = [0] * ((1 + 8 * (j + 1)) // 8)
for i1 in HelperFunctions.rangen(j): arr_long3[i1] = KuwoMusicClientUtils.des64(arr_long1, arr_long2[i1])
arr_byte1, l2 = msg[j * 8:], 0
for i1 in HelperFunctions.rangen(len(msg) % 8): l2 |= (arr_byte1[i1] & 0xFF) << (i1 * 8)
l2 = HelperFunctions.u64(l2)
if len(arr_byte1) != 0 or mode == 0: arr_long3[j] = KuwoMusicClientUtils.des64(arr_long1, l2)
out_bytes, i4 = bytearray(8 * len(arr_long3)), 0
for l3 in arr_long3:
for i6 in HelperFunctions.rangen(8): out_bytes[i4] = (l3 >> (i6 * 8)) & 0xFF; i4 += 1
return bytes(out_bytes)
'''encrypt'''
@staticmethod
def encrypt(msg: bytes) -> bytes:
return KuwoMusicClientUtils.crypt(msg, SECRET_KEY_SONG, 0)
'''decrypt'''
@staticmethod
def decrypt(msg: bytes) -> bytes:
return KuwoMusicClientUtils.crypt(msg, SECRET_KEY_SONG, 1)
'''encryptquery'''
@staticmethod
def encryptquery(query: str) -> str:
return base64.b64encode(KuwoMusicClientUtils.encrypt(query.encode("utf-8"))).decode("ascii")
'''xorencrypt'''
@staticmethod
def xorencrypt(data: bytes, key: bytes) -> bytes:
key_len = len(key)
output = bytearray(len(data))
for i in range(len(data)): output[i] = data[i] ^ key[i % key_len]
return bytes(output)
'''buildlyricsparams'''
@staticmethod
def buildlyricsparams(music_id, is_get_lyricx: bool = True):
params_str = f"user=12345,web,web,web&requester=localhost&req=1&rid=MUSIC_{music_id}"
if is_get_lyricx: params_str += '&lrcx=1'
buf_str = params_str.encode('utf-8')
encrypted_bytes = KuwoMusicClientUtils.xorencrypt(buf_str, SECRET_KEY_LYRIC)
final_params = base64.b64encode(encrypted_bytes).decode('utf-8')
return final_params
'''decodelyrics'''
@staticmethod
def decodelyrics(buf: bytes, is_get_lyricx: bool):
if buf[:10] != b'tp=content': return ''
try: split_index = buf.index(b'\r\n\r\n') + 4; compressed_data = buf[split_index:]
except ValueError: return ''
try: lrc_data = zlib.decompress(compressed_data)
except zlib.error: return ''
if not is_get_lyricx: return lrc_data.decode('gb18030', errors='ignore')
base64_str = lrc_data.decode('utf-8')
buf_str = base64.b64decode(base64_str)
decrypted_buffer = KuwoMusicClientUtils.xorencrypt(buf_str, SECRET_KEY_LYRIC)
final_lrc = decrypted_buffer.decode('gb18030', errors='ignore')
return final_lrc
'''formatlyricstime'''
@staticmethod
def formatlyricstime(ms):
if math.isnan(ms) or ms < 0: ms = 0
total_seconds = ms / 1000
minutes = math.floor(total_seconds / 60)
seconds = math.floor(total_seconds % 60)
milliseconds = round((ms % 1000))
return f"[{minutes:02}:{seconds:02}.{milliseconds:03}]"
'''convertrawlrc'''
@staticmethod
def convertrawlrc(raw_lrc: str) -> str:
out, i = [], 0
lines, rx_line, rx_word, rx_zh = re.split(r"\r\n|\r|\n", raw_lrc), re.compile(r"^\[(\d{2}:\d{2}\.\d{3})\](.*)$"), re.compile(r"<(-?\d+),(-?\d+)>([^<]*)"), re.compile(r"[\u4e00-\u9fa5]")
while i < len(lines):
line = lines[i]
m = rx_line.match(line)
if not m: out.append(line); i += 1; continue
ts, payload = m.group(1), m.group(2)
if not payload.replace("<0,0>", "").strip(): i += 1; continue
if payload.startswith("<0,0>") and rx_zh.search(payload): i += 1; continue
words = list(rx_word.finditer(payload))
lyric = "".join(w.group(3) for w in words) if words else payload.replace("<0,0>", "").strip(); trans = ""
if i + 1 < len(lines) and (nm := rx_line.match(lines[i + 1])):
next_payload = nm.group(2)
if next_payload.startswith("<0,0>") and rx_zh.search(next_payload): trans = next_payload.replace("<0,0>", "").strip(); i += 1
out.append(f"[{ts}]{lyric}")
if trans: out.append(f"[{ts}]{trans}")
i += 1
return "\n".join(out)
catalog-sync/musicdl/modules/utils/lanzouyparser.py
+143
View File
@@ -0,0 +1,143 @@
'''
Function:
Implementation of LanZouYParser
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import re
import json
import random
import requests
from urllib.parse import urljoin, urlparse
'''LanZouYParser'''
class LanZouYParser():
'''parsefromurl'''
@staticmethod
def parsefromurl(url: str, passcode: str = '', max_tries: int = 3):
for _ in range(max_tries):
try:
download_result, download_url = LanZouYParser._parsefromurl(url=url, passcode=passcode)
assert download_url and str(download_url).startswith('http')
break
except:
download_result, download_url = {}, ""
if not download_url or not str(download_url).startswith('http'):
file_id = urlparse(url).path.strip('/').split('/')[-1]
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36'}
try:
resp = requests.get(f'https://api-v2.cenguigui.cn/api/lanzou/api.php?url=https://cenguigui.lanzouw.com/{file_id}', headers=headers)
download_result = resp.json()
download_url = download_result['data']['downurl']
assert download_url and str(download_url).startswith('http')
break
except:
download_result, download_url = {}, ""
return download_result, download_url
'''_randip'''
@staticmethod
def _randip() -> str:
ip2 = round(random.randint(600000, 2550000) / 10000)
ip3 = round(random.randint(600000, 2550000) / 10000)
ip4 = round(random.randint(600000, 2550000) / 10000)
arr1 = ["218", "218", "66", "66", "218", "218", "60", "60", "202", "204", "66", "66", "66", "59", "61", "60", "222", "221", "66", "59", "60", "60", "66", "218", "218", "62", "63", "64", "66", "66", "122", "211"]
ip1 = random.choice(arr1)
return f"{ip1}.{ip2}.{ip3}.{ip4}"
'''_httpget'''
@staticmethod
def _httpget(url: str, user_agent: str = "", referer: str = "", cookies: dict = None, timeout: int = 10) -> str:
headers = {"X-FORWARDED-FOR": LanZouYParser._randip(), "CLIENT-IP": LanZouYParser._randip()}
if user_agent: headers["User-Agent"] = user_agent
if referer: headers["Referer"] = referer
resp = requests.get(url, headers=headers, cookies=cookies, timeout=timeout, verify=False, allow_redirects=True)
resp.raise_for_status()
resp.encoding = resp.apparent_encoding or "utf-8"
return resp.text
'''_httppost'''
@staticmethod
def _httppost(data: dict, url: str, referer: str = "", user_agent: str = "", timeout: int = 10) -> str:
headers = {"X-FORWARDED-FOR": LanZouYParser._randip(), "CLIENT-IP": LanZouYParser._randip()}
if user_agent: headers["User-Agent"] = user_agent
if referer: headers["Referer"] = referer
resp = requests.post(url, data=data, headers=headers, timeout=timeout, verify=False, allow_redirects=True)
resp.raise_for_status()
resp.encoding = resp.apparent_encoding or "utf-8"
return resp.text
'''_httpredirecturl'''
@staticmethod
def _httpredirecturl(url: str, referer: str, user_agent: str, cookie_str: str, timeout: int = 10) -> str:
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9", "Cache-Control": "no-cache", "Connection": "keep-alive", "Pragma": "no-cache", "Upgrade-Insecure-Requests": "1",
"User-Agent": user_agent, "Referer": referer, "Cookie": cookie_str,
}
resp = requests.get(url, headers=headers, timeout=timeout, verify=False, allow_redirects=False)
resp.raise_for_status()
loc = resp.headers.get("Location", "") or resp.headers.get("location", "")
if not loc: return ""
return urljoin(url, loc)
'''_acwscv2simple'''
@staticmethod
def _acwscv2simple(arg1: str):
if not arg1: return ""
mask = "3000176000856006061501533003690027800375"
pos_list = (15, 35, 29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36)
arg2 = "".join(arg1[p - 1] for p in pos_list if p <= len(arg1))
length = min(len(arg2), len(mask))
return "".join(f"{(int(arg2[i:i+2], 16) ^ int(mask[i:i+2], 16)):02x}" for i in range(0, length, 2))
'''_parsefromurl'''
@staticmethod
def _parsefromurl(url: str, passcode: str = ''):
# init
download_result, user_agent = {}, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
normalize_lanzou_url_func = lambda u: ("https://www.lanzouf.com/" + t.lstrip("/") if (t := (u.split(".com/", 1)[1] if ".com/" in u else None)) is not None else ("https://www.lanzouf.com" + u) if u.startswith("/") else u if u.startswith("http") else "https://www.lanzouf.com/" + u.lstrip("/"))
extract_first_func = lambda regex_list, text: next((m.group(1) for rgx in regex_list if (m := re.search(rgx, text, flags=re.S))), "")
# vist home page
url = normalize_lanzou_url_func(url)
homepage_url_html = LanZouYParser._httpget(url, user_agent=user_agent)
if "文件取消分享了" in homepage_url_html: raise
soft_name = extract_first_func([r'style="font-size: 30px;text-align: center;padding: 56px 0px 20px 0px;">(.*?)</div>', r'<div class="n_box_3fn".*?>(.*?)</div>', r"var filename = '(.*?)';", r'div class="b"><span>(.*?)</span></div>'], homepage_url_html)
soft_size = extract_first_func([r'<div class="n_filesize".*?>大小：(.*?)</div>', r'<span class="p7">文件大小：</span>(.*?)<br>'], homepage_url_html)
# with passcode
if "function down_p(){" in homepage_url_html:
segment = re.findall(r"'sign':'(.*?)',", homepage_url_html, flags=re.S)
ajaxm = re.findall(r"ajaxm\.php\?file=\d+", homepage_url_html, flags=re.S)
assert not (len(segment) < 2 or len(ajaxm) < 1)
post_data = {"action": "downprocess", "sign": segment[1], "p": passcode, "kd": 1}
post_url = "https://www.lanzouf.com/" + ajaxm[0]
parse_result = LanZouYParser._httppost(post_data, post_url, referer=url, user_agent=user_agent)
parse_result: dict = json.loads(parse_result)
soft_name = parse_result.get("inf") or soft_name
# without passcode
else:
link = extract_first_func([r'\n<iframe.*?name="[\s\S]*?"\ssrc="\/(.*?)"', r'<iframe.*?name="[\s\S]*?"\ssrc="\/(.*?)"'], homepage_url_html)
assert link
ifurl = "https://www.lanzouf.com/" + link.lstrip("/")
iframe_html = LanZouYParser._httpget(ifurl, user_agent=user_agent)
wp_sign = re.findall(r"wp_sign = '(.*?)'", iframe_html, flags=re.S)
ajaxdata = re.findall(r"ajaxdata = '(.*?)'", iframe_html, flags=re.S)
ajaxm = re.findall(r"ajaxm\.php\?file=\d+", iframe_html, flags=re.S)
assert not (len(wp_sign) < 1 or len(ajaxdata) < 1 or len(ajaxm) < 2)
post_data = {"action": "downprocess", "websignkey": ajaxdata[0], "signs": ajaxdata[0], "sign": wp_sign[0], "websign": "", "kd": 1, "ves": 1}
post_url = "https://www.lanzouf.com/" + ajaxm[1]
parse_result = LanZouYParser._httppost(post_data, post_url, referer=ifurl, user_agent=user_agent)
parse_result: dict = json.loads(parse_result)
# final parse
assert not (not isinstance(parse_result, dict) or parse_result.get("zt") != 1)
download_url = f"{parse_result['dom']}/file/{parse_result['url']}"
download_html = LanZouYParser._httpget(download_url, user_agent=user_agent)
arg1_list = re.findall(r"arg1='(.*?)'", download_html, flags=re.S)
if arg1_list:
decrypted = LanZouYParser._acwscv2simple(arg1_list[0])
cookie_str = f"down_ip=1; expires=Sat, 16-Nov-2019 11:42:54 GMT; path=/; domain=.baidupan.com; acw_sc__v2={decrypted}"
redirected_download_url = LanZouYParser._httpredirecturl(download_url, referer="https://developer.lanzoug.com", user_agent=user_agent, cookie_str=cookie_str)
if "http" in (redirected_download_url or ""): download_url = redirected_download_url
download_url = re.sub(r"pid=[^&]*&", "", download_url)
download_result = {"name": soft_name or "", "filesize": soft_size or "", "downUrl": download_url, "parse_result": parse_result}
# return
return download_result, download_url
catalog-sync/musicdl/modules/utils/logger.py
+326
View File
@@ -0,0 +1,326 @@
'''
Function:
Implementation of Logging Related Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import re
import os
import shutil
import logging
import collections.abc
import tabulate as tabmod
from wcwidth import wcswidth
from tabulate import tabulate
from prettytable import PrettyTable
from platformdirs import user_log_dir
from prompt_toolkit.layout import Layout
from prompt_toolkit.application import Application
from prompt_toolkit.key_binding import KeyBindings
from prompt_toolkit.layout.containers import HSplit, Window
from prompt_toolkit.application.current import get_app_or_none
from prompt_toolkit.layout.controls import FormattedTextControl
from prompt_toolkit.formatted_text import ANSI, to_formatted_text
from typing import Any, List, Optional, Sequence, Set, Tuple, Union, Dict
from prompt_toolkit.formatted_text.utils import fragment_list_width, split_lines, get_cwidth
'''settings'''
tabmod.WIDE_CHARS_MODE = True
NoTruncSpec = Optional[Sequence[Union[int, str]]]
ANSI_CSI_RE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]")
AMBIGUOUS_MAP: Dict[str, str] = {
"·": ".", "•": "*", "…": "...", "“": '"', "”": '"', "„": '"', "‟": '"', "‘": "'", "’": "'", "‚": "'", "‛": "'", "—": "-", "–": "-", "−": "-", "　": " ",
}
COLORS = {
'red': '\033[31m', 'green': '\033[32m', 'yellow': '\033[33m', 'blue': '\033[34m', 'pink': '\033[35m', 'cyan': '\033[36m', 'highlight': '\033[93m',
'number': '\033[96m', 'singer': '\033[93m', 'flac': '\033[95m', 'songname': '\033[91m'
}
'''LoggerHandle'''
class LoggerHandle():
appname, appauthor = 'musicdl', 'zcjin'
def __init__(self):
# set up log dir
log_dir = user_log_dir(appname=self.appname, appauthor=self.appauthor)
os.makedirs(log_dir, exist_ok=True)
log_file_path = os.path.join(log_dir, "musicdl.log")
self.log_file_path = log_file_path
# config logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", handlers=[logging.FileHandler(log_file_path, encoding="utf-8"), logging.StreamHandler()])
'''log'''
@staticmethod
def log(level, message):
message = str(message)
logger = logging.getLogger(LoggerHandle.appname)
logger.log(level, message)
'''debug'''
def debug(self, message, disable_print=False):
message = str(message)
if disable_print:
fp = open(self.log_file_path, 'a', encoding='utf-8')
fp.write(message + '\n')
else:
LoggerHandle.log(logging.DEBUG, message)
'''info'''
def info(self, message, disable_print=False):
message = str(message)
if disable_print:
fp = open(self.log_file_path, 'a', encoding='utf-8')
fp.write(message + '\n')
else:
LoggerHandle.log(logging.INFO, message)
'''warning'''
def warning(self, message, disable_print=False):
message = str(message)
if disable_print:
fp = open(self.log_file_path, 'a', encoding='utf-8')
fp.write(message + '\n')
else:
if '\033[31m' not in message: message = colorize(message, 'red')
LoggerHandle.log(logging.WARNING, message)
'''error'''
def error(self, message, disable_print=False):
message = str(message)
if disable_print:
fp = open(self.log_file_path, 'a', encoding='utf-8')
fp.write(message + '\n')
else:
if '\033[31m' not in message: message = colorize(message, 'red')
LoggerHandle.log(logging.ERROR, message)
'''colorize'''
def colorize(string, color):
string = str(string)
if color not in COLORS: return string
return COLORS[color] + string + '\033[0m'
'''printfullline'''
def printfullline(ch: str = "*", end: str = '\n', terminal_right_space_len: int = 1):
cols = shutil.get_terminal_size().columns - terminal_right_space_len
assert cols > 0, f'"terminal_right_space_len" should smaller than {shutil.get_terminal_size()}'
print(ch * cols, end=end)
'''printtable'''
def printtable(titles, items, terminal_right_space_len=4):
assert isinstance(titles, collections.abc.Sequence) and isinstance(items, collections.abc.Sequence), 'title and items should be iterable'
table = PrettyTable(titles)
for item in items: table.add_row(item)
max_width = shutil.get_terminal_size().columns - terminal_right_space_len
assert max_width > 0, f'"terminal_right_space_len" should smaller than {shutil.get_terminal_size()}'
table.max_table_width = max_width
print(table)
return table
'''ptsizefallback'''
def ptsizefallback() -> Tuple[int, int]:
app = get_app_or_none()
if app is not None and getattr(app, "output", None) is not None:
try:
sz = app.output.get_size()
cols, rows = int(sz.columns), int(sz.rows)
if cols > 0 and rows > 0: return cols, rows
except Exception:
pass
s = shutil.get_terminal_size(fallback=(80, 24))
return int(s.columns), int(s.lines)
'''stripansi'''
def stripansi(s: str) -> str:
return ANSI_CSI_RE.sub("", s)
'''dispwidth'''
def dispwidth(s: Any) -> int:
if s is None: return 0
w = wcswidth(stripansi(str(s)))
return max(0, w)
'''normalizeforconsole'''
def normalizeforconsole(text: Any, *, enable: bool) -> str:
s = "" if text is None else str(text)
if not s: return s
s = s.replace("\r", "")
s = s.replace("\n", " ").replace("\t", " ")
if enable: s = "".join(AMBIGUOUS_MAP.get(ch, ch) for ch in s)
return s
'''truncatebydispwidth'''
def truncatebydispwidth(text: Any, max_width: int, ellipsis: str = "...") -> str:
s = "" if text is None else str(text)
if max_width <= 0: return ""
if dispwidth(s) <= max_width: return s
ell_w = dispwidth(ellipsis)
target = max_width if max_width <= ell_w else (max_width - ell_w)
out, used, i, emitted_ansi = [], 0, 0, False
while i < len(s) and used < target:
if s[i] == "\x1b":
m = ANSI_CSI_RE.match(s, i)
if m: out.append(m.group(0)); emitted_ansi = True; i = m.end(); continue
i += 1; continue
ch = s[i]; ch_w = max(wcswidth(ch), 0)
if used + ch_w > target: break
out.append(ch); used += ch_w; i += 1
if emitted_ansi and (not out or not str(out[-1]).endswith("\x1b[0m")): out.append("\x1b[0m")
core = "".join(out)
return core if max_width <= ell_w else (core + ellipsis)
'''truncatefragmentstocols'''
def truncatefragmentstocols(fragments: Sequence[Tuple], cols: int) -> List[Tuple]:
if cols <= 0: return []
out, used = [], 0
for style, text, *rest in fragments:
if not text: continue
buf: List[str] = []
for ch in text:
cw = get_cwidth(ch)
if used + cw > cols: break
buf.append(ch); used += cw
if buf: out.append((style, "".join(buf), *rest))
if used >= cols: break
return out
'''truncateandpadline'''
def truncateandpadline(fragments: Sequence[Tuple], cols: int) -> List[Tuple]:
line = truncatefragmentstocols(fragments, cols)
pad = cols - fragment_list_width(line)
if pad > 0: return list(line) + [("", " " * pad)]
return truncatefragmentstocols(line, cols)
'''smarttrunctable'''
def smarttrunctable(headers: Sequence[Any], rows: Sequence[Sequence[Any]], *, max_col_width: int = 40, min_col_width: int = 4, terminal_right_space_len: int = 2, no_trunc_cols: NoTruncSpec = None, term_width: Optional[int] = None, tablefmt: str = "grid", max_iterations: int = 2000) -> str:
headers_s = ["" if h is None else str(h) for h in headers]
rows_s, ncols = [[("" if c is None else str(c)) for c in r] for r in rows], len(headers_s)
if any(len(r) != ncols for r in rows_s): raise ValueError("All rows must have the same number of columns as headers")
if term_width is None: term_width = ptsizefallback()[0]
target_width = max(1, term_width - max(0, terminal_right_space_len))
protected: Set[int] = set()
if no_trunc_cols:
header_to_idx = {h: i for i, h in enumerate(headers_s)}
for spec in no_trunc_cols:
if isinstance(spec, int) and 0 <= spec < ncols: protected.add(spec)
elif not isinstance(spec, int):
idx = header_to_idx.get(str(spec))
if idx is not None: protected.add(idx)
col_natural = [dispwidth(h) for h in headers_s]
col_natural = [max(col_natural[j], *(dispwidth(r[j]) for r in rows_s)) for j in range(len(col_natural))]
col_limit: List[Optional[int]] = []
for j in range(ncols):
if j in protected: col_limit.append(None)
else: cap = col_natural[j]; cap = min(cap, max_col_width) if max_col_width else cap; col_limit.append(max(min_col_width, cap))
def rendercurrent() -> str:
th = [h if col_limit[j] is None else truncatebydispwidth(h, col_limit[j]) for j, h in enumerate(headers_s)]
tr = [[cell if col_limit[j] is None else truncatebydispwidth(cell, col_limit[j]) for j, cell in enumerate(r)] for r in rows_s]
return tabulate(tr, headers=th, tablefmt=tablefmt)
def tablewidth(table_str: str) -> int:
return max((dispwidth(line) for line in table_str.splitlines()), default=0)
last = ""
for _ in range(max_iterations):
table_str = rendercurrent()
last = table_str
if tablewidth(table_str) <= target_width: return table_str
cur_w = [dispwidth(h if col_limit[j] is None else truncatebydispwidth(h, col_limit[j])) for j, h in enumerate(headers_s)]
any(cur_w.__setitem__(j, max(cur_w[j], dispwidth(cell if col_limit[j] is None else truncatebydispwidth(cell, col_limit[j])))) or False for r in rows_s for j, cell in enumerate(r))
shrinkable = [j for j in range(ncols) if col_limit[j] is not None and col_limit[j] > min_col_width]
if not shrinkable: return last
j_widest = max(shrinkable, key=lambda j: cur_w[j])
col_limit[j_widest] = max(min_col_width, int(col_limit[j_widest]) - 1)
return last
'''cursorpickintable'''
def cursorpickintable(headers: Sequence[Any], rows: Sequence[Sequence[Any]], row_ids: Sequence[Any], *, no_trunc_cols: NoTruncSpec = None, terminal_right_space_len: int = 2, normalize_ambiguous: Optional[bool] = None, tablefmt: Optional[str] = None) -> List[Any]:
if len(rows) != len(row_ids): raise ValueError("rows and row_ids length mismatch")
ncols = len(headers)
if any(len(r) != ncols for r in rows): raise ValueError("All rows must have same number of columns as headers")
if normalize_ambiguous is None: normalize_ambiguous = (os.name == "nt")
if tablefmt is None: tablefmt = "grid" if os.name == "nt" else "fancy_grid"
headers_s = [normalizeforconsole(h, enable=normalize_ambiguous) for h in headers]
rows_s = [[normalizeforconsole(c, enable=normalize_ambiguous) for c in r] for r in rows]
kb, current, picked, view_start = KeyBindings(), 0, set(), 0
FIRST_DATA_LINE, LINES_PER_ROW = 3, 2
def termsize() -> Tuple[int, int]: return ptsizefallback()
def maxvisiblerows(term_lines: int) -> int:
overhead = 10; usable = max(2, term_lines - overhead)
return max(1, usable // LINES_PER_ROW)
def computeview() -> Tuple[int, int]:
nonlocal view_start; _, term_lines = termsize()
page = maxvisiblerows(term_lines)
start = max(0, min(current - page // 2, len(rows_s) - page))
end, view_start = min(len(rows_s), start + page), start
return start, end
def buildtable() -> str:
cols, _ = termsize()
start, end = computeview()
def marker(i: int) -> str:
at, sel = (i == current), (row_ids[i] in picked)
if at and sel: return ">*"
if at: return "> "
if sel: return "* "
return " "
view_rows: List[List[str]] = []
for i in range(start, end): row = list(rows_s[i]); row[0] = marker(i) + row[0]; view_rows.append(row)
view_headers = list(headers_s)
view_headers[0] = f"{view_headers[0]} ({start+1}-{end}/{len(rows_s)})"
return smarttrunctable(headers=view_headers, rows=view_rows, no_trunc_cols=no_trunc_cols, terminal_right_space_len=terminal_right_space_len, term_width=cols, tablefmt=tablefmt)
def render() -> List[Tuple]:
cols, term_lines = termsize()
frags = to_formatted_text(ANSI(buildtable()))
highlight_line = FIRST_DATA_LINE + (current - view_start) * LINES_PER_ROW
out, line_count = [], 0
for li, line_frags in enumerate(split_lines(frags)):
if li == highlight_line: line_frags = [(((style + " reverse").strip() if style else "reverse"), text, *rest) for style, text, *rest in line_frags]
out.extend(truncateandpadline(line_frags, cols)); out.append(("", "\n")); line_count += 1
help_text = ("\nUse ↑/↓ to move, PgUp/PgDn to jump, <space> toggle, a: all, i: invert, <enter> confirm, q/Esc cancel.\n")
help_frags = to_formatted_text(ANSI(help_text))
for line_frags in split_lines(help_frags): out.extend(truncateandpadline(line_frags, cols)); out.append(("", "\n")); line_count += 1
while line_count < term_lines: out.append(("", " " * cols)); out.append(("", "\n")); line_count += 1
return out
def invalidate(event) -> None: event.app.invalidate()
@kb.add("up")
def _(event):
nonlocal current; current = max(0, current - 1)
invalidate(event)
@kb.add("down")
def _(event):
nonlocal current; current = min(len(rows_s) - 1, current + 1)
invalidate(event)
@kb.add("pageup")
def _(event):
nonlocal current; _, term_lines = termsize()
current = max(0, current - maxvisiblerows(term_lines))
invalidate(event)
@kb.add("pagedown")
def _(event):
nonlocal current; _, term_lines = termsize()
current = min(len(rows_s) - 1, current + maxvisiblerows(term_lines))
invalidate(event)
@kb.add(" ")
def _(event): rid = row_ids[current]; (picked.remove(rid) if rid in picked else picked.add(rid)); invalidate(event)
@kb.add("a")
@kb.add("A")
def _(event): picked.clear(); picked.update(row_ids); invalidate(event)
@kb.add("i")
@kb.add("I")
def _(event): picked.symmetric_difference_update(row_ids); invalidate(event)
@kb.add("enter")
def _(event): event.app.exit(result=[rid for rid in row_ids if rid in picked])
@kb.add("escape")
@kb.add("q")
def _(event): event.app.exit(result=[])
app = Application(layout=Layout(HSplit([Window(FormattedTextControl(render), wrap_lines=False)])), key_bindings=kb, full_screen=True)
return app.run()
catalog-sync/musicdl/modules/utils/lyric.py
+141
View File
@@ -0,0 +1,141 @@
'''
Function:
Implementation of Lyric Related Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import os
import re
import copy
import random
import tempfile
import requests
from typing import Optional
from .misc import resp2json
from urllib.parse import quote
from .importutils import optionalimportfrom
'''cleanlrc'''
cleanlrc = lambda text: "\n".join(line for raw in re.sub(r"\r\n?", "\n", str(text)).split("\n") if (line := raw.strip("\ufeff\u200b\u200c\u200d\u2060\u00a0 \t").strip()) and not re.fullmatch(r"\[(\d{2}:)?\d{2}:\d{2}(?:\.\d{1,3})?\]", line))
'''fractoseconds'''
def fractoseconds(frac: str | None) -> float:
if not frac: return 0.0
scale = 10 ** len(frac)
return int(frac) / scale
'''extractdurationsecondsfromlrc'''
def extractdurationsecondsfromlrc(lrc: str) -> Optional[float]:
if not lrc or (lrc == 'NULL'): return None
max_t, time_pattern_re = None, re.compile(r"\[(?:(\d{1,2}):)?(\d{1,2}):(\d{2})(?:\.(\d{1,3}))?\]")
for h, m, s, frac in time_pattern_re.findall(lrc):
hh = int(h) if h else 0; mm = int(m); ss = int(s)
t = hh * 3600 + mm * 60 + ss + fractoseconds(frac)
max_t = t if (max_t is None or t > max_t) else max_t
return max_t
'''WhisperLRC'''
class WhisperLRC:
def __init__(self, model_size_or_path="small", device="auto", compute_type="int8", cpu_threads=4, num_workers=1, **kwargs):
WhisperModel = optionalimportfrom('faster_whisper', 'WhisperModel')
self.whisper_model = WhisperModel(model_size_or_path, device=device, compute_type=compute_type, cpu_threads=cpu_threads, num_workers=num_workers, **kwargs) if WhisperModel else None
'''downloadtotmpdir'''
@staticmethod
def downloadtotmpdir(url: str, headers: dict = None, timeout: int = 300, cookies: dict = None, request_overrides: dict = None):
headers, cookies, request_overrides = headers or {}, cookies or {}, copy.deepcopy(request_overrides or {})
if 'headers' not in request_overrides: request_overrides['headers'] = headers
if 'timeout' not in request_overrides: request_overrides['timeout'] = timeout
if 'cookies' not in request_overrides: request_overrides['cookies'] = cookies
(resp := requests.get(url, stream=True, **request_overrides)).raise_for_status()
m = re.search(r"\.([a-z0-9]{2,5})(?:\?|$)", url, re.I)
fd, path = tempfile.mkstemp(suffix="."+(m.group(1).lower() if m else "bin"))
with os.fdopen(fd, "wb") as fp:
for ch in resp.iter_content(32768):
if ch: fp.write(ch)
return path
'''timestamp'''
@staticmethod
def timestamp(t):
t = max(0.0, float(t)); mm = int(t//60); ss = t - mm*60
return f"[{mm:02d}:{ss:05.2f}]"
'''fromurl'''
def fromurl(self, url: str, transcribe_overrides: dict = None, headers: dict = None, timeout: int = 300, cookies: dict = None, request_overrides: dict = None):
assert self.whisper_model is not None, 'faster_whisper should be installed via "pip install "faster_whisper"'
transcribe_overrides, headers, cookies, request_overrides, tmp_file_path = transcribe_overrides or {}, headers or {}, cookies or {}, request_overrides or {}, ''
try:
tmp_file_path = self.downloadtotmpdir(url, headers=headers, timeout=timeout, cookies=cookies, request_overrides=request_overrides)
(default_transcribe_settings := {'language': None, 'vad_filter': True, 'vad_parameters': dict(min_silence_duration_ms=300), 'chunk_length': 30, 'beam_size': 5}).update(transcribe_overrides)
segs, info = self.whisper_model.transcribe(tmp_file_path, **default_transcribe_settings)
lrc = "\n".join(f"{self.timestamp(s.start)}{s.text.strip()}" for s in segs)
result = {"language": info.language, "prob": info.language_probability, "duration": getattr(info, "duration", None), 'lyric': lrc}
return result
finally:
if tmp_file_path and os.path.exists(tmp_file_path): os.remove(tmp_file_path)
'''fromfilepath'''
def fromfilepath(self, file_path: str, transcribe_overrides: dict = None):
assert self.whisper_model is not None, 'faster_whisper should be installed via "pip install "faster_whisper"'
transcribe_overrides = transcribe_overrides or {}
default_transcribe_settings = {'language': None, 'vad_filter': True, 'vad_parameters': dict(min_silence_duration_ms=300), 'chunk_length': 30, 'beam_size': 5}
default_transcribe_settings.update(transcribe_overrides)
segs, info = self.whisper_model.transcribe(file_path, **default_transcribe_settings)
lrc = "\n".join(f"{self.timestamp(s.start)}{s.text.strip()}" for s in segs)
result = {"language": info.language, "prob": info.language_probability, "duration": getattr(info, "duration", None), 'lyric': lrc}
return result
'''LyricSearchClient'''
class LyricSearchClient():
'''search'''
@staticmethod
def search(track_name: str, artist_name: str, allowed_lyric_apis: tuple = ('searchbylrclibapig', 'searchbylrclibapis'), request_overrides: dict = None):
lyric_result, lyric = {}, 'NULL'
for lyric_api in allowed_lyric_apis:
if not callable(lyric_api): lyric_api = getattr(LyricSearchClient, lyric_api, None)
try: lyric_result, lyric = lyric_api(track_name=track_name, artist_name=artist_name, request_overrides=request_overrides)
except Exception: lyric_result, lyric = {}, 'NULL'
if lyric and (lyric not in {'NULL', 'None'}): return lyric_result, lyric
return lyric_result, lyric
'''searchbylrclibapig'''
@staticmethod
def searchbylrclibapig(track_name: str, artist_name: str, request_overrides: dict = None):
request_overrides = request_overrides or {}; headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"}
(resp := requests.get("https://lrclib.net/api/get", params={"artist_name": artist_name, "track_name": track_name}, headers=headers, timeout=10, **request_overrides)).raise_for_status()
lyric = cleanlrc((lyric_result := resp2json(resp=resp)).get('syncedLyrics') or lyric_result.get('plainLyrics') or 'NULL')
return lyric_result, lyric
'''searchbylrclibapis'''
@staticmethod
def searchbylrclibapis(track_name: str, artist_name: str, request_overrides: dict = None):
request_overrides = request_overrides or {}; headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"}
(resp := requests.get("https://lrclib.net/api/search", params={"q": f"{artist_name} {track_name}"}, headers=headers, timeout=10, **request_overrides)).raise_for_status()
lyric = cleanlrc((lyric_result := resp2json(resp=resp))[0].get('syncedLyrics') or lyric_result[0].get('plainLyrics') or 'NULL')
return lyric_result, lyric
'''searchbylyricsovhapiv1'''
@staticmethod
def searchbylyricsovhapiv1(track_name: str, artist_name: str, request_overrides: dict = None):
request_overrides = request_overrides or {}; headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"}
(resp := requests.get(f"https://api.lyrics.ovh/v1/{quote(artist_name, safe='')}/{quote(track_name, safe='')}", headers=headers, timeout=10, **request_overrides))
lyric = cleanlrc((lyric_result := resp2json(resp=resp)).get('lyrics') or 'NULL')
return lyric_result, lyric
'''searchbyhappiapiv1'''
@staticmethod
def searchbyhappiapiv1(track_name: str, artist_name: str, request_overrides: dict = None):
request_overrides = request_overrides or {}; headers = {'accept': 'application/json', 'x-happi-token': 'hk254-C1VegxwlJjYdYFPtdUDpg8qiVpmAXVl0aA'}
(resp := requests.get('https://api.happi.dev/v1/lyrics', params={'artist': artist_name, 'track': track_name}, headers=headers, timeout=10, **request_overrides))
lyric = cleanlrc((lyric_result := resp2json(resp=resp))['result'][0]['lyrics'] or 'NULL')
return lyric_result, lyric
'''searchbymusixmatchapi'''
@staticmethod
def searchbymusixmatchapi(track_name: str, artist_name: str, request_overrides: dict = None):
candidate_req_keys = ['3bc1042fde1ac8c1979c400d6f921320']
request_overrides = request_overrides or {}; headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"}
(resp := requests.get(f"https://api.musixmatch.com/ws/1.1/matcher.lyrics.get?apikey={random.choice(candidate_req_keys)}&q_track={track_name}&q_artist={artist_name}", headers=headers, timeout=10, **request_overrides))
lyric = cleanlrc((lyric_result := resp2json(resp=resp))['message']['body']['lyrics']['lyrics_body'] or 'NULL')
return lyric_result, lyric
catalog-sync/musicdl/modules/utils/misc.py
+394
View File
@@ -0,0 +1,394 @@
'''
Function:
Implementation of Common Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import re
import os
import html
import copy
import emoji
import errno
import pickle
import shutil
import bleach
import hashlib
import requests
import functools
import json_repair
import unicodedata
from io import BytesIO
from pathlib import Path
from mutagen.mp3 import MP3
from mutagen.mp4 import MP4
from mutagen.asf import ASF
from mutagen.flac import FLAC
from mutagen.aiff import AIFF
from mutagen.wave import WAVE
from bs4 import BeautifulSoup
from http.cookies import SimpleCookie
from .importutils import optionalimport
from mutagen import File as MutagenFile
from mutagen.oggvorbis import OggVorbis
from pathvalidate import sanitize_filepath, sanitize_filename
def remove_suffix(value: str, suffix: str) -> str:
if suffix and value.endswith(suffix):
return value[: -len(suffix)]
return value
'''estimatedurationwithfilesizebr'''
def estimatedurationwithfilesizebr(file_size_bytes: int, br_kbps: float, return_seconds: bool = False) -> str:
if not file_size_bytes or not br_kbps or br_kbps <= 0: return "-:-:-"
total_bits = file_size_bytes * 8
duration_seconds = int(total_bits / (br_kbps * 1000))
if return_seconds: return duration_seconds
hours = duration_seconds // 3600
minutes = (duration_seconds % 3600) // 60
seconds = duration_seconds % 60
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
'''estimatedurationwithfilelink'''
def estimatedurationwithfilelink(filelink: str = '', headers: dict = None, request_overrides: dict = None):
headers, request_overrides = headers or {}, request_overrides or {}
try:
(resp := requests.get(filelink, headers=headers, timeout=10, **request_overrides)).raise_for_status()
audio = MutagenFile(BytesIO(resp.content))
length = getattr(audio.info, "length", 0)
return int(length)
except:
return 0
'''cookies2dict'''
def cookies2dict(cookies: str | dict = None):
if not cookies: cookies = {}
if isinstance(cookies, dict): return cookies
if isinstance(cookies, str): (c := SimpleCookie()).load(cookies); return {k: morsel.value for k, morsel in c.items()}
raise TypeError(f'cookies type is "{type(cookies)}", expect cookies to "str" or "dict" or "None".')
'''cookies2string'''
def cookies2string(cookies: str | dict = None):
if not cookies: cookies = ""
if isinstance(cookies, str): return cookies
if isinstance(cookies, dict): return (lambda c: ([c.__setitem__(k, "" if v is None else str(v)) for k, v in cookies.items()], "; ".join(m.OutputString() for m in c.values()))[1])(SimpleCookie())
raise TypeError(f'cookies type is "{type(cookies)}", expect cookies to "str" or "dict" or "None".')
'''touchdir'''
def touchdir(directory, exist_ok=True, mode=511, auto_sanitize=True):
if auto_sanitize: directory = sanitize_filepath(directory)
return os.makedirs(directory, exist_ok=exist_ok, mode=mode)
'''replacefile'''
def replacefile(src: str, dest: str):
try:
os.replace(src, dest)
except OSError as exc:
if exc.errno != errno.EXDEV: raise Exception
if os.path.exists(dest):
if os.path.isdir(dest): raise Exception
os.remove(dest)
shutil.move(src, dest)
'''legalizestring'''
def legalizestring(string: str, fit_gbk: bool = True, max_len: int = 255, fit_utf8: bool = True, replace_null_string: str = 'NULL'):
if not string: return replace_null_string
string = str(string)
string = string.replace(r'\"', '"')
string = re.sub(r"<\\/", "</", string)
string = re.sub(r"\\/>", "/>", string)
string = re.sub(r"\\u([0-9a-fA-F]{4})", lambda m: chr(int(m.group(1), 16)), string)
# html.unescape
for _ in range(2):
new_string = html.unescape(string)
if new_string == string: break
string = new_string
# bleach.clean
try: string = BeautifulSoup(string, "lxml").get_text(separator="")
except: string = bleach.clean(string, tags=[], attributes={}, strip=True)
# unicodedata.normalize
string = unicodedata.normalize("NFC", string)
# emoji.replace_emoji
string = emoji.replace_emoji(string, replace="")
# isprintable
string = "".join([ch for ch in string if ch.isprintable() and not unicodedata.category(ch).startswith("C")])
# sanitize_filename
string = sanitize_filename(string, max_len=max_len)
# fix encoding
if fit_gbk: string = string.encode("gbk", errors="ignore").decode("gbk", errors="ignore")
if fit_utf8: string = string.encode("utf-8", errors="ignore").decode("utf-8", errors="ignore")
# return
string = re.sub(r"\s+", " ", string).strip()
if not string: string = replace_null_string
return string
'''shortenpathsinsonginfos'''
def shortenpathsinsonginfos(song_infos: list, max_path: int = 240, keep_ext: bool = True, with_hash_suffix: bool = False):
used_paths = set()
for info in song_infos:
raw_path = (info.save_path or "").strip()
if not raw_path or raw_path.upper() == "NULL": continue
src_path = Path(raw_path); output_dir = src_path.parent.resolve(); output_dir.mkdir(parents=True, exist_ok=True)
ext = src_path.suffix if keep_ext else ""; stem = src_path.stem
digest = hashlib.md5(str(src_path).encode("utf-8")).hexdigest()
for hash_len in (8, 10):
hash_suffix = f"-{digest[:hash_len]}" if with_hash_suffix else ""
max_stem_len = max(1, max_path - (len(str(output_dir)) + 1 + len(hash_suffix) + len(ext)))
safe_stem = (stem[:max_stem_len].rstrip(" .") or "NULL")
out_path = str(output_dir / f"{safe_stem}{hash_suffix}{ext}")
if out_path.lower() not in used_paths: break
used_paths.add(out_path.lower()); info._save_path = out_path
return song_infos
'''seconds2hms'''
def seconds2hms(seconds: int):
try:
seconds = int(float(seconds))
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
hms = '%02d:%02d:%02d' % (h, m, s)
if hms == '00:00:00': hms = '-:-:-'
except:
hms = '-:-:-'
return hms
'''byte2mb'''
def byte2mb(size: int):
try:
size = int(float(size))
if size == 0: return 'NULL'
size = round(size / 1024 / 1024, 2)
if size == 0.0: return 'NULL'
size = f'{size} MB'
except:
size = 'NULL'
return size
'''resp2json'''
def _valid_response_types():
response_types = [requests.Response]
curl_cffi = optionalimport('curl_cffi')
curl_requests = getattr(curl_cffi, 'requests', None) if curl_cffi else None
curl_response = getattr(curl_requests, 'Response', None) if curl_requests else None
if curl_response is not None:
response_types.append(curl_response)
return tuple(response_types)
'''resp2json'''
def resp2json(resp: requests.Response):
valid_resp_object = _valid_response_types()
if not isinstance(resp, valid_resp_object): return {}
try: result = resp.json()
except: result = json_repair.loads(resp.text)
if not result: result = dict()
return result
'''isvalidresp'''
def isvalidresp(resp: requests.Response, valid_status_codes: list | tuple | set = {200, 206}):
valid_resp_object = _valid_response_types()
if not isinstance(resp, valid_resp_object): return False
if resp is None or resp.status_code not in valid_status_codes: return False
return True
'''safeextractfromdict'''
def safeextractfromdict(data, progressive_keys, default_value = None):
try:
result = data
for key in progressive_keys: result = result[key]
except:
result = default_value
return result
'''cachecookies'''
def cachecookies(client_name: str = '', cache_cookie_path: str = '', client_cookies: dict = None):
if os.path.exists(cache_cookie_path):
with open(cache_cookie_path, 'rb') as fp: cookies = pickle.load(fp)
else:
cookies = dict()
with open(cache_cookie_path, 'wb') as fp:
cookies[client_name] = client_cookies
pickle.dump(cookies, fp)
'''usedownloadheaderscookies'''
def usedownloadheaderscookies(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
self.default_headers = self.default_download_headers
if hasattr(self, 'default_download_cookies'): self.default_cookies = self.default_download_cookies
if hasattr(self, 'enable_download_curl_cffi'): self.enable_curl_cffi = self.enable_download_curl_cffi
if hasattr(self, '_initsession'): self._initsession()
return func(self, *args, **kwargs)
return wrapper
'''useparseheaderscookies'''
def useparseheaderscookies(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
self.default_headers = self.default_parse_headers
if hasattr(self, 'default_parse_cookies'): self.default_cookies = self.default_parse_cookies
if hasattr(self, 'enable_parse_curl_cffi'): self.enable_curl_cffi = self.enable_parse_curl_cffi
if hasattr(self, '_initsession'): self._initsession()
return func(self, *args, **kwargs)
return wrapper
'''usesearchheaderscookies'''
def usesearchheaderscookies(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
self.default_headers = self.default_search_headers
if hasattr(self, 'default_search_cookies'): self.default_cookies = self.default_search_cookies
if hasattr(self, 'enable_search_curl_cffi'): self.enable_curl_cffi = self.enable_search_curl_cffi
if hasattr(self, '_initsession'): self._initsession()
return func(self, *args, **kwargs)
return wrapper
'''searchdictbykey'''
def searchdictbykey(obj, target_key: str):
results = []
if isinstance(obj, dict):
for k, v in obj.items():
if k == target_key: results.append(v)
results.extend(searchdictbykey(v, target_key))
elif isinstance(obj, list):
for item in obj: results.extend(searchdictbykey(item, target_key))
return results
'''naiveguessextfromaudiobytes'''
def naiveguessextfromaudiobytes(content: bytes):
if (audio := MutagenFile(BytesIO(content))) is None: return None
if isinstance(audio, MP3): return "mp3"
if isinstance(audio, FLAC): return "flac"
if isinstance(audio, MP4): return "m4a"
if isinstance(audio, OggVorbis): return "ogg"
if isinstance(audio, WAVE): return "wav"
if isinstance(audio, AIFF): return "aiff"
if isinstance(audio, ASF): return "wma"
return None
'''AudioLinkTester'''
class AudioLinkTester(object):
VALID_AUDIO_EXTS = {
"aac", "aax", "aaxc", "ac3", "adts", "aif", "aifc", "aiff", "alac", "amr", "ape", "au", "avr", "awb", "caf", "cda", "dff", "dfsf", "dsf", "dss", "dts", "dtshd", "ec3", "f32",
"f64", "flac", "gsm", "hca", "htk", "iff", "ima", "ircam", "kar", "kss", "la", "l16", "m15", "m3u8", "m4a", "m4b", "m4p", "m4r", "mat4", "mat5", "med", "midi", "mid", "mlp",
"mod", "mo3", "mp1", "mp2", "mp3", "mpa", "mpc", "mp+", "mpp", "mptm", "msv", "mt2", "mtm", "mxmf", "nist", "nsf", "oga", "ogg", "okt", "oma", "ofr", "ofs", "opus", "paf",
"pcm", "ptm", "pvf", "ra", "ram", "rf64", "rmi", "rmj", "rmm", "rmx", "roq", "raw", "s3m", "sap", "sds", "sd2", "sd2f", "sf", "shn", "sid", "snd", "spc", "spx", "stm", "tak",
"tta", "thd", "ul", "ult", "umx", "voc", "vgm", "vgz", "wav", "wave", "wax", "w64", "wma", "wve", "wv", "wvx", "xi", "xm", "8svx", "16svx", "669", "amf", "dmf", "far", "gbs",
"gym", "hes", "it", "mdl", "mpc2k", "nsa", "psf", "psf1", "psf2", "ssf", "miniusf", "usf", "2sf", "gsf", "qsf", "spu", "at3", "aa3", "at9", "3ga", "m4s"
}
AUDIO_CT_PREFIX = "audio/"
AUDIO_CT_EXTRA = {"application/octet-stream", "application/x-flac", "application/flac", "application/x-mpegurl", "video/mp4"}
MAGIC = [(b"ID3", "mp3"), (b"\xFF\xFB", "mp3"), (b"fLaC", "flac"), (b"RIFF", "wav"), (b"OggS", "ogg"), (b"MThd", "midi"), (b"\x00\x00\x00\x18ftyp", "mp4/m4a")]
CTYPE_TO_EXT = {"audio/mpeg": "mp3", "audio/mp3": "mp3", "audio/mp4": "m4a", "audio/x-m4a": "m4a", "audio/aac": "aac", "audio/wav": "wav", "video/mp4": "mp4", "audio/x-wav": "wav", "audio/flac": "flac", "audio/x-flac": "flac", "audio/ogg": "ogg", "audio/opus": "opus", "audio/x-aac": "ogg", "audio/x-ogg": "ogg", "audio/x-m4p": "m4a"}
def __init__(self, timeout=(5, 15), headers: dict = None, cookies: dict = None):
self.session = requests.Session()
self.timeout = timeout
self.headers = {'Accept': '*/*', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36'}
self.headers.update(headers or {})
self.cookies = cookies or {}
'''isaudioct'''
@staticmethod
def isaudioct(ct: str):
if not ct: return False
ct = ct.lower().split(";", 1)[0].strip()
return ct.startswith(AudioLinkTester.AUDIO_CT_PREFIX) or ct in AudioLinkTester.AUDIO_CT_EXTRA
'''sniffmagic'''
@staticmethod
def sniffmagic(b: str):
for sig, fmt in AudioLinkTester.MAGIC:
if b.startswith(sig): return fmt
if len(b) >= 2 and b[0] == 0xFF and (b[1] & 0xF0) == 0xF0: return "aac/adts"
return None
'''probe'''
def probe(self, url: str, request_overrides: dict = None):
request_overrides, naive_guess_ext = copy.deepcopy(request_overrides or {}), url.split('?')[0].split('.')[-1]
if 'headers' not in request_overrides: request_overrides['headers'] = self.headers
if 'timeout' not in request_overrides: request_overrides['timeout'] = self.timeout
if 'cookies' not in request_overrides: request_overrides['cookies'] = self.cookies
outputs = dict(file_size='NULL', ctype='NULL', ext='NULL', download_url=url, final_url='NULL')
# HEAD probe
try:
(resp := self.session.head(url, allow_redirects=True, **request_overrides)).raise_for_status()
resp_headers, final_url = resp.headers, resp.url; resp.close()
file_size, ctype = byte2mb(resp_headers.get('content-length')), remove_suffix(str(resp_headers.get('content-type')), '; charset=UTF-8')
if ctype == 'image/jpg; charset=UTF-8' or ctype == 'image/jpg': ctype = 'audio/mpeg'
if ctype == 'text/plain' and naive_guess_ext == 'm4s': ctype = 'audio/mp4'
ext = self.CTYPE_TO_EXT.get(ctype, 'NULL')
outputs = dict(file_size=file_size, ctype=ctype, ext=ext, download_url=url, final_url=final_url)
except:
outputs = dict(file_size='NULL', ctype='NULL', ext='NULL', download_url=url, final_url='NULL')
if outputs['file_size'] and outputs['file_size'] not in ('NULL',): return outputs
# GETSTREAM probe
try:
(resp := self.session.get(url, allow_redirects=True, stream=True, **request_overrides)).raise_for_status()
resp_headers, final_url = resp.headers, resp.url; resp.close()
file_size, ctype = byte2mb(resp_headers.get('content-length')), remove_suffix(str(resp_headers.get('content-type')), '; charset=UTF-8')
if ctype == 'image/jpg; charset=UTF-8' or ctype == 'image/jpg': ctype = 'audio/mpeg'
if ctype == 'text/plain' and naive_guess_ext == 'm4s': ctype = 'audio/mp4'
ext = self.CTYPE_TO_EXT.get(ctype, 'NULL')
outputs = dict(file_size=file_size, ctype=ctype, ext=ext, download_url=url, final_url=final_url)
except:
outputs = dict(file_size='NULL', ctype='NULL', ext='NULL', download_url=url, final_url='NULL')
return outputs
'''test'''
def test(self, url: str, request_overrides: dict = None):
request_overrides, naive_guess_ext = copy.deepcopy(request_overrides or {}), url.split('?')[0].split('.')[-1]
if 'headers' not in request_overrides: request_overrides['headers'] = self.headers
if 'timeout' not in request_overrides: request_overrides['timeout'] = self.timeout
if 'cookies' not in request_overrides: request_overrides['cookies'] = self.cookies
outputs = dict(ok=False, status=0, method="", final_url=None, ctype=None, clen=None, range=None, fmt=None, reason="")
# HEAD test
try:
resp = self.session.head(url, allow_redirects=True, **request_overrides)
clen = resp.headers.get("Content-Length")
clen = int(clen) if clen and clen.isdigit() else None
outputs.update(dict(status=resp.status_code, method="HEAD", final_url=str(resp.url), ctype=resp.headers.get("Content-Type"), clen=clen, range=(resp.headers.get("Accept-Ranges") or "").lower() == "bytes"))
if outputs["ctype"] == 'text/plain' and naive_guess_ext == 'm4s': outputs["ctype"] = 'audio/mp4'
if 200 <= resp.status_code < 300 and ((self.isaudioct(outputs["ctype"]) or (naive_guess_ext in ('m4s',))) and (outputs["clen"] or outputs["range"])): outputs.update(dict(ok=True, reason="HEAD success")); return outputs
except Exception as err:
outputs["reason"] = f"HEAD error: {err}"
# RANGEGET test
try:
resp = self.session.get(url, stream=True, allow_redirects=True, **request_overrides)
outputs.update(dict(status=resp.status_code, method="RANGEGET", final_url=str(resp.url)))
if resp.status_code not in (200, 206): outputs["reason"] = f"RANGEGET error: response status {resp.status_code}"; return outputs
chunk = b""
for b in resp.iter_content(chunk_size=16): chunk = b; break
resp.close()
outputs["ctype"] = outputs["ctype"] or resp.headers.get("Content-Type")
if outputs["ctype"] == 'text/plain' and naive_guess_ext == 'm4s': outputs["ctype"] = 'audio/mp4'
outputs["range"] = outputs["range"] or (resp.status_code == 206) or (resp.headers.get("Content-Range") is not None)
clen = resp.headers.get("Content-Length") or (resp.headers.get("Content-Range") or "").split("/")[-1]
if clen and clen.isdigit(): outputs["clen"] = int(clen)
outputs["fmt"] = self.sniffmagic(chunk)
if self.isaudioct(outputs["ctype"]) or outputs["fmt"] or (naive_guess_ext in ('m4s',)): outputs.update(dict(ok=True, reason="RANGEGET success"))
else: outputs.update(dict(ok=False, reason="RANGEGET error: Not audio-like (CT/magic)"))
except Exception as err:
outputs["reason"] = f"RANGEGET error: {err}"
# return
return outputs
catalog-sync/musicdl/modules/utils/modulebuilder.py
+73
View File
@@ -0,0 +1,73 @@
'''
Function:
Implementation of BaseModuleBuilder
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import copy
import collections
'''BaseModuleBuilder'''
class BaseModuleBuilder():
REGISTERED_MODULES = collections.OrderedDict()
def __init__(self, requires_register_modules=None, requires_renew_modules=None):
if requires_register_modules is not None and isinstance(requires_register_modules, (dict, collections.OrderedDict)):
for name, module in requires_register_modules.items(): self.register(name, module)
if requires_renew_modules is not None and isinstance(requires_renew_modules, (dict, collections.OrderedDict)):
for name, module in requires_renew_modules.items(): self.renew(name, module)
self.validate()
'''build'''
def build(self, module_cfg):
module_cfg = copy.deepcopy(module_cfg)
module_type = module_cfg.pop('type')
module = self.REGISTERED_MODULES[module_type](**module_cfg)
return module
'''register'''
def register(self, name, module):
assert callable(module)
assert name not in self.REGISTERED_MODULES
self.REGISTERED_MODULES[name] = module
'''renew'''
def renew(self, name, module):
assert callable(module)
assert name in self.REGISTERED_MODULES
self.REGISTERED_MODULES[name] = module
'''validate'''
def validate(self):
for _, module in self.REGISTERED_MODULES.items():
assert callable(module)
'''delete'''
def delete(self, name):
assert name in self.REGISTERED_MODULES
del self.REGISTERED_MODULES[name]
'''pop'''
def pop(self, name):
assert name in self.REGISTERED_MODULES
module = self.REGISTERED_MODULES.pop(name)
return module
'''get'''
def get(self, name):
assert name in self.REGISTERED_MODULES
module = self.REGISTERED_MODULES.get(name)
return module
'''items'''
def items(self):
return self.REGISTERED_MODULES.items()
'''clear'''
def clear(self):
return self.REGISTERED_MODULES.clear()
'''values'''
def values(self):
return self.REGISTERED_MODULES.values()
'''keys'''
def keys(self):
return self.REGISTERED_MODULES.keys()
'''copy'''
def copy(self):
return self.REGISTERED_MODULES.copy()
'''update'''
def update(self, requires_update_modules):
return self.REGISTERED_MODULES.update(requires_update_modules)
catalog-sync/musicdl/modules/utils/neteaseutils.py
+86
View File
@@ -0,0 +1,86 @@
'''
Function:
Implementation of NeteaseMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import os
import json
import base64
import urllib
import codecs
import urllib.parse
from hashlib import md5
from Crypto.Cipher import AES
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
'''settings'''
MUSIC_QUALITIES = ['jymaster', 'dolby', 'sky', 'jyeffect', 'hires', 'lossless', 'exhigh', 'standard']
DEFAULT_COOKIES = {'MUSIC_U': '1eb9ce22024bb666e99b6743b2222f29ef64a9e88fda0fd5754714b900a5d70d993166e004087dd3b95085f6a85b059f5e9aba41e3f2646e3cebdbec0317df58c119e5'}
'''EapiCryptoUtils'''
class EapiCryptoUtils(object):
'''hexdigest'''
@staticmethod
def hexdigest(data: bytes):
return "".join([hex(d)[2:].zfill(2) for d in data])
'''hashdigest'''
@staticmethod
def hashdigest(text: str):
return md5(text.encode("utf-8")).digest()
'''hashhexdigest'''
@staticmethod
def hashhexdigest(text: str):
return EapiCryptoUtils.hexdigest(EapiCryptoUtils.hashdigest(text))
'''encryptparams'''
@staticmethod
def encryptparams(url: str, payload: dict, aes_key: bytes = b"e82ckenh8dichen8"):
url_path = urllib.parse.urlparse(url).path.replace("/eapi/", "/api/")
digest = EapiCryptoUtils.hashhexdigest(f"nobody{url_path}use{json.dumps(payload)}md5forencrypt")
params = f"{url_path}-36cd479b6b5-{json.dumps(payload)}-36cd479b6b5-{digest}"
padder = padding.PKCS7(algorithms.AES(aes_key).block_size).padder()
padded_data = padder.update(params.encode()) + padder.finalize()
cipher = Cipher(algorithms.AES(aes_key), modes.ECB())
encryptor = cipher.encryptor()
enc = encryptor.update(padded_data) + encryptor.finalize()
return EapiCryptoUtils.hexdigest(enc)
'''WeapiCryptoUtils'''
class WeapiCryptoUtils(object):
'''createsecretkey'''
@staticmethod
def createsecretkey(size: int):
return (''.join(map(lambda xx: (hex(ord(xx))[2:]), str(os.urandom(size)))))[0: 16]
'''aesencrypt'''
@staticmethod
def aesencrypt(string: str, sec_key: str):
pad = 16 - len(string) % 16
if isinstance(string, bytes): string = string.decode('utf-8')
string = string + str(pad * chr(pad))
sec_key = sec_key.encode('utf-8')
encryptor = AES.new(sec_key, 2, b'0102030405060708')
string = string.encode('utf-8')
ciphertext = encryptor.encrypt(string)
ciphertext = base64.b64encode(ciphertext)
return ciphertext
'''rsaencrypt'''
@staticmethod
def rsaencrypt(string: str, pub_key: str = '010001', modulus: str = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'):
string = string[::-1]
rs = int(codecs.encode(string.encode('utf-8'), 'hex_codec'), 16) ** int(pub_key, 16) % int(modulus, 16)
return format(rs, 'x').zfill(256)
'''encryptparams'''
@staticmethod
def encryptparams(params: dict):
string = json.dumps(params)
sec_key = WeapiCryptoUtils.createsecretkey(16)
enc_string = WeapiCryptoUtils.aesencrypt(string=WeapiCryptoUtils.aesencrypt(string=string, sec_key='0CoJUm6Qyw8W8jud'), sec_key=sec_key)
enc_sec_key = WeapiCryptoUtils.rsaencrypt(string=sec_key)
post_data = {'params': enc_string, 'encSecKey': enc_sec_key}
return post_data
catalog-sync/musicdl/modules/utils/qqutils.py
+323
View File
@@ -0,0 +1,323 @@
'''
Function:
Implementation of QQMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import re
import time
import orjson
import base64
import random
import string
import hashlib
import requests
import binascii
from enum import Enum
from uuid import uuid4
from datetime import datetime, timedelta
from dataclasses import dataclass, field, asdict
from typing import ClassVar, TypedDict, Any, cast
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
'''settings'''
PUBLIC_KEY = """-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDEIxgwoutfwoJxcGQeedgP7FG9qaIuS0qzfR8gWkrkTZKM2iWHn2ajQpBRZjMSoSf6+KJGvar2ORhBfpDXyVtZCKpqLQ+FLkpncClKVIrBwv6PHyUvuCb0rIarmgDnzkfQAqVufEtR64iazGDKatvJ9y6B9NMbHddGSAUmRTCrHQIDAQAB
-----END PUBLIC KEY-----"""
SECRET = "ZdJqM15EeO2zWc08"
APP_KEY = "0AND0HD6FE4HY80F"
'''SongFileType'''
class SongFileType(Enum):
MASTER = ("AI00", ".flac")
ATMOS_2 = ("Q000", ".flac")
ATMOS_51 = ("Q001", ".flac")
FLAC = ("F000", ".flac")
OGG_640 = ("O801", ".ogg")
OGG_320 = ("O800", ".ogg")
OGG_192 = ("O600", ".ogg")
OGG_96 = ("O400", ".ogg")
MP3_320 = ("M800", ".mp3")
MP3_128 = ("M500", ".mp3")
ACC_192 = ("C600", ".m4a")
ACC_96 = ("C400", ".m4a")
ACC_48 = ("C200", ".m4a")
SORTED_QUALITIES = [
("AI00", ".flac"), ("Q000", ".flac"), ("Q001", ".flac"), ("F000", ".flac"), ("O801", ".ogg"), ("O800", ".ogg"), ("O600", ".ogg"), ("O400", ".ogg"),
("M800", ".mp3"), ("M500", ".mp3"), ("C600", ".m4a"), ("C400", ".m4a"), ("C200", ".m4a")
]
'''EncryptedSongFileType'''
class EncryptedSongFileType(Enum):
MASTER = ("AIM0", ".mflac")
ATMOS_2 = ("Q0M0", ".mflac")
ATMOS_51 = ("Q0M1", ".mflac")
FLAC = ("F0M0", ".mflac")
OGG_640 = ("O801", ".mgg")
OGG_320 = ("O800", ".mgg")
OGG_192 = ("O6M0", ".mgg")
OGG_96 = ("O4M0", ".mgg")
SORTED_QUALITIES = [
("AIM0", ".mflac"), ("Q0M0", ".mflac"), ("Q0M1", ".mflac"), ("F0M0", ".mflac"), ("O801", ".mgg"), ("O800", ".mgg"), ("O6M0", ".mgg"), ("O4M0", ".mgg")
]
'''ThirdPartVKeysAPISongFileType'''
class ThirdPartVKeysAPISongFileType(Enum):
TRIAL_LISTEN = (0,)
LOSSY_QUALITY = (1, 2, 3)
STANDARD_QUALITY = (4, 5, 6, 7)
HQ_QUALITY = (8,)
HQ_QUALITY_ENHANCED = (9,)
SQ_LOSSLESS_QUALITY = (10,)
HI_RES_QUALITY = (11,)
DOLBY_ATMOS = (12,)
PREMIUM_SPATIAL_AUDIO = (13,)
PREMIUM_MASTER_2_0 = (14,)
AI_ACCOMPANIMENT_MODE_4TRACK = (15,)
AI_5_1_QUALITY_6TRACK = (16,)
ID_TO_NAME = {
0: "TRIAL_LISTEN", 1: "LOSSY_QUALITY", 2: "LOSSY_QUALITY", 3: "LOSSY_QUALITY", 4: "STANDARD_QUALITY", 5: "STANDARD_QUALITY", 6: "STANDARD_QUALITY", 7: "STANDARD_QUALITY",
8: "HQ_QUALITY", 9: "HQ_QUALITY_ENHANCED", 10: "SQ_LOSSLESS_QUALITY", 11: "HI_RES_QUALITY", 12: "DOLBY_ATMOS", 13: "PREMIUM_SPATIAL_AUDIO", 14: "PREMIUM_MASTER_2_0",
15: "AI_ACCOMPANIMENT_MODE_4TRACK", 16: "AI_5_1_QUALITY_6TRACK",
}
'''SearchType'''
class SearchType(Enum):
SONG = 0
SINGER = 1
ALBUM = 2
SONGLIST = 3
MV = 4
LYRIC = 7
USER = 8
AUDIO_ALBUM = 15
AUDIO = 18
'''QimeiResult'''
class QimeiResult(TypedDict):
q16: str
q36: str
'''OSVersion'''
@dataclass
class OSVersion:
incremental: str = "5891938"
release: str = "10"
codename: str = "REL"
sdk: int = 29
'''Device'''
@dataclass
class Device:
display: str = field(default_factory=lambda: f"QMAPI.{random.randint(100000, 999999)}.001")
product: str = "iarim"
device: str = "sagit"
board: str = "eomam"
model: str = "MI 6"
fingerprint: str = field(default_factory=lambda: f"xiaomi/iarim/sagit:10/eomam.200122.001/{random.randint(1000000, 9999999)}:user/release-keys")
boot_id: str = field(default_factory=lambda: str(uuid4()))
proc_version: str = field(default_factory=lambda: f"Linux 5.4.0-54-generic-{''.join(random.choices(string.ascii_letters + string.digits, k=8))} (android-build@google.com)")
imei: str = field(default_factory=lambda: (lambda d: "".join(map(str, d)) + str(sum((x * 2 // 10 + x * 2 % 10) if i % 2 == 0 else x for i, x in enumerate(d)) * 9 % 10))([random.randint(0, 9) for _ in range(14)]))
brand: str = "Xiaomi"
bootloader: str = "U-boot"
base_band: str = ""
version: OSVersion = field(default_factory=OSVersion)
sim_info: str = "T-Mobile"
os_type: str = "android"
mac_address: str = "00:50:56:C0:00:08"
ip_address: ClassVar[list[int]] = [10, 0, 1, 3]
wifi_bssid: str = "00:50:56:C0:00:08"
wifi_ssid: str = "<unknown ssid>"
imsi_md5: list[int] = field(default_factory=lambda: list(hashlib.md5(bytes([random.randint(0, 255) for _ in range(16)])).digest()))
android_id: str = field(default_factory=lambda: binascii.hexlify(bytes([random.randint(0, 255) for _ in range(8)])).decode("utf-8"))
apn: str = "wifi"
vendor_name: str = "MIUI"
vendor_os_name: str = "qmapi"
qimei: None | str = None
'''Credential'''
@dataclass
class Credential:
openid: str = ""
refresh_token: str = ""
access_token: str = ""
expired_at: int = 0
musicid: int = 0
musickey: str = ""
unionid: str = ""
str_musicid: str = ""
refresh_key: str = ""
encrypt_uin: str = ""
login_type: int = 0
extra_fields: dict[str, Any] = field(default_factory=dict)
'''postinit'''
def __post_init__(self):
if not self.login_type: self.login_type = 1 if self.musickey and self.musickey.startswith("W_X") else 2
'''todict'''
def todict(self) -> dict:
d = asdict(self)
d["loginType"], d["encryptUin"] = d.pop("login_type"), d.pop("encrypt_uin")
return d
'''asjson'''
def asjson(self) -> str:
data = self.todict()
data.update(data.pop("extra_fields"))
return orjson.dumps(data).decode()
'''fromcookiesdict'''
@classmethod
def fromcookiesdict(cls, cookies: dict[str, Any]):
return cls(
openid=cookies.get("openid") or cookies.get("psrf_qqopenid") or cookies.get("wxopenid"), refresh_token=cookies.get("refresh_token") or cookies.get("psrf_qqrefresh_token") or cookies.get("wxrefresh_token"),
access_token=cookies.get("access_token") or cookies.get("psrf_qqaccess_token") or cookies.get("wxaccess_token"), expired_at=cookies.get("expired_at") or cookies.get("psrf_access_token_expiresAt"), extra_fields=cookies,
musicid=int(cookies.get("musicid", 0) or cookies.get("uin", 0)), musickey=cookies.get("musickey") or cookies.get("qqmusic_key"), unionid=cookies.get("unionid") or cookies.get("psrf_qqunionid") or cookies.get("wxunionid"),
str_musicid=cookies.get("str_musicid") or cookies.get("musicid") or cookies.get("uin"), refresh_key=cookies.get("refresh_key"), encrypt_uin=cookies.get("encryptUin"), login_type=cookies.get("loginType") or cookies.get("tmeLoginType"),
)
'''QQMusicClientUtils'''
class QQMusicClientUtils(object):
version, version_code, qimei_result, device = "13.2.5.8", 13020508, {}, Device()
endpoint = "https://u.y.qq.com/cgi-bin/musicu.fcg"
enc_endpoint = "https://u.y.qq.com/cgi-bin/musics.fcg"
music_domain = "https://isure.stream.qqmusic.qq.com/"
COMMON_DEFAULTS: ClassVar[dict[str, str]] = {"ct": "11", "tmeAppID": "qqmusic", "format": "json", "inCharset": "utf-8", "outCharset": "utf-8", "uid": "3931641530"}
@property
def qimei(self) -> QimeiResult:
if self.qimei_result: return self.qimei_result
self.qimei_result = QQMusicClientUtils.obtainqimei(version=QQMusicClientUtils.version, device=QQMusicClientUtils.device)
return self.qimei_result
'''rsaencrypt'''
@staticmethod
def rsaencrypt(content: bytes):
key = cast(RSAPublicKey, serialization.load_pem_public_key(PUBLIC_KEY.encode()))
return key.encrypt(content, padding.PKCS1v15())
'''aesencrypt'''
@staticmethod
def aesencrypt(key: bytes, content: bytes):
cipher = Cipher(algorithms.AES(key), modes.CBC(key))
padding_size = 16 - len(content) % 16
encryptor = cipher.encryptor()
return encryptor.update(content + (padding_size * chr(padding_size)).encode()) + encryptor.finalize()
'''calcmd5'''
@staticmethod
def calcmd5(*strings: str | bytes):
md5 = hashlib.md5()
for item in strings:
assert isinstance(item, (str, bytes))
if isinstance(item, bytes): md5.update(item)
elif isinstance(item, str): md5.update(item.encode())
return md5.hexdigest()
'''hash33'''
@staticmethod
def hash33(s: str, h: int = 0) -> int:
for c in s: h = (h << 5) + h + ord(c)
return 2147483647 & h
'''sign'''
@staticmethod
def sign(request: dict) -> str:
PART_1_INDEXES = [23, 14, 6, 36, 16, 40, 7, 19]
PART_2_INDEXES = [16, 1, 32, 12, 19, 27, 8, 5]
SCRAMBLE_VALUES = [89, 39, 179, 150, 218, 82, 58, 252, 177, 52, 186, 123, 120, 64, 242, 133, 143, 161, 121, 179]
PART_1_INDEXES = filter(lambda x: x < 40, PART_1_INDEXES)
hash = hashlib.sha1(orjson.dumps(request)).hexdigest().upper()
part1, part2, part3 = "".join(hash[i] for i in PART_1_INDEXES), "".join(hash[i] for i in PART_2_INDEXES), bytearray(20)
for i, v in enumerate(SCRAMBLE_VALUES): part3[i] = v ^ int(hash[i * 2 : i * 2 + 2], 16)
b64_part = re.sub(rb"[\\/+=]", b"", base64.b64encode(part3)).decode("utf-8")
return f"zzc{part1}{b64_part}{part2}".lower()
'''randombeaconid'''
@staticmethod
def randombeaconid():
beacon_id, time_month, rand1, rand2 = "", datetime.now().strftime("%Y-%m-") + "01", random.randint(100000, 999999), random.randint(100000000, 999999999)
for i in range(1, 41):
if i in [1, 2, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30, 33, 34, 37, 38]: beacon_id += f"k{i}:{time_month}{rand1}.{rand2}"
elif i == 3: beacon_id += "k3:0000000000000000"
elif i == 4: beacon_id += f"k4:{''.join(random.choices('123456789abcdef', k=16))}"
else: beacon_id += f"k{i}:{random.randint(0, 9999)}"
beacon_id += ";"
return beacon_id
'''randompayloadbydevice'''
@staticmethod
def randompayloadbydevice(device: Device, version: str):
fixed_rand = random.randint(0, 14400)
reserved = {
"harmony": "0", "clone": "0", "containe": "", "oz": "UhYmelwouA+V2nPWbOvLTgN2/m8jwGB+yUB5v9tysQg=", "oo": "Xecjt+9S1+f8Pz2VLSxgpw==",
"kelong": "0", "uptimes": (datetime.now() - timedelta(seconds=fixed_rand)).strftime("%Y-%m-%d %H:%M:%S"), "multiUser": "0", "bod": device.brand,
"dv": device.device, "firstLevel": "", "manufact": device.brand, "name": device.model, "host": "se.infra", "kernel": device.proc_version,
}
return {
"androidId": device.android_id, "platformId": 1, "appKey": APP_KEY, "appVersion": version, "beaconIdSrc": QQMusicClientUtils.randombeaconid(),
"brand": device.brand, "channelId": "10003505", "cid": "", "imei": device.imei, "imsi": "", "mac": "", "model": device.model, "networkType": "unknown",
"oaid": "", "osVersion": f"Android {device.version.release},level {device.version.sdk}", "qimei": "", "qimei36": "", "sdkVersion": "1.2.13.6",
"targetSdkVersion": "33", "audit": "", "userId": "{}", "packageId": "com.tencent.qqmusic", "deviceType": "Phone", "sdkName": "", "reserved": orjson.dumps(reserved).decode(),
}
'''obtainqimei'''
@staticmethod
def obtainqimei(version: str, device: Device):
try:
payload, ts = QQMusicClientUtils.randompayloadbydevice(device, version), int(time.time())
crypt_key, nonce = "".join(random.choices("adbcdef1234567890", k=16)), "".join(random.choices("adbcdef1234567890", k=16))
key = base64.b64encode(QQMusicClientUtils.rsaencrypt(crypt_key.encode())).decode()
params = base64.b64encode(QQMusicClientUtils.aesencrypt(crypt_key.encode(), orjson.dumps(payload))).decode()
extra = '{"appKey":"' + APP_KEY + '"}'
sign = QQMusicClientUtils.calcmd5(key, params, str(ts * 1000), nonce, SECRET, extra)
resp = requests.post("https://api.tencentmusic.com/tme/trpc/proxy",
headers={
"Host": "api.tencentmusic.com", "method": "GetQimei", "service": "trpc.tme_datasvr.qimeiproxy.QimeiProxy", "appid": "qimei_qq_android",
"sign": QQMusicClientUtils.calcmd5("qimei_qq_androidpzAuCmaFAaFaHrdakPjLIEqKrGnSOOvH", str(ts)), "user-agent": "QQMusic", "timestamp": str(ts),
},
json={"app": 0, "os": 1, "qimeiParams": {"key": key, "params": params, "time": str(ts), "nonce": nonce, "sign": sign, "extra": extra}},
)
data = orjson.loads(orjson.loads(resp.content)["data"])["data"]
device.qimei = data["q36"]
return QimeiResult(q16=data["q16"], q36=data["q36"])
except:
result = QimeiResult(q16="", q36="6c9d3cd110abca9b16311cee10001e717614")
return result
'''randomguid'''
@staticmethod
def randomguid():
return "".join(random.choices("abcdef1234567890", k=32))
'''randomsearchid'''
@staticmethod
def randomsearchid():
e = random.randint(1, 20)
t = e * 18014398509481984
n = random.randint(0, 4194304) * 4294967296
a = time.time()
r = round(a * 1000) % (24 * 60 * 60 * 1000)
return str(t + n + r)
'''buildcommonparams'''
@staticmethod
def buildcommonparams(credential: Credential = None, common_override: dict = None) -> dict[str, Any]:
common_override, credential = common_override or {}, credential or Credential()
qimei_result = QQMusicClientUtils().qimei
common = {"cv": QQMusicClientUtils.version_code, "v": QQMusicClientUtils.version_code, "QIMEI36": qimei_result['q36']}
common.update(QQMusicClientUtils.COMMON_DEFAULTS)
if bool(credential.musicid) and bool(credential.musickey): common.update({"qq": str(credential.musicid), "authst": credential.musickey, "tmeLoginType": str(credential.login_type)})
common.update(common_override)
return common
'''builddata'''
@staticmethod
def builddata(params: dict, module: str, method: str, process_bool: bool = True):
params = {k: int(v) if isinstance(v, bool) else v for k, v in params.items()} if process_bool else params
return {"module": module, "method": method, "param": params}
'''buildrequestdata'''
@staticmethod
def buildrequestdata(params: dict, module: str, method: str, credential: Credential = None, common_override: dict = None, process_bool: bool = True) -> dict[str, Any]:
return {"comm": QQMusicClientUtils.buildcommonparams(credential, common_override), f"{module}.{method}": QQMusicClientUtils.builddata(params, module, method, process_bool)}
catalog-sync/musicdl/modules/utils/quarkparser.py
+152
View File
@@ -0,0 +1,152 @@
'''
Function:
Implementation of QuarkParser
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import time
import requests
from urllib.parse import urlparse
from .misc import resp2json, cookies2dict
'''QuarkParser'''
class QuarkParser():
'''parsefromdirurl'''
@staticmethod
def parsefromdirurl(url: str, passcode: str = '', cookies: str | dict = '', max_tries: int = 3):
for _ in range(max_tries):
try: download_result, download_url = QuarkParser._parsefromdirurl(url=url, passcode=passcode, cookies=cookies); break
except Exception: download_result, download_url = {}, ""
return download_result, download_url
'''_parsefromdirurl'''
@staticmethod
def _parsefromdirurl(url: str, passcode: str = '', cookies: str | dict = ''):
# init
session, download_result = requests.Session(), {}
pwd_id = urlparse(url).path.strip('/').split('/')[-1]
cookies = cookies2dict(cookies)
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Core/1.94.225.400 QQBrowser/12.2.5544.400',
'origin': 'https://pan.quark.cn', 'referer': 'https://pan.quark.cn/', 'accept-language': 'zh-CN,zh;q=0.9',
}
# share/sharepage/token
json_data = {'pwd_id': pwd_id, 'passcode': passcode, 'support_visit_limit_private_share': 'true'}
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', '__dt': '597', '__t': f'{str(int(time.time() * 1000))}'}
(resp := session.post('https://drive-h.quark.cn/1/clouddrive/share/sharepage/token', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
token_data = resp2json(resp=resp); stoken = token_data['data']['stoken']; download_result['token_data'] = token_data; time.sleep(0.1)
# share/sharepage/detail-1
params = {
'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', 'ver': '2', 'pwd_id': pwd_id, 'stoken': stoken, 'pdir_fid': '0', 'force': '0', '_page': '1', '_size': '50', '_fetch_banner': '1',
'_fetch_share': '1', 'fetch_relate_conversation': '1', '_fetch_total': '1', '_sort': 'file_type:asc,file_name:asc', '__dt': '951', '__t': f'{int(time.time() * 1000)}',
}
(resp := session.get('https://drive-h.quark.cn/1/clouddrive/share/sharepage/detail', params=params, cookies=cookies, headers=headers)).raise_for_status()
detail_data = resp2json(resp=resp); pdir_fid = detail_data["data"]["list"][0]["fid"]; download_result['detail_data-1'] = detail_data; time.sleep(0.1)
# clouddrive/file/info/path_list
params = {"pr": "ucpro", "fr": "pc", "uc_param_str": "", "__dt": "1266", "__t": f"{int(time.time() * 1000)}"}
json_data = {"file_path": ["/来自：分享"]}
(resp := session.post('https://drive-pc.quark.cn/1/clouddrive/file/info/path_list', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
path_list_data = resp2json(resp=resp); to_pdir_fid = path_list_data["data"][0]["fid"]; download_result['path_list_data'] = path_list_data; time.sleep(0.1)
# share/sharepage/detail-2
params = {
'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', 'ver': '2', 'pwd_id': pwd_id, 'stoken': stoken, 'pdir_fid': pdir_fid,
'force': '0', '_page': '1', '_size': '50', '_fetch_banner': '0', '_fetch_share': '0', 'fetch_relate_conversation': '0',
'_fetch_total': '1', '_sort': 'file_type:asc,file_name:asc', '__dt': '1804336', '__t': f'{int(time.time() * 1000)}',
}
(resp := session.get('https://drive-h.quark.cn/1/clouddrive/share/sharepage/detail', params=params, cookies=cookies, headers=headers)).raise_for_status()
detail_data = resp2json(resp=resp); file_list: list[dict] = detail_data["data"]["list"]; file_list = sorted(file_list, key=lambda x: x.get("size", 0), reverse=True)
pdir_fid = file_list[0]['pdir_fid']; download_result['detail_data-2'] = detail_data; time.sleep(0.1)
# share/sharepage/save
params = {"pr": "ucpro", "fr": "pc", "uc_param_str": "", "__dt": "1233372", "__t": f"{int(time.time() * 1000)}"}
json_data = {
'pwd_id': pwd_id, 'stoken': stoken, 'pdir_fid': pdir_fid, 'to_pdir_fid': to_pdir_fid, 'fid_list': [file_list[0]['fid']],
'fid_token_list': [file_list[0]['share_fid_token']], 'scene': 'link',
}
(resp := session.post(url='https://drive-pc.quark.cn/1/clouddrive/share/sharepage/save', params=params, cookies=cookies, json=json_data, headers=headers)).raise_for_status()
save_data = resp2json(resp=resp); task_id = save_data['data']['task_id']; download_result['save_data'] = save_data; time.sleep(0.1)
# clouddrive/task
for retry_index in range(5):
try:
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', 'task_id': task_id, 'retry_index': str(retry_index), '__dt': '1234221', '__t': f'{str(int(time.time() * 1000))}'}
(resp := session.get('https://drive-pc.quark.cn/1/clouddrive/task', params=params, cookies=cookies, headers=headers)).raise_for_status()
task_data = resp2json(resp=resp); fid_encrypt = task_data['data']['save_as']['save_as_top_fids'][0]
download_result['task_data'] = task_data; break
except:
time.sleep(0.1); continue
# clouddrive/file/download
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.56 Chrome/100.0.4896.160 Electron/18.3.5.12-a038f7b798 Safari/537.36 Channel/pckk_other_ch",
"Accept": "application/json, text/plain, */*", "Content-Type": "application/json", "accept-language": "zh-CN", "origin": "https://pan.quark.cn", "referer": "https://pan.quark.cn/",
}
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', '__dt': '1235217', '__t': f'{str(int(time.time() * 1000))}'}
json_data = {'fids': [fid_encrypt]}
(resp := session.post('https://drive-pc.quark.cn/1/clouddrive/file/download', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
download_data = resp2json(resp=resp); download_url = download_data["data"][0]["download_url"]; download_result['download_data'] = download_data
# return
return download_result, download_url
'''parsefromurl'''
@staticmethod
def parsefromurl(url: str, passcode: str = '', cookies: str | dict = '', max_tries: int = 3):
for _ in range(max_tries):
try: download_result, download_url = QuarkParser._parsefromurl(url=url, passcode=passcode, cookies=cookies); break
except Exception: download_result, download_url = {}, ""
return download_result, download_url
'''_parsefromurl'''
@staticmethod
def _parsefromurl(url: str, passcode: str = '', cookies: str | dict = ''):
# init
session, download_result = requests.Session(), {}
pwd_id = urlparse(url).path.strip('/').split('/')[-1]
cookies = cookies2dict(cookies)
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Core/1.94.225.400 QQBrowser/12.2.5544.400',
'origin': 'https://pan.quark.cn', 'referer': 'https://pan.quark.cn/', 'accept-language': 'zh-CN,zh;q=0.9',
}
# share/sharepage/token
json_data = {'pwd_id': pwd_id, 'passcode': passcode}
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', '__dt': '596', '__t': f'{str(int(time.time() * 1000))}'}
(resp := session.post('https://drive-h.quark.cn/1/clouddrive/share/sharepage/token', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
token_data = resp2json(resp=resp); stoken = token_data['data']['stoken']; download_result['token_data'] = token_data; time.sleep(0.1)
# share/sharepage/detail
params = {
"pr": "ucpro", "fr": "pc", "uc_param_str": "", "ver": "2", "pwd_id": pwd_id, "stoken": stoken, "pdir_fid": "0", "force": "0",
"_page": "1", "_size": "50", "_fetch_banner": "1", "_fetch_share": "1", "fetch_relate_conversation": "1", "_fetch_total": "1",
"_sort": "file_type:asc,file_name:asc", "__dt": "1020", "__t": f"{int(time.time() * 1000)}"
}
(resp := session.get('https://drive-h.quark.cn/1/clouddrive/share/sharepage/detail', params=params, cookies=cookies, headers=headers)).raise_for_status()
detail_data = resp2json(resp=resp); fid = detail_data["data"]["list"][0]["fid"]; share_fid_token = detail_data["data"]["list"][0]["share_fid_token"]
download_result['detail_data'] = detail_data; time.sleep(0.1)
# clouddrive/file/info/path_list
params = {"pr": "ucpro", "fr": "pc", "uc_param_str": "", "__dt": "1266", "__t": f"{int(time.time() * 1000)}"}
json_data = {"file_path": ["/来自：分享"]}
(resp := session.post('https://drive-pc.quark.cn/1/clouddrive/file/info/path_list', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
path_list_data = resp2json(resp=resp); to_pdir_fid = path_list_data["data"][0]["fid"]; download_result['path_list_data'] = path_list_data; time.sleep(0.1)
# share/sharepage/save
params = {"pr": "ucpro", "fr": "pc", "uc_param_str": "", "__dt": "5660", "__t": f"{int(time.time() * 1000)}"}
json_data = {"pwd_id": pwd_id, "stoken": stoken, "pdir_fid": "0", "to_pdir_fid": to_pdir_fid, "fid_list": [fid], "fid_token_list": [share_fid_token], "scene": "link"}
(resp := session.post(url='https://drive-pc.quark.cn/1/clouddrive/share/sharepage/save', params=params, cookies=cookies, json=json_data, headers=headers)).raise_for_status()
save_data = resp2json(resp=resp); task_id = save_data['data']['task_id']; download_result['save_data'] = save_data; time.sleep(0.1)
# clouddrive/task
for retry_index in range(5):
try:
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', 'task_id': task_id, 'retry_index': str(retry_index), '__dt': '6355', '__t': f'{str(int(time.time() * 1000))}'}
(resp := session.get('https://drive-pc.quark.cn/1/clouddrive/task', params=params, cookies=cookies, headers=headers)).raise_for_status()
task_data = resp2json(resp=resp); fid_encrypt = task_data['data']['save_as']['save_as_top_fids'][0]
download_result['task_data'] = task_data; break
except:
time.sleep(0.1); continue
# clouddrive/file/download
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.56 Chrome/100.0.4896.160 Electron/18.3.5.12-a038f7b798 Safari/537.36 Channel/pckk_other_ch",
"Accept": "application/json, text/plain, */*", "Content-Type": "application/json", "accept-language": "zh-CN", "origin": "https://pan.quark.cn", "referer": "https://pan.quark.cn/",
}
params = {'pr': 'ucpro', 'fr': 'pc', 'uc_param_str': '', '__dt': '6743', '__t': f'{str(int(time.time() * 1000))}'}
json_data = {'fids': [fid_encrypt]}
(resp := session.post('https://drive-pc.quark.cn/1/clouddrive/file/download', params=params, json=json_data, cookies=cookies, headers=headers)).raise_for_status()
download_data = resp2json(resp=resp); download_url = download_data["data"][0]["download_url"]; download_result['download_data'] = download_data
# return
return download_result, download_url
catalog-sync/musicdl/modules/utils/sodautils.py
+172
View File
@@ -0,0 +1,172 @@
'''
Function:
Implementation of SodaMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import re
import struct
import base64
from typing import Dict, Any, List
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
'''SpadeDecryptor'''
class SpadeDecryptor:
'''bitcount'''
@staticmethod
def bitcount(n):
n = n & 0xFFFFFFFF
n = n - ((n >> 1) & 0x55555555)
n = (n & 0x33333333) + ((n >> 2) & 0x33333333)
return ((n + (n >> 4) & 0xF0F0F0F) * 0x1010101) >> 24
'''decodebase36'''
@staticmethod
def decodebase36(c):
if 48 <= c <= 57: return c - 48
if 97 <= c <= 122: return c - 97 + 10
return 0xFF
'''decryptspadeinner'''
@staticmethod
def decryptspadeinner(spade_key_bytes):
result = bytearray(len(spade_key_bytes))
buff = bytearray([0xFA, 0x55]) + spade_key_bytes
for i in range(len(result)):
v = (spade_key_bytes[i] ^ buff[i]) - SpadeDecryptor.bitcount(i) - 21
while v < 0: v += 255
result[i] = v
return result
'''extractkey'''
@classmethod
def extractkey(cls, play_auth_str):
binary_string = base64.b64decode(play_auth_str)
bytes_data = bytearray(binary_string)
if len(bytes_data) < 3: return None
padding_len = (bytes_data[0] ^ bytes_data[1] ^ bytes_data[2]) - 48
if len(bytes_data) < padding_len + 2: return None
inner_input = bytes_data[1: len(bytes_data)-padding_len]
tmp_buff = cls.decryptspadeinner(inner_input)
if len(tmp_buff) == 0: return None
skip_bytes = cls.decodebase36(tmp_buff[0])
decoded_message_len = len(bytes_data) - padding_len - 2
end_index = 1 + decoded_message_len - skip_bytes
final_bytes = tmp_buff[1:end_index]
return final_bytes.decode('utf-8')
'''AudioDecryptor'''
class AudioDecryptor:
'''readuint32be'''
@staticmethod
def readuint32be(data, offset):
return struct.unpack(">I", data[offset: offset+4])[0]
'''findbox'''
@staticmethod
def findbox(data: bytes, box_type: str, start: int = 0, end: int = None):
if end is None: end = len(data)
pos = start
while pos + 8 <= end:
size = AudioDecryptor.readuint32be(data, pos)
if size < 8: break
current_type_bytes = data[pos+4: pos+8]
try: current_type = current_type_bytes.decode('ascii', errors='ignore')
except: current_type = "????"
if current_type == box_type: return {'offset': pos, 'size': size, 'data': data[pos+8: pos+size]}
pos += size
return None
'''decrypt'''
@staticmethod
def decrypt(file_data: bytes, play_auth: str, output_filepath: str = "./decrypted.m4a"):
hex_key = SpadeDecryptor.extractkey(play_auth)
if not hex_key: return
moov = AudioDecryptor.findbox(file_data, 'moov')
if not moov: return
senc = AudioDecryptor.findbox(file_data, 'senc', start=moov['offset'] + 8, end=moov['offset'] + moov['size'])
trak = AudioDecryptor.findbox(file_data, 'trak', start=moov['offset'] + 8, end=moov['offset'] + moov['size'])
if not trak: return
mdia = AudioDecryptor.findbox(file_data, 'mdia', start=trak['offset'] + 8, end=trak['offset'] + trak['size'])
if not mdia: return
minf = AudioDecryptor.findbox(file_data, 'minf', start=mdia['offset'] + 8, end=mdia['offset'] + mdia['size'])
if not minf: return
stbl = AudioDecryptor.findbox(file_data, 'stbl', start=minf['offset'] + 8, end=minf['offset'] + minf['size'])
if not stbl: return
stsz = AudioDecryptor.findbox(file_data, 'stsz', start=stbl['offset'] + 8, end=stbl['offset'] + stbl['size'])
if not stsz: return
stsz_data = stsz['data']
sample_size_fixed, sample_count, sample_sizes = struct.unpack(">I", stsz_data[4: 8])[0], struct.unpack(">I", stsz_data[8: 12])[0], []
if sample_size_fixed != 0: sample_sizes = [sample_size_fixed] * sample_count
else:
for i in range(sample_count): sample_sizes.append(struct.unpack(">I", stsz_data[12 + i*4 : 16 + i*4])[0])
if not senc: senc = AudioDecryptor.findbox(file_data, 'senc', start=stbl['offset'] + 8, end=stbl['offset'] + stbl['size'])
if not senc: return
senc_body = senc['data']
senc_flags, senc_sample_count, ivs, ptr = struct.unpack(">I", senc_body[0:4])[0] & 0x00FFFFFF, struct.unpack(">I", senc_body[4:8])[0], [], 8
has_subsamples = (senc_flags & 0x02) != 0
for _ in range(senc_sample_count):
ivs.append(senc_body[ptr : ptr+8] + b'\x00'*8); ptr += 8
if has_subsamples: sub_count = struct.unpack(">H", senc_body[ptr: ptr+2])[0]; ptr += 2 + (sub_count * 6)
mdat = AudioDecryptor.findbox(file_data, 'mdat')
if not mdat: return
key_bytes, backend, decrypted_mdat, read_ptr = bytes.fromhex(hex_key), default_backend(), bytearray(), mdat['offset'] + 8
for i in range(len(sample_sizes)):
size = sample_sizes[i]
if i < len(ivs):
cipher = Cipher(algorithms.AES(key_bytes), modes.CTR(ivs[i]), backend=backend)
decryptor = cipher.decryptor()
plain_chunk = decryptor.update(file_data[read_ptr: read_ptr + size]) + decryptor.finalize()
decrypted_mdat.extend(plain_chunk)
else:
decrypted_mdat.extend(file_data[read_ptr: read_ptr + size])
read_ptr += size
stsd = AudioDecryptor.findbox(file_data, 'stsd', start=stbl['offset'] + 8, end=stbl['offset'] + stbl['size'])
if stsd:
offset, length = stsd['offset'], stsd['size']
original_stsd = file_data[offset: offset+length]
new_stsd = original_stsd.replace(b'enca', b'mp4a', 1)
file_data[offset: offset+length] = new_stsd
if len(decrypted_mdat) == mdat['size'] - 8: file_data[mdat['offset']+8: mdat['offset']+mdat['size']] = decrypted_mdat
else: pass
with open(output_filepath, "wb") as fp: fp.write(file_data)
'''SodaTimedLyricsParser'''
class SodaTimedLyricsParser:
LINE_PATTERN_RE = re.compile(r"^\[(\d+),(\d+)\]")
TOKEN_PATTERN_RE = re.compile(r"<(\d+),(\d+),(\d+)>")
'''parsetimedlyrics'''
@staticmethod
def parsetimedlyrics(text: str) -> List[Dict[str, Any]]:
if not text or text in {'NULL'}: return []
text = text.replace(r"\u003C", "<").replace(r"\u003E", ">")
lines_out: List[Dict[str, Any]] = []
for raw_line in text.splitlines():
if not (raw_line := raw_line.rstrip("\n")).strip(): continue
if not (m := SodaTimedLyricsParser.LINE_PATTERN_RE.match(raw_line.strip())): continue
line_start, line_dur = int(m.group(1)), int(m.group(2))
line_end, rest, tokens, pieces = line_start + line_dur, raw_line[m.end():], [], []
matches = list(SodaTimedLyricsParser.TOKEN_PATTERN_RE.finditer(rest))
for i, tm in enumerate(matches):
offset, dur, flag, seg_start = int(tm.group(1)), int(tm.group(2)), int(tm.group(3)), tm.end()
seg_end = matches[i + 1].start() if i + 1 < len(matches) else len(rest)
if (token_text := rest[seg_start: seg_end].replace("\r", "")) == "": continue
abs_start, abs_end = line_start + offset, line_start + offset + dur
tokens.append({"text": token_text, "offset_ms": offset, "duration_ms": dur, "flag": flag, "start_ms": abs_start, "end_ms": abs_end}); pieces.append(token_text)
lines_out.append({"line_start_ms": line_start, "line_duration_ms": line_dur, "line_end_ms": line_end, "text": "".join(pieces), "tokens": tokens, "raw": rest})
return lines_out
'''toplaintext'''
@staticmethod
def toplaintext(parsed: List[Dict[str, Any]]) -> str:
if not parsed: return
return "\n".join(line["text"] for line in parsed)
'''tolrclinelevel'''
@staticmethod
def tolrclinelevel(parsed: List[Dict[str, Any]], use_centiseconds: bool = True) -> str:
if not parsed: return
def fmt(ms: int) -> str:
mm, ss = ms // 60000, (ms % 60000) // 1000
if use_centiseconds: xx = (ms % 1000) // 10; return f"{mm:02d}:{ss:02d}.{xx:02d}"
else: return f"{mm:02d}:{ss:02d}"
return "\n".join(f"[{fmt(line['line_start_ms'])}]{line['text']}" for line in parsed)
catalog-sync/musicdl/modules/utils/songinfoutils.py
+298
View File
@@ -0,0 +1,298 @@
'''
Function:
Implementation of SongInfoUtils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
from __future__ import annotations
import os
import base64
import shutil
import requests
import tempfile
from pathlib import Path
from mutagen import File
from .data import SongInfo
from tinytag import TinyTag
from .lyric import WhisperLRC
from mimetypes import guess_type
from .logger import LoggerHandle
from mutagen.flac import Picture
from mutagen.mp4 import MP4Cover
from .misc import seconds2hms, byte2mb
from mutagen.id3 import ID3, USLT, APIC, TIT2, TALB, TPE1
'''SongInfoUtils'''
class SongInfoUtils:
'''supplsonginfothensavelyricsthenwritetags'''
@staticmethod
def supplsonginfothensavelyricsthenwritetags(song_info: SongInfo, logger_handle: LoggerHandle, disable_print: bool, auto_save_lyrics_then_write_tags: bool = True, enable_whisperlrc: bool = False) -> SongInfo:
path = Path(song_info.save_path)
# correct file size
size = path.stat().st_size
song_info.file_size_bytes = size
song_info.file_size = byte2mb(size=size)
# tinytag parse
try: tag = TinyTag.get(str(path))
except Exception as err: logger_handle.warning(f'SongInfoUtils.supplsonginfothensavelyricsthenwritetags >>> {str(path)} (Err: {err})', disable_print=disable_print); tag = None
if tag and tag.duration: song_info.duration_s = int(round(tag.duration)); song_info.duration = seconds2hms(tag.duration)
if tag and tag.bitrate: song_info.bitrate = int(round(tag.bitrate))
if tag and tag.samplerate: song_info.samplerate = int(tag.samplerate)
if tag and tag.channels: song_info.channels = int(tag.channels)
if tag and getattr(tag, "codec", None): song_info.codec = tag.codec
elif tag and getattr(tag, "extra", None) and isinstance(tag.extra, dict): song_info.codec = tag.extra.get("codec") or tag.extra.get("mime-type")
# lyric
if ((os.environ.get('ENABLE_WHISPERLRC', 'False').lower() == 'true') or enable_whisperlrc) and ((not song_info.lyric) or (song_info.lyric in {'NULL'})):
lyric_result = WhisperLRC(model_size_or_path='small').fromfilepath(str(path))
lyric = lyric_result['lyric']; song_info.lyric = lyric; song_info.raw_data['lyric'] = lyric_result
# write tags to audio file
if auto_save_lyrics_then_write_tags:
try: SongInfoUtils.savelyricsthenwritetagstoaudio(song_info, overwrite=False)
except: pass
# return
return song_info
'''savelyricsthenwritetagstoaudio'''
@staticmethod
def savelyricsthenwritetagstoaudio(song_info: SongInfo, overwrite: bool = False, *, timeout: int = 15) -> dict:
lyrics_text = SongInfoUtils.normalizetext(getattr(song_info, "lyric", None)); title = SongInfoUtils.normalizetext(getattr(song_info, "song_name", None))
album = SongInfoUtils.normalizetext(getattr(song_info, "album", None)); artists = SongInfoUtils.normalizetext(getattr(song_info, "singers", None))
cover_source = SongInfoUtils.normalizetext(getattr(song_info, "cover_url", None)); audio_path = Path(song_info.save_path)
results = {"lyrics_embedded": False, "basic_tags_embedded": False, "cover_embedded": False, "lrc_saved": False}
if lyrics_text: results["lrc_saved"] = SongInfoUtils.savelrctofile(audio_path, lyrics_text, overwrite=overwrite)
if lyrics_text: results["lyrics_embedded"] = SongInfoUtils.safeeditaudio(audio_path=audio_path, editor=SongInfoUtils.embedlyrics, overwrite=overwrite, lyrics_text=lyrics_text)
if title or album or artists: results["basic_tags_embedded"] = SongInfoUtils.safeeditaudio(audio_path=audio_path, editor=SongInfoUtils.embedbasictags, overwrite=overwrite, title=title, album=album, artists=artists)
if cover_source and SongInfoUtils.lookslikecoversource(cover_source): results["cover_embedded"] = SongInfoUtils.safeeditaudio(audio_path=audio_path, editor=SongInfoUtils.embedcover, overwrite=overwrite, cover_source=cover_source, timeout=timeout)
return results
'''savelrctofile'''
@staticmethod
def savelrctofile(audio_path: Path, lyrics_text: str, *, overwrite: bool = False) -> bool:
lrc_path = audio_path.with_suffix(".lrc")
if lrc_path.exists() and not overwrite: return False
content = (lyrics_text or "").replace("\r\n", "\n").strip()
if not content: return False
if not content.endswith("\n"): content += "\n"
return SongInfoUtils.atomicwritetext(lrc_path, content)
'''safeeditaudio'''
@staticmethod
def safeeditaudio(audio_path: Path, editor, **editor_kwargs) -> bool:
if not audio_path.exists(): return False
if not SongInfoUtils.audioreadable(audio_path): return False
temp_path = SongInfoUtils.maketemppath(audio_path)
backup_path = audio_path.with_suffix(audio_path.suffix + ".bak")
try:
shutil.copy2(audio_path, temp_path)
changed = bool(editor(temp_path, **editor_kwargs))
if not changed: return False
if not SongInfoUtils.audioreadable(temp_path): return False
backup_path.unlink(missing_ok=True)
os.replace(audio_path, backup_path)
os.replace(temp_path, audio_path)
if not SongInfoUtils.audioreadable(audio_path): os.replace(backup_path, audio_path); return False
backup_path.unlink(missing_ok=True)
return True
except Exception:
if (not audio_path.exists()) and backup_path.exists():
try: os.replace(backup_path, audio_path)
except Exception: pass
return False
finally:
temp_path.unlink(missing_ok=True)
'''safegeteditabletags'''
@staticmethod
def safegeteditabletags(audio):
if (tags := getattr(audio, "tags", None)) is not None: return tags
try: audio.add_tags()
except Exception: pass
return getattr(audio, "tags", None) or {}
'''embedlyrics'''
@staticmethod
def embedlyrics(audio_path: Path, *, overwrite: bool, lyrics_text: str) -> bool:
# init
audio = File(audio_path)
if audio is None: return False
cls = audio.__class__.__name__; text = (lyrics_text or "").replace("\r\n", "\n").strip()
if not text: return False
# MP3
if cls == "MP3":
id3 = SongInfoUtils.loadorcreateid3(audio_path)
has = any(k.startswith("USLT") for k in id3.keys())
if has and not overwrite: return False
if overwrite: id3.delall("USLT")
id3.add(USLT(encoding=3, lang="eng", desc="Lyrics", text=text))
id3.save(audio_path, v2_version=3)
return True
# MP4/M4A
if cls == "MP4":
tags = SongInfoUtils.safegeteditabletags(audio=audio); key = "\xa9lyr"
if tags.get(key) and not overwrite: return False
tags[key] = [text]; audio.tags = tags; audio.save()
return True
# FLAC/OGG/OPUS
if cls in {"FLAC", "OggVorbis", "OggOpus", "OggSpeex", "OggTheora"}:
tags = SongInfoUtils.safegeteditabletags(audio=audio); has = bool(tags.get("LYRICS"))
if has and not overwrite: return False
tags["LYRICS"] = [text]; audio.tags = tags; audio.save()
return True
# ASF/WMA
if cls == "ASF":
tags = SongInfoUtils.safegeteditabletags(audio=audio); key = "WM/Lyrics"
if tags.get(key) and not overwrite: return False
tags[key] = [text]; audio.tags = tags; audio.save()
return True
return False
'''embedbasictags'''
@staticmethod
def embedbasictags(audio_path: Path, *, overwrite: bool, title: str | None, album: str | None, artists: list[str] | None) -> bool:
# init
audio = File(audio_path)
if audio is None: return False
cls = audio.__class__.__name__; changed = False
# MP3
if cls == "MP3":
id3 = SongInfoUtils._load_or_create_id3(audio_path)
if title and (overwrite or not id3.getall("TIT2")): id3.setall("TIT2", [TIT2(encoding=3, text=title)]); changed = True
if album and (overwrite or not id3.getall("TALB")): id3.setall("TALB", [TALB(encoding=3, text=album)]); changed = True
if artists and (overwrite or not id3.getall("TPE1")): id3.setall("TPE1", [TPE1(encoding=3, text=artists)]); changed = True
if changed: id3.save(audio_path, v2_version=3)
return changed
# MP4/M4A
if cls == "MP4":
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if title and (overwrite or not tags.get("\xa9nam")): tags["\xa9nam"] = [title]; changed = True
if album and (overwrite or not tags.get("\xa9alb")): tags["\xa9alb"] = [album]; changed = True
if artists and (overwrite or not tags.get("\xa9ART")): tags["\xa9ART"] = artists; changed = True
if changed: audio.tags = tags; audio.save()
return changed
# FLAC / OGG / OPUS
if cls in {"FLAC", "OggVorbis", "OggOpus", "OggSpeex", "OggTheora"}:
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if title and (overwrite or not tags.get("TITLE")): tags["TITLE"] = [title]; changed = True
if album and (overwrite or not tags.get("ALBUM")): tags["ALBUM"] = [album]; changed = True
if artists and (overwrite or not tags.get("ARTIST")): tags["ARTIST"] = artists; changed = True
if changed: audio.tags = tags; audio.save()
return changed
# ASF/WMA
if cls == "ASF":
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if title and (overwrite or not tags.get("Title")): tags["Title"] = [title]; changed = True
if album and (overwrite or not tags.get("WM/AlbumTitle")): tags["WM/AlbumTitle"] = [album]; changed = True
if artists and (overwrite or not tags.get("Author")): tags["Author"] = artists; changed = True
if changed: audio.tags = tags; audio.save()
return changed
return False
'''embedcover'''
@staticmethod
def embedcover(audio_path: Path, *, overwrite: bool, cover_source: str, timeout: int = 15) -> bool:
audio = File(audio_path)
if audio is None: return False
cls = audio.__class__.__name__
cover_bytes, mime = SongInfoUtils.loadimagebytesandmime(cover_source, timeout=timeout)
# MP3
if cls == "MP3":
id3 = SongInfoUtils._load_or_create_id3(audio_path)
has = any(k.startswith("APIC") for k in id3.keys())
if has and not overwrite: return False
if overwrite: id3.delall("APIC")
id3.add(APIC(encoding=3, mime=mime, type=3, desc="Cover", data=cover_bytes))
id3.save(audio_path, v2_version=3)
return True
# MP4
if cls == "MP4":
if mime not in {"image/jpeg", "image/png"}: return False
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if tags.get("covr") and not overwrite: return False
image_format = MP4Cover.FORMAT_PNG if mime == "image/png" else MP4Cover.FORMAT_JPEG
tags["covr"] = [MP4Cover(cover_bytes, imageformat=image_format)]
audio.tags = tags; audio.save()
return True
# FLAC
if cls == "FLAC":
has = bool(getattr(audio, "pictures", []))
if has and not overwrite: return False
picture = Picture()
picture.type = 3; picture.mime = mime; picture.desc = "Cover"; picture.data = cover_bytes
if overwrite: audio.clear_pictures()
audio.add_picture(picture); audio.save()
return True
# OGG/OPUS
if cls in {"OggVorbis", "OggOpus", "OggSpeex", "OggTheora"}:
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if tags.get("METADATA_BLOCK_PICTURE") and not overwrite: return False
picture = Picture()
picture.type = 3; picture.mime = mime; picture.desc = "Cover"; picture.data = cover_bytes
tags["METADATA_BLOCK_PICTURE"] = [base64.b64encode(picture.write()).decode("ascii")]
audio.tags = tags; audio.save()
return True
# ASF/WMA
if cls == "ASF":
try: from mutagen.asf import ASFPicture
except Exception: return False
tags = SongInfoUtils.safegeteditabletags(audio=audio)
if tags.get("WM/Picture") and not overwrite: return False
picture = ASFPicture()
picture.type = 3; picture.mime_type = mime; picture.description = "Cover"; picture.data = cover_bytes
tags["WM/Picture"] = [picture]
audio.tags = tags; audio.save()
return True
return False
'''loadimagebytesandmime'''
@staticmethod
def loadimagebytesandmime(cover: str | Path, *, timeout: int = 15) -> tuple[bytes, str]:
cover_str = str(cover).strip()
if not cover_str: raise ValueError("Empty cover")
# local path
if not cover_str.startswith("http"): cover_path = Path(cover_str); data = cover_path.read_bytes(); mime = (guess_type(str(cover_path))[0] or "image/jpeg").split(";", 1)[0].lower(); return data, mime
# url
(resp := requests.get(cover_str, timeout=timeout, headers={"User-Agent": "Mozilla/5.0"}, allow_redirects=True)).raise_for_status()
data = resp.content or b""
content_type = (resp.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
mime = (content_type or (guess_type(cover_str)[0] or "image/jpeg")).split(";", 1)[0].lower()
# minimal signature fallback
signature = data[:8]
if signature.startswith(b"\xFF\xD8\xFF"): mime = "image/jpeg"
elif signature.startswith(b"\x89PNG\r\n\x1a\n"): mime = "image/png"
if not mime.startswith("image/"): raise ValueError(f"Not an image (Content-Type={content_type!r})")
return data, mime
'''normalizetext'''
@staticmethod
def normalizetext(value) -> str | None:
if not value or value in {'NULL', 'null', 'None', 'none'}: return None
text = str(value).strip()
return text or None
'''lookslikecoversource'''
@staticmethod
def lookslikecoversource(cover_source: str) -> bool:
return cover_source.startswith("http") or Path(cover_source).exists()
'''audioreadable'''
@staticmethod
def audioreadable(audio_path: Path) -> bool:
try:
if not audio_path.exists() or audio_path.stat().st_size <= 0: return False
audio = File(audio_path)
if audio is None or getattr(audio, "info", None) is None: return False
TinyTag.get(str(audio_path))
return True
except Exception:
return False
'''maketemppath'''
@staticmethod
def maketemppath(audio_path: Path) -> Path:
fd, temp_name = tempfile.mkstemp(prefix=audio_path.stem + ".", suffix=audio_path.suffix, dir=str(audio_path.parent))
os.close(fd)
return Path(temp_name)
'''atomicwritetext'''
@staticmethod
def atomicwritetext(path: Path, text: str) -> bool:
fd, temp_name = tempfile.mkstemp(prefix=path.stem + ".", suffix=path.suffix, dir=str(path.parent))
os.close(fd); temp_path = Path(temp_name)
try: temp_path.write_text(text, encoding="utf-8"); os.replace(temp_path, path); return True
except Exception: return False
finally: temp_path.unlink(missing_ok=True)
'''loadorcreateid3'''
@staticmethod
def loadorcreateid3(audio_path: Path) -> ID3:
try: return ID3(audio_path)
except Exception: return ID3()
catalog-sync/musicdl/modules/utils/spotifyutils.py
+169
View File
@@ -0,0 +1,169 @@
'''
Function:
Implementation of SpotifyMusicClient Utils
Author:
Zhenchao Jin
WeChat Official Account (微信公众号):
Charles的皮卡丘
'''
import re
import copy
import time
import hmac
import base64
import hashlib
import requests
import json_repair
from typing import Dict, List, Tuple
from .misc import resp2json, safeextractfromdict
'''SpotifyMusicClientUtils'''
class SpotifyMusicClientUtils():
BROWSER_VERSION = '145'
COMMON_HEADERS = {'Content-Type': 'application/json', 'User-Agent': f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{BROWSER_VERSION}.0.0.0 Safari/537.36', 'Sec-Ch-Ua': f'"Chromium";v="{BROWSER_VERSION}", "Not(A:Brand";v="24", "Google Chrome";v="{BROWSER_VERSION}"'}
'''getlatesttotpsecret'''
@staticmethod
def getlatesttotpsecret(version: int = 61) -> dict:
VERSION_TO_SECRET = {
59: [123, 105, 79, 70, 110, 59, 52, 125, 60, 49, 80, 70, 89, 75, 80, 86, 63, 53, 123, 37, 117, 49, 52, 93, 77, 62, 47, 86, 48, 104, 68, 72],
60: [79, 109, 69, 123, 90, 65, 46, 74, 94, 34, 58, 48, 70, 71, 92, 85, 122, 63, 91, 64, 87, 87],
61: [44, 55, 47, 42, 70, 40, 34, 114, 76, 74, 50, 111, 120, 97, 75, 76, 94, 102, 43, 69, 49, 120, 118, 80, 64, 78],
}
return {"version": version, "secret": VERSION_TO_SECRET[version]}
'''generatetotp'''
@staticmethod
def generatetotp(secret: List[int]) -> str:
transformed = [e ^ ((t % 33) + 9) for t, e in enumerate(secret)]
hex_str = ("".join(str(num) for num in transformed)).encode('ascii').hex()
base32_secret = base64.b64encode(bytes.fromhex(hex_str)).decode('utf-8').replace('=', '')
base32_bytes = base64.b64decode(base32_secret + '==')
time_step = int(time.time() / 30); time_hex = format(time_step, '016x')
digest = hmac.new(base32_bytes, bytes.fromhex(time_hex), hashlib.sha1).digest()
offset = digest[19] & 0xf; code = int.from_bytes(digest[offset: offset+4], byteorder='big') & 0x7fffffff
return str(code % 1000000).zfill(6)
'''getaccesstoken'''
@staticmethod
def getaccesstoken(session: requests.Session, totp: str, totp_ver: int, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
params = {'reason': 'init', 'productType': 'web-player', 'totp': totp, 'totpVer': str(totp_ver), 'totpServer': totp}
(resp := session.get("https://open.spotify.com/api/token", params=params, headers=SpotifyMusicClientUtils.COMMON_HEADERS, **request_overrides)).raise_for_status()
return {"accessToken": (data := resp2json(resp=resp)).get('accessToken'), "clientId": data.get('clientId')}
'''getclienttoken'''
@staticmethod
def getclienttoken(session: requests.Session, client_version: str, client_id: str, device_id: str, request_overrides: dict = None) -> str:
request_overrides = request_overrides or {}
payload = {"client_data": {"client_version": client_version, "client_id": client_id, "js_sdk_data": {"device_brand": "unknown", "device_model": "unknown", "os": "windows", "os_version": "NT 10.0", "device_id": device_id, "device_type": "computer"}}}
headers = SpotifyMusicClientUtils.COMMON_HEADERS.copy()
headers.update({'Authority': 'clienttoken.spotify.com', 'Accept': 'application/json'})
(resp := session.post('https://clienttoken.spotify.com/v1/clienttoken', headers=headers, json=payload, **request_overrides)).raise_for_status()
return safeextractfromdict(resp2json(resp=resp), ['granted_token', 'token'], '')
'''extractjslinks'''
@staticmethod
def extractjslinks(html: str) -> List[str]:
script_tag_regex = re.compile(r'<script[^>]+src="([^"]+\.js)"[^>]*>')
return script_tag_regex.findall(html)
'''getsessiondata'''
@staticmethod
def getsessiondata(session: requests.Session, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
(resp := session.get('https://open.spotify.com', headers=SpotifyMusicClientUtils.COMMON_HEADERS, **request_overrides)).raise_for_status()
cookie_match = re.search(r'sp_t=([^;]+)', resp.headers.get('set-cookie', '')); device_id = cookie_match.group(1) if cookie_match else ''
app_server_config_match, client_version = re.search(r'<script id="appServerConfig" type="text/plain">([^<]+)</script>', resp.text), ''
try: client_version = json_repair.loads(base64.b64decode(app_server_config_match.group(1)).decode("utf-8")).get("clientVersion", "") if app_server_config_match else (m.group(1) if (m := re.search(r'"clientVersion":"([^"]+)"', resp.text)) else "")
except Exception: client_version = m.group(1) if (m := re.search(r'"clientVersion":"([^"]+)"', resp.text)) else ""
all_js_links, js_pack_relative = SpotifyMusicClientUtils.extractjslinks(resp.text), ''
js_pack_relative = next((link for link in all_js_links if 'web-player/web-player' in link and link.endswith('.js')), js_pack_relative)
if js_pack_relative.startswith('http'): js_pack = js_pack_relative
else: js_pack = f'https://open.spotify.com{js_pack_relative}' if js_pack_relative else ''
return {"deviceId": device_id, "clientVersion": client_version, "jsPack": js_pack}
'''SpotifyMusicClientPlaylistUtils'''
class SpotifyMusicClientPlaylistUtils():
'''extractmappings'''
@staticmethod
def extractmappings(js_code: str) -> Tuple[Dict[str, str], Dict[str, str]]:
matches = re.compile(r'\{\d+:"[^"]+"(?:,\d+:"[^"]+")*\}').findall(js_code)
if not matches or len(matches) < 5: return {}, {}
parse_match_func = lambda match_str: {key.strip(): value.strip().strip('"') for entry in re.split(r',(?=\d+:)', match_str[1:-1]) for key, sep, value in [entry.partition(':')] if sep}
return parse_match_func(matches[3]), parse_match_func(matches[4])
'''combinechunks'''
@staticmethod
def combinechunks(str_mapping: Dict[str, str], hash_mapping: Dict[str, str]) -> List[str]:
chunks = []
for key, string_val in str_mapping.items():
if (hash_val := hash_mapping.get(key)): chunks.append(f"{string_val}.{hash_val}.js")
return chunks
'''getsha256hash'''
@staticmethod
def getsha256hash(session: requests.Session, js_pack: str, request_overrides: dict = None) -> str:
fallback_hash, request_overrides = 'a67612f8c59f4cb4a9723d8e0e0e7b7cb8c5c3d45e3d8c4f5e6f7e8f9a0b1c2d', request_overrides or {}
if not js_pack: return fallback_hash
try:
(resp := session.get(js_pack, headers=SpotifyMusicClientUtils.COMMON_HEADERS, **request_overrides)).raise_for_status()
raw_hashes = resp.text; str_mapping, hash_mapping = SpotifyMusicClientPlaylistUtils.extractmappings(raw_hashes)
chunks = SpotifyMusicClientPlaylistUtils.combinechunks(str_mapping, hash_mapping)
for chunk in chunks:
chunk_url = f"https://open.spotifycdn.com/cdn/build/web-player/{chunk}"
try: raw_hashes += session.get(chunk_url, headers=SpotifyMusicClientUtils.COMMON_HEADERS, **request_overrides).text
except Exception: pass
return (m.group(1) if (m := re.search(r'"fetchPlaylist","(?:query|mutation)","([^"]+)"', raw_hashes)) else fallback_hash)
except Exception: return fallback_hash
'''fetchplaylist'''
@staticmethod
def fetchplaylist(session: requests.Session, access_token: str, client_token: str, client_version: str, playlist_id: str, js_pack: str, offset: int = 0, limit: int = 25, request_overrides: dict = None) -> dict:
request_overrides = request_overrides or {}
sha256_hash = SpotifyMusicClientPlaylistUtils.getsha256hash(session, js_pack, request_overrides=request_overrides)
payload = {"operationName": "fetchPlaylist", "variables": {"uri": f"spotify:playlist:{playlist_id}", "offset": offset, "limit": limit, "enableWatchFeedEntrypoint": False}, "extensions": {"persistedQuery": {"version": 1, "sha256Hash": sha256_hash}}}
headers = {'User-Agent': f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{SpotifyMusicClientUtils.BROWSER_VERSION}.0.0.0 Safari/537.36', 'Sec-Ch-Ua': f'"Chromium";v="{SpotifyMusicClientUtils.BROWSER_VERSION}", "Not(A:Brand";v="24", "Google Chrome";v="{SpotifyMusicClientUtils.BROWSER_VERSION}"', 'Authorization': f'Bearer {access_token}', 'Client-Token': client_token, 'Spotify-App-Version': client_version, 'Content-Type': 'application/json;charset=UTF-8'}
(resp := session.post('https://api-partner.spotify.com/pathfinder/v2/query', headers=headers, json=payload, **request_overrides)).raise_for_status()
return resp2json(resp=resp)
'''getalltracks'''
@staticmethod
def getalltracks(session: requests.Session, access_token: str, client_token: str, client_version: str, playlist_id: str, js_pack: str, request_overrides: dict = None) -> List[dict]:
tracks, offset, limit, request_overrides, playlist_result_first = [], 0, 343, request_overrides or {}, {}
while True:
playlist_result = SpotifyMusicClientPlaylistUtils.fetchplaylist(session, access_token, client_token, client_version, playlist_id, js_pack, offset, limit, request_overrides=request_overrides)
if not playlist_result_first: playlist_result_first = copy.deepcopy(playlist_result)
if not (content := safeextractfromdict(playlist_result, ['data', 'playlistV2', 'content'], {})): break
tracks.extend(content.get('items', [])); total_count = content.get('totalCount', 0)
if total_count <= offset + limit: break
offset += limit
return tracks, playlist_result_first
'''parse'''
@staticmethod
def parse(session: requests.Session, playlist_id: str, request_overrides: dict = None) -> dict:
session, request_overrides = session or requests.Session(), request_overrides or {}
try:
session_data = SpotifyMusicClientUtils.getsessiondata(session, request_overrides=request_overrides)
device_id, client_version, js_pack = session_data['deviceId'], session_data['clientVersion'], session_data['jsPack']
secret_data = SpotifyMusicClientUtils.getlatesttotpsecret(); totp = SpotifyMusicClientUtils.generatetotp(secret_data['secret'])
token_data = SpotifyMusicClientUtils.getaccesstoken(session, totp, secret_data['version'], request_overrides=request_overrides)
access_token, client_id = token_data['accessToken'], token_data['clientId']; client_token = SpotifyMusicClientUtils.getclienttoken(session, client_version, client_id, device_id, request_overrides=request_overrides)
tracks, playlist_result_first = SpotifyMusicClientPlaylistUtils.getalltracks(session, access_token, client_token, client_version, playlist_id, js_pack, request_overrides=request_overrides)
for item in tracks: uri: str = safeextractfromdict(item, ['itemV2', 'data', 'uri'], None); item['id'], item['song_link'] = uri.split(':')[2], f"https://open.spotify.com/track/{uri.split(':')[2]}"
return tracks, playlist_result_first
except Exception: return [], {}
'''SpotifyMusicClientSearchUtils'''
class SpotifyMusicClientSearchUtils():
'''query'''
@staticmethod
def query(session: requests.Session, payload: dict, request_overrides: dict = None) -> dict:
session, request_overrides = session or requests.Session(), request_overrides or {}
session_data = SpotifyMusicClientUtils.getsessiondata(session, request_overrides=request_overrides)
device_id, client_version = session_data['deviceId'], session_data['clientVersion']
secret_data = SpotifyMusicClientUtils.getlatesttotpsecret(); totp = SpotifyMusicClientUtils.generatetotp(secret_data['secret'])
token_data = SpotifyMusicClientUtils.getaccesstoken(session, totp, secret_data['version'], request_overrides=request_overrides)
access_token, client_id = token_data['accessToken'], token_data['clientId']; client_token = SpotifyMusicClientUtils.getclienttoken(session, client_version, client_id, device_id, request_overrides=request_overrides)
headers = {'User-Agent': f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{SpotifyMusicClientUtils.BROWSER_VERSION}.0.0.0 Safari/537.36', 'Sec-Ch-Ua': f'"Chromium";v="{SpotifyMusicClientUtils.BROWSER_VERSION}", "Not(A:Brand";v="24", "Google Chrome";v="{SpotifyMusicClientUtils.BROWSER_VERSION}"', 'Authorization': f'Bearer {access_token}', 'Client-Token': client_token, 'Spotify-App-Version': client_version, 'Content-Type': 'application/json;charset=UTF-8'}
(resp := session.post("https://api-partner.spotify.com/pathfinder/v2/query", json=payload, headers=headers, **request_overrides)).raise_for_status()
return resp2json(resp=resp)
'''searchbykeyword'''
@staticmethod
def searchbykeyword(session: requests.Session, query: str, limit: int, offset: int, rule: dict = None, request_overrides: dict = None) -> list:
request_overrides, rule = request_overrides or {}, rule or {}
(payload := {"variables": {"searchTerm": query, "offset": offset, "limit": limit, "numberOfTopResults": 5, "includeAudiobooks": True, "includeArtistHasConcertsField": False, "includePreReleases": True, "includeAuthors": False}, "operationName": "searchDesktop", "extensions": {"persistedQuery": {"version": 1, "sha256Hash": "fcad5a3e0d5af727fb76966f06971c19cfa2275e6ff7671196753e008611873c"}}}).update(rule)
return SpotifyMusicClientSearchUtils.query(session, payload, request_overrides=request_overrides)
catalog-sync/musicdl/modules/utils/tidalutils.py
+1156
View File
File diff suppressed because it is too large Load Diff
catalog-sync/musicdl/modules/utils/youtubeutils.py
+3609
View File
File diff suppressed because it is too large Load Diff