From 73799f8a6e71036ed11e91f564950272bb4fadce Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Thu, 29 Oct 2026 12:07:30 -0400 Subject: [PATCH 1/2] feat: download models from our mirror first and verify SHA-256 (v0.48.0) Model files were fetched from the UVR repo (TRvlvr/model_repo) first even when our model-configs release had an identical copy, and nothing was verified apart from Demucs's filename checksums. - Try the audio-separator model-configs release first for every file, falling back to the original source (UVR repo, Meta's CDN). VIP models still come from Anjok07's repo first. - Ship model-checksums.json: SHA-256 for all 243 published model files, covering every file the library can download. Downloads that don't match are discarded and the next source is tried; cached files are checked once per process and replaced if they don't match. - Download to a .partial file and rename when complete, so an interrupted download no longer leaves a truncated model that later runs treat as present. - tools/update-model-checksums.py regenerates the manifest after new uploads (only hashes files it doesn't already know). Hashes were streamed from all three release repos plus Meta's CDN on the build host: every byte count matches GitHub's listed size, the 89 files published in more than one place are byte-identical everywhere, and the Demucs hashes match the checksums embedded in their filenames. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012RYLVTA9CYSnmncNv3GRHJ --- README.md | 6 + audio_separator/model-checksums.json | 248 ++++++++++++++++++++ audio_separator/separator/separator.py | 122 +++++++--- pyproject.toml | 2 +- tests/unit/test_model_download_integrity.py | 149 ++++++++++++ tools/update-model-checksums.py | 108 +++++++++ 6 files changed, 599 insertions(+), 36 deletions(-) create mode 100644 audio_separator/model-checksums.json create mode 100644 tests/unit/test_model_download_integrity.py create mode 100644 tools/update-model-checksums.py diff --git a/README.md b/README.md index d7c031ec..58ba5178 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,12 @@ For programmatic use, you can output the model list in JSON format: audio-separator -l --list_format=json ``` +### Where Models Are Downloaded From + +Model files are downloaded on first use into `model_file_dir` (or `AUDIO_SEPARATOR_MODEL_DIR`). Each file is fetched from the [audio-separator models release](https://github.com/nomadkaraoke/python-audio-separator/releases/tag/model-configs) first, with the original source (the UVR model repo, or Meta's CDN for Demucs) as a fallback. UVR VIP models are still fetched from Anjok07's repo first. + +Every model file must match the SHA-256 published in [`audio_separator/model-checksums.json`](audio_separator/model-checksums.json). A download that doesn't match is discarded and the next source is tried. A file already in the model directory is checked too, and replaced if it doesn't match. Maintainers who upload new model files should regenerate the manifest with `tools/update-model-checksums.py`. + ### Processing Large Files For very long audio files (>1 hour), you may encounter out-of-memory errors. The `--chunk_duration` option automatically splits large files into smaller chunks, processes them separately, and merges the results: diff --git a/audio_separator/model-checksums.json b/audio_separator/model-checksums.json new file mode 100644 index 00000000..d75f78e8 --- /dev/null +++ b/audio_separator/model-checksums.json @@ -0,0 +1,248 @@ +{ + "_comment": "SHA-256 of every published model file Separator downloads, keyed by filename. Regenerate with tools/update-model-checksums.py.", + "sha256": { + "04573f0d-f3cf25b2.th": "f3cf25b222c4eed7cd49dd8b2c9597d50c18bd154090f7b919cfa5f93cf22c49", + "10_SP-UVR-2B-32000-1.pth": "a178d8a8bc2c992bc5e5a045ef5ab8706d55c375d1bfe34700c1946d6c2d28b6", + "11_SP-UVR-2B-32000-2.pth": "9e80e69ecd2244496710fa77e7d2c785bad93e2a87a7def65c38272cc9fd1613", + "12_SP-UVR-3B-44100.pth": "5b826e6a44dbe007bde36c720bb2afeed008348cef1b61e35898fee5282d3918", + "13_SP-UVR-4B-44100-1.pth": "afb84b3d1da4a7e24033e5a0e85dbde2ac5f5e4b1e839788748735ac3bcc5046", + "14_SP-UVR-4B-44100-2.pth": "78404d85d6e4e11a3d2362f6577fc398f77406b7881fbac402e0c6191d6efca1", + "15_SP-UVR-MID-44100-1.pth": "bd03d4061ce34569e8df9e0a13a235af7ff0d3c0c3def491db4439f9ff344dd0", + "16_SP-UVR-MID-44100-2.pth": "f0e5025987670f963a339db861b078d8526bf912f0ce2e00f732a7f0a3e846cc", + "17_HP-Wind_Inst-UVR.pth": "acc6d472b4b478da9c9ab5af45b167749e05a7f65b30c7d5988b3700a513aeee", + "1_HP-UVR.pth": "d9c654305888fe09668078ade76962b32d3782848f158192da7403a650e935a6", + "2_HP-UVR.pth": "26d6400d2d2e90811ba2381828c8c3519196097375af524c3d0edd969acf5356", + "3_HP-Vocal-UVR.pth": "7d3f9be265b2f2c3d684f23a9905674627c2552e0f6b5f36325cef8dc1f0ff6b", + "4_HP-Vocal-UVR.pth": "295d1e1e3a52c17a040a4a6fe390214ad8ca0797f091b7c6d8db97d247a27271", + "5_HP-Karaoke-UVR.pth": "fe00891defbb61f4261500af22f7624f1a3df8dc75fa3998d1aece02e6be4537", + "5c90dfd2-34c22ccb.th": "34c22ccb381c6f9fdbf324f04e1e2fe21aaaf293f5ded163a162697ff9a02ddd", + "6_HP-Karaoke-UVR.pth": "4ce7eaaa9e56f09366b788aebf6d3a72aec8145692c56f1e090e4e7e2d7ce65f", + "75fc33f5-1941ce65.th": "1941ce654b11df4132b9f4eae408556b4c83fad6fe26b4bc0dbcb36b975befb3", + "7_HP2-UVR.pth": "545ee733b704a263bf6f75328c5ed14f0fb439dcda7a6e8f53c8cecb92877afe", + "8_HP2-UVR.pth": "e072299a87cf3fcdd36da6adc6d2837bed0c07a208775a7678c3d5deeea58f79", + "92cfc3b6-ef3bcb9c.th": "ef3bcb9c8b40d14ae5d51b6db2587339cc12c6b77c0be151ce6d69002e087bf2", + "955717e8-8726e21a.th": "8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4", + "9_HP2-UVR.pth": "a99d032ed8c9d59077a8a41f0ceccb279d59ec10c87102167a283151d25ad9a2", + "BS-Roformer-SW.ckpt": "24e7d35ee9c64415673d3fd33e06a67cac2c103c5df6267ba1576459c775916e", + "BS-Roformer-SW.yaml": "b558996f1e25eb48798bd6502505a5de94c4f966d6edfb1a0420f06cc40b501a", + "Kim_Inst.onnx": "86b1940e7122fbdd2beadc65507cbff6c352d79012a8a7e60d56db98532af5f7", + "Kim_Vocal_1.onnx": "f313140ef8fecc3041881b60ecb993d985a0281a138b2fb634aa8901aebc38cb", + "Kim_Vocal_2.onnx": "ce74ef3b6a6024ce44211a07be9cf8bc6d87728cc852a68ab34eb8e58cde9c8b", + "MDX23C-8KFFT-InstVoc_HQ.ckpt": "49d51472769e34a2501cd1da782346a3212555c3a5619fc2c53507445528d816", + "MDX23C-8KFFT-InstVoc_HQ_2.ckpt": "7d960d8e40a458120412c1bd807e013d2dbca7b959cc9da2bbcb0eb203d1daea", + "MDX23C-De-Reverb-aufr33-jarredou.ckpt": "eae2471b707758d74db38ac1b1d5800e12f57c4e9d1ebbb2faf004b8e086e914", + "MDX23C-DrumSep-aufr33-jarredou.ckpt": "d2a4aa53eb584d21eead358a4e66d1882ad182911be018f052b5da73be9096d0", + "MDX23C_D1581.ckpt": "d7d4d2137f12764950205b095da20032fef1d41f077bacc8582f20ed40e8cb28", + "MGM_HIGHEND_v4.pth": "67bf5241072ae938ae0e6bec44d2d8307d9db81a5299382a5cfe12188131f9ce", + "MGM_LOWEND_A_v4.pth": "c5def7a35afecde33a77db214b3e9eeff0e47e8329ff5522d33332bc02b4b0e3", + "MGM_LOWEND_B_v4.pth": "93d6a08abf6357e5f37d4eb6892267520f5bb9e90f45486b08a0857d5fcb2ec8", + "MGM_MAIN_v4.pth": "0e6f0c0592333a3b215f61ac1e01f6c24c059f903f0789cf634e92daffae1dce", + "MelBandRoformerBigSYHFTV1.ckpt": "e2327e3e81f19e67c307f8c830c54267c09ecb0e9c6ad2b40a80c310899c955f", + "MelBandRoformerSYHFT.ckpt": "f319dfcde4396ea3106658f457f5eb0bc577e113491f61ae8bab216fe84b0c0c", + "MelBandRoformerSYHFTV2.5.ckpt": "916e3a2c1e63b1457bcad823b98ca705e4933deffd2a5ab3a370e10f68bf47e2", + "MelBandRoformerSYHFTV2.ckpt": "2e99f8efa5315300c197295592bd7e56c21c1d77e1884c904b5128c54a2a4632", + "MelBandRoformerSYHFTV3Epsilon.ckpt": "4c886092e4aae13aa089263a0d54d483643f58c16ec221aed37268e2c1031397", + "Reverb_HQ_By_FoxJoy.onnx": "233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80", + "UVR-BVE-4B_SN-44100-1.pth": "56165bdfa0dd5df7930ff76652200dc287e1bfef622bba5c812c736dc22067fe", + "UVR-BVE-4B_SN-44100-2.pth": "9bc591f9d91151317589cf93c7853449af8bbb7a82de3dcea2fff4296904efbc", + "UVR-De-Echo-Aggressive.pth": "1bd1d79d9c5d1b17d20f96f8a9f8aff1b55a83014f70712446bf420c0188e0a0", + "UVR-De-Echo-Normal.pth": "b849dd575643b075c257fb7a96c2ef5a79d7a5e7df74a2b319ad47118f1ee769", + "UVR-De-Reverb-aufr33-jarredou.pth": "fff726b0adf09f2eadc2151645557db81e1a01fe15d91f9bbbfa9b50a007f1fd", + "UVR-DeEcho-DeReverb.pth": "e644028ec82865dc0fe082bc6fea85a43f7c71cfe375caee2da2d154aa661ee7", + "UVR-DeNoise-Lite.pth": "0023492fe98c406817b5253965de19ede65d1c147db015a3a428f07602e99571", + "UVR-DeNoise.pth": "5addf43ece5bddd18da9f575a02d7ffdb32342414e6ad7ac8d1dd7a04138a628", + "UVR-DeReverb-aufr33-jarredou_4band_v4_ms_fullband.pth": "fff726b0adf09f2eadc2151645557db81e1a01fe15d91f9bbbfa9b50a007f1fd", + "UVR-MDX-NET-Inst_1.onnx": "6ca53f94b7a0cbb04fcfcc8f3ea5ec1ae22cd8ad044f5e673588859f83976f5e", + "UVR-MDX-NET-Inst_2.onnx": "e3a96a664d28b52db9def0a9cae9a16dbb524d8325bfe8f0ac64ac5d231456bc", + "UVR-MDX-NET-Inst_3.onnx": "2b7834e2972158d8c9864e7376e3a7d084079c80a23f38dc31c4b0a4e901a1cb", + "UVR-MDX-NET-Inst_HQ_1.onnx": "38a045c4ded87e3bf97b609ec5be7910e8a7cecec455f507227ab12b5e29f7f9", + "UVR-MDX-NET-Inst_HQ_2.onnx": "197f8ab296df850f961e68c595f6649acb7d9e621b5600b460f3458967299112", + "UVR-MDX-NET-Inst_HQ_3.onnx": "317554b07fe1ea5279a77f2b1520a41ea4b93432560c4ffd08792c30fddf9adc", + "UVR-MDX-NET-Inst_HQ_4.onnx": "3c4b5b9b05090fdf238f38ba5046813982d50e2a652e9cb3324ea79720c3c9c8", + "UVR-MDX-NET-Inst_HQ_5.onnx": "811cb24095d865763752310848b7ec86aeede0626cb05749ab35350e46897000", + "UVR-MDX-NET-Inst_Main.onnx": "8ab401dfe4a548b87deb64f975294bd56ff946aa32903f53b4b24bb13b2cce1e", + "UVR-MDX-NET-Inst_full_292.onnx": "020f6b65fa219fb7c285e4f3fc2863bf22daf03c4c93e547b6d13d5f2757a7ec", + "UVR-MDX-NET-Voc_FT.onnx": "534b2070fcc7df514b13ef660dc8cbb328679c2374d04354a5c42bb14ecce111", + "UVR-MDX-NET_Crowd_HQ_1.onnx": "313b7bf869c411fdafe005cf0d5a635c405cb3d0df137178a64091952d75225c", + "UVR-MDX-NET_Inst_187_beta.onnx": "c74566f3c3033cacba996328b2ee90bf77ef79ea6c35b7841df183b7906f54a5", + "UVR-MDX-NET_Inst_82_beta.onnx": "c6c268302f09ab53687072618e056a611272a7e2c3fd9b3b59164da152f3588e", + "UVR-MDX-NET_Inst_90_beta.onnx": "0d902868a46575aea6ee2335736ff3b53faf497a6bdaa1b864e0fd84eb1b42a5", + "UVR-MDX-NET_Main_340.onnx": "78792633b4007755af12ecde20f709b4f0b99563b1d25fe0a501ed2122aff218", + "UVR-MDX-NET_Main_390.onnx": "286c4f0847ca837e2c3f4c4058f756d5f150cbf080506aa6f33a2847aba92e8c", + "UVR-MDX-NET_Main_406.onnx": "f158816a44eef1f0ba0f48b813cbfcf460ed1c70a754af3609ade44aaf7d1b23", + "UVR-MDX-NET_Main_427.onnx": "95275802a27801b97e3c0552b6eaa69f9bb3bd7df53cdf0536cce0a753f702cc", + "UVR-MDX-NET_Main_438.onnx": "d5e1ad93587a163a0987a0168b99a2ad875c0d9bfc3afb596b7c36b09c7f5c26", + "UVR_Demucs_Model_1.yaml": "40dd1254be3d410a0ab18522c40f535b0a9fcb5691a7224449fae35b83f5b382", + "UVR_Demucs_Model_2.yaml": "4b0305adbbbdcc6a465099e9a7dc905e3c984a4bc6b59bd2baaef4429a70599c", + "UVR_Demucs_Model_Bag.yaml": "ddfd0cf4a872536b8b5bfe92ca40073a8dd3eb19edd9887f06a6a221350baf43", + "UVR_MDXNET_1_9703.onnx": "229ad3bb96a037e89d8ed86732d6d3675856e6a07c3e3f02896eac01ec7ee4be", + "UVR_MDXNET_2_9682.onnx": "1deb7295acd3206bc9582a5d92f1b0a74bf3f41c7c1fb78a0ac0123cde4372db", + "UVR_MDXNET_3_9662.onnx": "e02220e80d8253f4c2209f8924298b2b686bbdf2868b788ff5500fb9bd94aadc", + "UVR_MDXNET_9482.onnx": "f4f365207c56deb115bceedff3ad8fe98a751c745f9e370cecec6226b8b47184", + "UVR_MDXNET_KARA.onnx": "e3167c87333a48548413e972a286bf40bf5694001d2853861eb1435953f02d63", + "UVR_MDXNET_KARA_2.onnx": "bf32e15105a09c0f7dddd2b67346146334d6f3ecb399ed7638eba2ab07cbf5f4", + "UVR_MDXNET_Main.onnx": "8289784cda38543ff431add4070662813311a8cccfc0112ca82f76d9dba2b4ca", + "aspiration_mel_band_roformer_less_aggr_sdr_18.1201.ckpt": "83bfe991cec4fbadde9f30d1f79cd5293ad0b1f936256be327bba5cbb4883374", + "aspiration_mel_band_roformer_sdr_18.9845.ckpt": "9e791258c866c6c8da66052693d8cc3b64f1f42c01e052dbdc570cd278380cc5", + "bs_roformer_instrumental_resurrection_gabox.ckpt": "d28b3a422531e1ca482d29302a5b67614eb45c09d72ce954258011ce24ef0919", + "bs_roformer_instrumental_resurrection_unwa.ckpt": "16311025a5133ae6411760ccfe9e3e66b31a01d9d8bec0a03fa7ec4bedac7a15", + "bs_roformer_karaoke_anvuew.ckpt": "206d04757cb5f75ca3b55f8a0a48f5c26aa2351d4ff3c7adbfc9affa30ea3ae4", + "bs_roformer_karaoke_frazer_becruily.ckpt": "eb90ee24c1154d83fbcfd27e96182f19e061557cc6e4746953125e08c29389f9", + "bs_roformer_male_female_by_aufr33_sdr_7.2889.ckpt": "3cf11736d1b42a11ae55d8299316585921477dd2a671b24b663660846ca9861b", + "bs_roformer_vocals_gabox.ckpt": "18d58efe5e949e70fab11b875329af6d06ef11ccc29574bfe943fb57cc827f38", + "bs_roformer_vocals_resurrection_unwa.ckpt": "9dbfe5cb572e4ed32a15ec727d7bd06c8d7aba97509e6fda5bc008bb1e0b2dd5", + "bs_roformer_vocals_revive_unwa.ckpt": "f1d7e4bfdfef07c6b2bc1d65283a7d03c3c38f8c7dbc8d729b785f93c8b8699a", + "bs_roformer_vocals_revive_v2_unwa.ckpt": "58098850c882a7472dad39f99fb8040ce6eaafe671cfe9881d89aea276bbb5f5", + "bs_roformer_vocals_revive_v3e_unwa.ckpt": "1b0751b9a15c591407c3b77f08eb4ad3005e42e96051f3f2b39760f1130c467b", + "calculate-model-hashes.py": "24a1ddcdfccd074e2ae3dd488a5a9ab1de80f882fc355f31f60498324b040a96", + "checkpoint-multi_fixed.ckpt": "20bcd513dc7eb0541dd045909a4e7dff8dab474cc2efba4904101c76524aee85", + "config_aspiration_mel_band_roformer.yaml": "d24136a81ada622dd982fa64986e1ee689c01d44ff7b95b6f8c920c9337d3d48", + "config_bs_roformer_instrumental_resurrection_unwa.yaml": "d28c16f35e3118c8db3b5b28320d70e3fbb7ab46ab1ff1e60e0332b1e5871ba1", + "config_bs_roformer_karaoke_anvuew.yaml": "5cb3f127ecbc6a8e37f31ea7e05f60f360a44da43e857bde805b7b68558f6338", + "config_bs_roformer_karaoke_frazer_becruily.yaml": "1d3b58b473025183d0d3c91e7a444cb5f1418d9f34e8f46b8d2fb10d2cc8ab34", + "config_bs_roformer_vocals_gabox.yaml": "2bfdd16c656bd9519aba757cc4f8834b7ede675eb1e00ec4772d74ae1c41af7f", + "config_bs_roformer_vocals_resurrection_unwa.yaml": "7d6010fdc12a4919c886b099b2567ec475ae2fa8b0acf438bd83385bc0cb31d9", + "config_bs_roformer_vocals_revive_unwa.yaml": "6b9d5fb6aeda6b0941f937e4e4883643e3187331ff403fb820c7aa6af4b02dbb", + "config_chorus_male_female_bs_roformer.yaml": "363ef6bef5f0fd89b69e6e0f90dcc102dec6aaeb6e55790cae1796bebdf5097e", + "config_dereverb-echo_mel_band_roformer.yaml": "1271188ec57c383a6657f19ba5903509ed4699be6c26be12e224c775d903050a", + "config_dereverb-echo_mel_band_roformer_sdr_13.4843_v2.yaml": "9abd9b811b432beca3e6974a31eb3e9c2c72d3f4d7297473cce6a4721ba135c1", + "config_dereverb_echo_mel_band_roformer_v2.yaml": "b801c49dbc09372f85281127adde7ded28f475d98e9940efba4a2f45202b38f2", + "config_dereverb_echo_mel_band_roformer_v2.yaml.1": "b801c49dbc09372f85281127adde7ded28f475d98e9940efba4a2f45202b38f2", + "config_dereverb_echo_mel_band_roformer_v2.yaml.2": "b801c49dbc09372f85281127adde7ded28f475d98e9940efba4a2f45202b38f2", + "config_dereverb_mdx23c.yaml": "a0cf11216913ab8941afb96fa7ab333390d1740b4a74a5d2f4b81ca8a218c756", + "config_drumsep_mdx23c.yaml": "17d1649a227f841165bdb4c11a42082898192a1ea3ceab7e7e0b9293d6589dd6", + "config_mel_band_roformer_bleed_suppressor_v1.yaml": "bca5755de9946ded7fc37f36c1c9c213ed9f091fe0bc19c6fffaaa2ea7815972", + "config_mel_band_roformer_instrumental_becruily.yaml": "a222235643d329817a1123d82d8eb38392d919e90ed597e053de49c092d3b442", + "config_mel_band_roformer_instrumental_gabox.yaml": "723af6755b5624be0a58351a13c930c472b51ef677cf2c7943394fefed7c3d4d", + "config_mel_band_roformer_karaoke_becruily.yaml": "cd37b0dcc285fc22d88090415722ac7127ee1d9ea2f3346c3b8d8fcc61e0c74b", + "config_mel_band_roformer_karaoke_gabox.yaml": "16b726891076fd0bebab9ce038240b51afb22b04eac6026a7f0356178dcb65b7", + "config_mel_band_roformer_kim_ft_unwa.yaml": "c910a0b1493fd3f9cee7a576a7498e44e660510dcaaf5d0d50d5363dde1d0010", + "config_mel_band_roformer_vocal_fullness_aname.yaml": "0353adb4e799f971c6038bac7e5675b9dbf85e103ed1be648b67d04990654172", + "config_mel_band_roformer_vocals_becruily.yaml": "186792a36db251fc5834a883c519b609ca80abbd2bfc4668eea43ee3a4223985", + "config_mel_band_roformer_vocals_gabox.yaml": "32419582fa8c313199ab49be8f5cb2b5e3860769ca045b4788a960d72140cd39", + "config_melband_roformer_big_beta5e.yaml": "cc615390b1a38b80a0115f26c0fc29140cb59ab255fbda1380d9f466bd7d4201", + "config_melbandroformer_big_beta4.yaml": "464f21925d2744be6af64cf9ee78d5937d11cfde0b91f819cb1f631acccab603", + "config_melbandroformer_big_beta6.yaml": "14334637d4520bb4934df0815768a32845c9c1c5a1150429d1151e71864d6f24", + "config_melbandroformer_big_beta6x.yaml": "42a037e9d80dc1dbc91005047cd20f4afe7c1267163fc878a9fbaf6be21959f3", + "config_melbandroformer_inst.yaml": "64c0cf23730f299c9c73470e4ba4b32120d8fec4fb1a0326dde1d61378cc6fc8", + "config_melbandroformer_inst_v2.yaml": "4b902a7360a930c178edb4846b30e4e326aa1219d1b2daf660d46a311e0cd50b", + "config_melbandroformer_instvoc_duality.yaml": "62dbc3ecf29c7ac99df35003f8cb72da3348d646cb5e6d50e07323551c3d968f", + "config_vocals_mel_band_roformer_big_v1_ft.yaml": "cc615390b1a38b80a0115f26c0fc29140cb59ab255fbda1380d9f466bd7d4201", + "config_vocals_mel_band_roformer_ft.yaml": "d9ecc767ba148545de27dae65b8d2282297652067b6ffa5955d43d37f605408e", + "config_vocals_mel_band_roformer_kim.yaml": "32419582fa8c313199ab49be8f5cb2b5e3860769ca045b4788a960d72140cd39", + "d12395a8-e57c48e6.th": "e57c48e6b0e38af4f7118d7bd08c49f0a0c0edf7d09143bdd902ea0d237303e6", + "denoise_mel_band_roformer_aufr33_aggr_sdr_27.9768.ckpt": "a25e3b233722cd81e2de7b8e798a3fef29d4b9799ccacda60b0dc958a1e2a5bb", + "denoise_mel_band_roformer_aufr33_aggr_sdr_27.9768_config.yaml": "5d7d83b2e9d232da60941b717b0abdc345155d45cff3f79715cdb2790ba18c36", + "denoise_mel_band_roformer_aufr33_sdr_27.9959.ckpt": "7c1c39191edc34e942ca7f2346ce6b6c0e1208a5f76349ffce6f696bd12910de", + "denoise_mel_band_roformer_aufr33_sdr_27.9959_config.yaml": "5d7d83b2e9d232da60941b717b0abdc345155d45cff3f79715cdb2790ba18c36", + "dereverb-echo_mel_band_roformer_sdr_10.0169.ckpt": "cd2b737a394cfb80cd48cc9fcbaf89f5f4062f6b93066c2911617a06d8b7860a", + "dereverb-echo_mel_band_roformer_sdr_13.4843_v2.ckpt": "396432f5af25992fe82d0286634bd879027c073721db6ab10199e75459708b9f", + "dereverb_big_mbr_ep_362.ckpt": "0506455e74ffc02bbec700df9863ae243597034003815f1418227c6dee33b6ea", + "dereverb_echo_mbr_fused.ckpt": "1596b1063238f487d54a0510a8c92cb28c000c803a271dd618ac49efc99ef3f7", + "dereverb_mdx23c_sdr_6.9096.ckpt": "eae2471b707758d74db38ac1b1d5800e12f57c4e9d1ebbb2faf004b8e086e914", + "dereverb_mel_band_roformer_anvuew.yaml": "1599d9ea717ea2b5b3bc55d936f752b8d0f67baaa3de95acd6d03259a2f37784", + "dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt": "9262877b87e9ebb0fb808a456b0a411fa677f5df31c8383c1254af531c078970", + "dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt": "0db8f1b41c00cead1112e967262a12802fd32e76c0c3a8eb207e772bae25d07b", + "dereverb_mel_band_roformer_mono_anvuew.ckpt": "f099ee717eb57fb0ad5eb0e7c9ad6787c36168140b61ce2b158b90c2c4ecee79", + "dereverb_super_big_mbr_ep_346.ckpt": "26dda242bce4405555f2d6086d079fe8cc23f1f04e02e501d2689bfe3ece0489", + "deverb_bs_roformer_8_384dim_10depth.ckpt": "9c38653aaa5e49f2f7b84dd3be2b6b679e0cbea23978e6b48389ee6f0a914768", + "deverb_bs_roformer_8_384dim_10depth_config.yaml": "57c7d6b6742e2dc64d091892ae6fc1160410b365c39d1db9c9d1f772f3a4d8ce", + "ebf34a2d.th": "0556f2145e97e96d3c2a33c3879711e7559d7d261fd9773b20e2c580a3beffd0", + "ebf34a2db.th": "dbdd521017d706829716055df8a86e7ee5d05a590508488c82ffb551c6edb918", + "f7e0c4bc-ba3fe64a.th": "ba3fe64ae8ef66ac9a4857222ce48efbdc5eb3ad375cb79dd13debee5aaa4066", + "hdemucs_mmi.yaml": "be8b1c6ef2ca406c620917f3d94c15a0d240718604ce89bfa6765bcc52631f05", + "htdemucs.yaml": "239c445d0b14454d541ad8bd9bb271c9e536d267e8a4625208744cbb2e7bb66c", + "htdemucs_6s.yaml": "207405151270af8fd81c2373c25d27950916682ac91dca7884a11ce13dad6f58", + "htdemucs_ft.yaml": "69470b8c1bbd674437b51bc9fb491327a10ab0396b702c93389b9cf750016346", + "kuielab_a_bass.onnx": "0c3e77b9963185b1ea6bb46a4b8924137d9370fc1ccdefec7b1b416ef550dcaa", + "kuielab_a_drums.onnx": "40f586b7091934dd6f5563f0cba8f14bad57ce88440da1098bf388ea716c2901", + "kuielab_a_other.onnx": "7b67a1dcb5f232153528c59960b4c7bf8dc736b8114de360af0e719633f53358", + "kuielab_a_vocals.onnx": "daba83c2ee1afee9139766ad64c9b6808d6b6f092fff04bed3338be50baac721", + "kuielab_b_bass.onnx": "b4b7080fe501d0bece62076c5d4eda4d6590c5207ed78ec84a57bac0740a061d", + "kuielab_b_drums.onnx": "a6fecee758059b33ed99f6dabba297439b3e7cacfac4b1097bd324aff8052208", + "kuielab_b_other.onnx": "b0d0b63950ac332333fea2d58f68c92fd3ab0aae071398c2a8beeae1ad15b655", + "kuielab_b_vocals.onnx": "9b7dcb9d878acb0f3e64ff3fd27750faae96577013f6d50f5996875bf4250713", + "mdx.yaml": "aa6c377673bfa31dffc9b1de314c1c5335f96be18be869e4cc91f172aae1f4a4", + "mdx_extra.yaml": "5e079e0b6f9b5a6c737b5a5c95b590a292f0c7a683323c7d0cde3860a0828356", + "mdx_extra_q.yaml": "579d5faab585e3b55fce1d32d696975cdb176436f0fea8403961b146c67a32d0", + "mdx_q.yaml": "0b0b5877bda9c4c9917ad9301c50ceb8d3f7425880b2b81184540dcdf23ceafd", + "mel_band_roformer_bleed_suppressor_v1.ckpt": "a9a9d10faa7f8997676a78e66d741d7acb9cc449334763f3c8f626d68ec6e575", + "mel_band_roformer_crowd_aufr33_viperx_sdr_8.7144.ckpt": "ca8799531fe51c94172cc047226209ed48bf7d8c02e04671795a15d2a1c318af", + "mel_band_roformer_crowd_aufr33_viperx_sdr_8.7144_config.yaml": "029e5dce3abae9d9895ea19449c2b1fe749417437751676474affe255846d0de", + "mel_band_roformer_denoise_debleed_gabox.ckpt": "91aa7a546ed2e93482e4629c982d35b0d258bb3de6eeab497fd91658cc86c7fd", + "mel_band_roformer_instrumental_2_gabox.ckpt": "e03ca459c339f88b7521c367c897d0c3f5362b38a6cdb96cb28e625ca0f9931e", + "mel_band_roformer_instrumental_3_gabox.ckpt": "f9ec9f299cf617bf6afe1c382f4b0761cd9bee78323da94889951812328e10fb", + "mel_band_roformer_instrumental_becruily.ckpt": "a8da6632a1c25efb1c9be783ce9ea367d226d4b918cd6c3717c8b1d7a396041d", + "mel_band_roformer_instrumental_bleedless_v1_gabox.ckpt": "de972fb724601beef237abe94c8b934c73218e9baf3e344ab4c2333276e5cfe7", + "mel_band_roformer_instrumental_bleedless_v2_gabox.ckpt": "6109687febb8f18cd5a45207fee35f18ba8b9467b18a4b2e982a3b7dc04a9d72", + "mel_band_roformer_instrumental_bleedless_v3_gabox.ckpt": "5578545e094e584835b3184310ed1b12072f15d4b6ed8f4359ecc17358a66676", + "mel_band_roformer_instrumental_fullness_noise_v4_gabox.ckpt": "f50296e913b9af3b5b3b961e92877ef0d4a74f9a433e796e89960c4c2b1abe53", + "mel_band_roformer_instrumental_fullness_v1_gabox.ckpt": "31140eccf271d2a9e8a538b092b1f70dfd6471aa5ad163b22bccc758b9f38b62", + "mel_band_roformer_instrumental_fullness_v2_gabox.ckpt": "c446c34551333dd3d45b8d0708658a10f28c5e289f8ec27b5f0e22803681bef3", + "mel_band_roformer_instrumental_fullness_v3_gabox.ckpt": "fbb229209a8942d34664e19d2f4862e357ea3108a4e8c04b69aa0aba523a4481", + "mel_band_roformer_instrumental_fullness_v4_gabox.ckpt": "b03d9bf0fc643dabf006dee38fd798140b4884c1b65c573b4488c1a2a876ad84", + "mel_band_roformer_instrumental_fv7z_gabox.ckpt": "ef229f0dcd370c1767e4292981c59e5248814da45f32bfacebcc0f28adaa30a1", + "mel_band_roformer_instrumental_fv8_gabox.ckpt": "50bfa127d21f419e0da89730867d28c7ac4484c9473e6f313d036bc8b429df80", + "mel_band_roformer_instrumental_fv8b_gabox.ckpt": "5a28f79ae38b92caaeb37c5ccdf3912316b135414353ec0d5553d507e3de8ef7", + "mel_band_roformer_instrumental_fvx_gabox.ckpt": "545ef13b0cdbac505818a38db98e09c54e7c03ea17b4e0c895a531bfa352fa59", + "mel_band_roformer_instrumental_gabox.ckpt": "7b3ad6bd8bed3aaaa4d9320ea2ca910d140196a2302186db1754f3a8d8e16fb1", + "mel_band_roformer_instrumental_instv5_gabox.ckpt": "38e935cf1e97afcc1de84d0bdb87dd8090bad530fa0df28e707d16448e1d38e2", + "mel_band_roformer_instrumental_instv5n_gabox.ckpt": "175203923fac3e52ae00e7e37d41e8a7fef5020b6ee4e4144f4786daabc54b34", + "mel_band_roformer_instrumental_instv6_gabox.ckpt": "677951b8556a27abe32e39705640638826e78101fa901a51ad73d20522be6d25", + "mel_band_roformer_instrumental_instv6n_gabox.ckpt": "802f3e5d183d7c4b50dea147c320e61634f5be6ff55fa899fdebeaf0f3cf7f42", + "mel_band_roformer_instrumental_instv7_gabox.ckpt": "1b4f12d81fd7fb02f38609216d59f0e42b3dca655fd90ca275ab5321b3e4d9aa", + "mel_band_roformer_instrumental_instv7n_gabox.ckpt": "b0ca36af5d1314be46b56c8a53b6be02f98511fa5d7e3e196fd895755e65be3c", + "mel_band_roformer_instrumental_instv8_gabox.ckpt": "4d7d97418012e1d241853260d82330869c3e945ffdef9d7841fd90f5b24f20ff", + "mel_band_roformer_instrumental_instv8n_gabox.ckpt": "25920c876e601d4ccbf1684b19ecdac4b9fcfcc7f48d2c095d81040ec3fecbea", + "mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt": "1de20d459332fe8869aeb01327a31df0032262706e1365114e852dc271779813", + "mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956_config.yaml": "b35077d94861f068097cce1a5e54633c055e7dcc2613eade4e4dc7c7c9c3f48b", + "mel_band_roformer_karaoke_becruily.ckpt": "d3aa262ac01df870b9fc033e9c7b6cad33fe04fc9c148b6c40841326a515a0e0", + "mel_band_roformer_karaoke_gabox.ckpt": "303fc631e7aa587e9dc1e6ac4bb3667c6ba53aacb6b6a90abcfcf57935b92bd8", + "mel_band_roformer_karaoke_gabox_v2.ckpt": "ec34be50327aeaf1a996c27977f5c30d1ac80c0076d69683d3e5184c31ea29d3", + "mel_band_roformer_kim_ft2_bleedless_unwa.ckpt": "3c450bd66a98b49dd03231fc5ebb84121eef8418236b179423c2b171d62b04d9", + "mel_band_roformer_kim_ft2_unwa.ckpt": "5ed7b9e4c2eebbec7a7e5e8113058f7b68ba5e6048db8eaccfbbeb884c7884c0", + "mel_band_roformer_kim_ft3_unwa.ckpt": "1b10db0095c42576a32aeb3a1c8054af9dea0333e1c38477091d78316a007e52", + "mel_band_roformer_kim_ft_unwa.ckpt": "e6bd8d333880191254a6ef6be3cb0ffa4dda9d3282e36b0cce2e88a660e00d39", + "mel_band_roformer_vocal_fullness_aname.ckpt": "a64a27a672b457de23d9decd1fc7b58b0664a9f4f24bb43af154708e2ef07d2f", + "mel_band_roformer_vocals_becruily.ckpt": "a05961310cc55fbb901290c2e8be02682942f73522b6ac76bf2ec11e347ed95a", + "mel_band_roformer_vocals_fv1_gabox.ckpt": "c4dff354d81152d1b4321f6491f242c060919148239fbfe22a1015513de4a7fe", + "mel_band_roformer_vocals_fv2_gabox.ckpt": "2888813aa5b519941fa8548efc5a4331d63c61909007eb17fe95c367be230196", + "mel_band_roformer_vocals_fv3_gabox.ckpt": "49d81446b34a7848446efde7898b25bdc32fe872c2393617acb5356649f7ea93", + "mel_band_roformer_vocals_fv4_gabox.ckpt": "1a9657de5fd3ed87ad4fd1a9d2069743ecb33424836973ad0f3288e2a64e90bc", + "mel_band_roformer_vocals_fv5_gabox.ckpt": "6ede0504ddc55cb44b966a8212dac75a364f8157974cc40c8e92b9f5d4f17ce2", + "mel_band_roformer_vocals_fv6_gabox.ckpt": "25033d944288cb032fc51faab044bbd7f90bb81e82cada637ecdec699c2ff773", + "mel_band_roformer_vocals_fv7b_gabox.ckpt": "6bb98488b80e7797e6f447c0dd8f89f2b7040eebc5e839b5a59871b7a3a19265", + "mel_band_roformer_vocals_gabox.ckpt": "ff802a67501fac70587c3ff4e8dbc89c2558e7d8911c92222dfea2aaac208517", + "mel_band_roformer_vocals_v2_gabox.ckpt": "fac81dbebc0992503df55110d64d86c4fb74a1529527a819a253f3d20ef72bc1", + "melband_roformer_big_beta4.ckpt": "700a9bd3831d4f7f44cc0019b238774e31045bcbc361fbb69235535c40fc1454", + "melband_roformer_big_beta5e.ckpt": "32b876e1163716a9a007438b5a5107069586aa9b9ca653a5f63013b1edf6920c", + "melband_roformer_big_beta6.ckpt": "f51cbb94b4ed5c36cb36fd2024236a8af3ed6886567981702ad6f094b2c6c820", + "melband_roformer_big_beta6x.ckpt": "e16d702f4e20f13d60b293541c1dea75cb4414a5846b36780e28ef70352a4e5c", + "melband_roformer_inst_v1.ckpt": "6f88d96958b2b7dec32286b0ced00bbcbd37e28741cad9038758b1eaf9b5c057", + "melband_roformer_inst_v1_plus.ckpt": "25fe6ab4db95a6c20468f6c082f9bfc30904b8727ed8d069110bd0960da4879a", + "melband_roformer_inst_v1e.ckpt": "df2bcdb8838b88264f5381dbb0ccd84a9926c9775cf548c34d8846f5cd20fe96", + "melband_roformer_inst_v1e_plus.ckpt": "6a4ddba739f0352407fb6e18b29206b82318ec427fe37fcedb0f83241e4e15fb", + "melband_roformer_inst_v2.ckpt": "bd19766620f7d6f58fdf7aaada7e89907fe41bc64490ce3faa9a6dab15d6e1f2", + "melband_roformer_instvoc_duality_v1.ckpt": "4640a59d454bf9f69d67460592ab71e7cdce3afa0c0a6f0cf4500bb4ac0b8381", + "melband_roformer_instvox_duality_v2.ckpt": "b4a69558708f2857e36ac86a0e03ed95c4e3d8b9c5b8113963987d0d7df7e20f", + "model_2_stem_061321.yaml": "d4caa0f42e619d22a250e83f59fcef99b65986be738f2642833c0f52e5b70933", + "model_2_stem_full_band_8k.yaml": "451765e869b78dcb9ca9188a74da31f581b7254ff0e8b532aa76b974148de947", + "model_bandit_plus_dnr_sdr_11.47.ckpt": "c48284779f7d1258a6527d3aaa18a532d45c1f506e2dcc25d5ab179a8c5e2573", + "model_bs_roformer_ep_317_sdr_12.9755.ckpt": "5b84f37e8d444c8cb30c79d77f613a41c05868ff9c9ac6c7049c00aefae115aa", + "model_bs_roformer_ep_317_sdr_12.9755.yaml": "2bfdd16c656bd9519aba757cc4f8834b7ede675eb1e00ec4772d74ae1c41af7f", + "model_bs_roformer_ep_368_sdr_12.9628.ckpt": "f6c94864adfb73bbb0ca58ec14d58dd0b364549e9fb61433ae51916f3e2f8d0b", + "model_bs_roformer_ep_368_sdr_12.9628.yaml": "aea599b3f9bd4892a9c6bf5ac7c44787d3c99f717903d16054702665d477c86b", + "model_bs_roformer_ep_937_sdr_10.5309.ckpt": "a2e825a03bc908cb04dbd88eddeefbf5147dd1cf1f95cebf453d9dbfabec494b", + "model_bs_roformer_ep_937_sdr_10.5309.yaml": "302b6cee54adf39743b097b145ad4f64c37f3bd31b84791da32f963fb3692d04", + "model_chorus_bs_roformer_ep_267_sdr_24.1275.ckpt": "123c00786bdbc6bd462dddb35cd21fd6ae99ab8319f93f63a8abc1012e593d94", + "model_mel_band_roformer_ep_3005_sdr_11.4360.ckpt": "21b9d0958e35b8ebfbe2afe69bbd5444e5ffe2f5d80ae0d583b833d2f3c0d139", + "model_mel_band_roformer_ep_3005_sdr_11.4360.yaml": "d9b083b48dfdd0bd10f8a29a9c18777b0419496d938827f48a1db31bf0193aa3", + "new.txt": "904bcc59e61620cfee09a67e5d752beb03e40f0189cde92085655c9b9c0a9819", + "repro_mdx_a.yaml": "28932e1c47d4750b183e73bbfa537b699693857f858403440075ae79a307e90a", + "repro_mdx_a_hybrid_only.yaml": "25ebdda777fbce91881e723f1fdb98a0e746638ac7c46100b74259669c509cfc", + "repro_mdx_a_time_only.yaml": "43a3fada0508ab5097405df94ca5121125c4c09ae6482598b8251304dbbe93de", + "sync-to-github.py": "ffd6387e4441ea6831022346f09a5be5dc300443a39c684913d987742b536d6e", + "vocals_mel_band_roformer.ckpt": "87201f4d31afb5bc79993230fc49446918425574db48c01c405e44f365c7559e", + "vocals_mel_band_roformer.yaml": "b958b29c8f7195f0d86bee6759a33980db675c4ecaf2fcaa80fa125828e6cd38" + } +} diff --git a/audio_separator/separator/separator.py b/audio_separator/separator/separator.py index ece2f10a..cfea2720 100644 --- a/audio_separator/separator/separator.py +++ b/audio_separator/separator/separator.py @@ -277,6 +277,9 @@ def __init__( self.model_is_uvr_vip = False self.model_friendly_name = None + self._model_checksums = None + self._verified_model_files = set() + if not info_only: self.setup_accelerated_inferencing_device() @@ -552,14 +555,18 @@ def get_model_hash(self, model_path): self.logger.error(f"Error calculating hash for {model_path}: {e}") raise # Re-raise other errors - def download_file_if_not_exists(self, url, output_path): + def download_file_if_not_exists(self, url, output_path, verify_checksum=False): """ This method downloads a file from a given URL to a given output path, if the file does not already exist. + With verify_checksum, both an existing file and a fresh download must match model-checksums.json. """ if os.path.isfile(output_path): - self.logger.debug(f"File already exists at {output_path}, skipping download") - return + if not verify_checksum or self.verify_model_file_checksum(output_path): + self.logger.debug(f"File already exists at {output_path}, skipping download") + return + self.logger.warning(f"Existing file {output_path} does not match its published SHA-256 checksum; re-downloading it") + os.remove(output_path) self.logger.debug(f"Downloading file from {url} to {output_path} with timeout 300s") response = requests.get(url, stream=True, timeout=300) @@ -568,14 +575,72 @@ def download_file_if_not_exists(self, url, output_path): total_size_in_bytes = int(response.headers.get("content-length", 0)) progress_bar = tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) - with open(output_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - progress_bar.update(len(chunk)) - f.write(chunk) - progress_bar.close() + # Download beside the target and rename once complete, so an interrupted + # download never leaves a truncated file that later runs treat as present. + partial_path = f"{output_path}.partial" + try: + with open(partial_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + progress_bar.update(len(chunk)) + f.write(chunk) + progress_bar.close() + if verify_checksum and not self.verify_model_file_checksum(partial_path, filename=os.path.basename(output_path)): + raise RuntimeError(f"Downloaded file from {url} does not match its published SHA-256 checksum") + os.replace(partial_path, output_path) + finally: + if os.path.exists(partial_path): + os.remove(partial_path) else: raise RuntimeError(f"Failed to download file from {url}, response code: {response.status_code}") + def download_file_from_first_available(self, urls, output_path): + """Download output_path from the first URL that serves a file passing checksum verification.""" + errors = [] + for url in urls: + try: + self.download_file_if_not_exists(url, output_path, verify_checksum=True) + return + except (RuntimeError, requests.RequestException) as e: + self.logger.debug(f"Could not get {os.path.basename(output_path)} from {url}: {e}") + errors.append(f"{url}: {e}") + raise RuntimeError(f"Failed to download {os.path.basename(output_path)} from any source: " + "; ".join(errors)) + + def verify_model_file_checksum(self, path, filename=None): + """ + Check a model file against the SHA-256 published in model-checksums.json. + Returns True when it matches or when no checksum is published for that filename. + Files are hashed once per process unless their size or mtime changes. + """ + filename = filename or os.path.basename(path) + expected = self.get_model_checksums().get(filename) + if expected is None: + self.logger.debug(f"No published checksum for {filename}, skipping verification") + return True + + stat = os.stat(path) + cache_key = (os.path.abspath(path), stat.st_size, stat.st_mtime_ns) + if cache_key in self._verified_model_files: + return True + + digest = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1024 * 1024), b""): + digest.update(block) + + if digest.hexdigest() != expected: + self.logger.error(f"SHA-256 mismatch for {filename}: expected {expected}, got {digest.hexdigest()}") + return False + + self._verified_model_files.add(cache_key) + return True + + def get_model_checksums(self): + """Load the {filename: sha256} map of published model files bundled with the package.""" + if self._model_checksums is None: + with resources.open_text("audio_separator", "model-checksums.json") as f: + self._model_checksums = json.load(f)["sha256"] + return self._model_checksums + def list_supported_model_files(self): """ This method lists the supported model files for audio-separator, by fetching the same file UVR uses to list these. @@ -783,37 +848,24 @@ def download_model_files(self, model_filename): self.model_friendly_name = model_friendly_name self.print_uvr_vip_message() - # Download each required file for this model + # Download each required file for this model. The audio-separator models repo + # mirrors nearly every public file, so it is tried first and the original + # source is the fallback. VIP models are still fetched from Anjok07's repo first. for file_to_download in model_info["download_files"]: - # For URLs, extract just the filename portion if file_to_download.startswith("http"): filename = file_to_download.split("/")[-1] - download_path = os.path.join(self.model_file_dir, filename) - self.download_file_if_not_exists(file_to_download, download_path) - continue - - download_path = os.path.join(self.model_file_dir, file_to_download) - - # For MDXC models, handle YAML config files specially - if model_type == "MDXC" and file_to_download.endswith(".yaml"): + upstream_url = file_to_download + elif model_type == "MDXC" and file_to_download.endswith(".yaml"): + filename = file_to_download yaml_config_filename = file_to_download - try: - yaml_url = f"{model_repo_url_prefix}/mdx_model_data/mdx_c_configs/{file_to_download}" - self.download_file_if_not_exists(yaml_url, download_path) - except RuntimeError: - self.logger.debug("YAML config not found in UVR repo, trying audio-separator models repo...") - yaml_url = f"{audio_separator_models_repo_url_prefix}/{file_to_download}" - self.download_file_if_not_exists(yaml_url, download_path) - continue - - # For regular model files, try UVR repo first, then audio-separator repo - try: - download_url = f"{model_repo_url_prefix}/{file_to_download}" - self.download_file_if_not_exists(download_url, download_path) - except RuntimeError: - self.logger.debug("Model not found in UVR repo, trying audio-separator models repo...") - download_url = f"{audio_separator_models_repo_url_prefix}/{file_to_download}" - self.download_file_if_not_exists(download_url, download_path) + upstream_url = f"{model_repo_url_prefix}/mdx_model_data/mdx_c_configs/{file_to_download}" + else: + filename = file_to_download + upstream_url = f"{model_repo_url_prefix}/{file_to_download}" + + mirror_url = f"{audio_separator_models_repo_url_prefix}/{filename}" + candidate_urls = [upstream_url, mirror_url] if self.model_is_uvr_vip else [mirror_url, upstream_url] + self.download_file_from_first_available(candidate_urls, os.path.join(self.model_file_dir, filename)) return model_filename, model_type, model_friendly_name, model_path, yaml_config_filename diff --git a/pyproject.toml b/pyproject.toml index b1169c4c..19c1e986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dml = ["onnxruntime-directml>=1.17", "torch_directml"] [tool.poetry] requires-poetry = ">=2.0.0" -version = "0.47.2" +version = "0.48.0" description = "Easy to use audio stem separation, using various models from UVR trained primarily by @Anjok07" authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/test_model_download_integrity.py b/tests/unit/test_model_download_integrity.py new file mode 100644 index 00000000..f76f86e3 --- /dev/null +++ b/tests/unit/test_model_download_integrity.py @@ -0,0 +1,149 @@ +"""Model downloads come from our mirror first and must match their published SHA-256.""" + +import hashlib +import json +from importlib import resources +from unittest.mock import MagicMock, patch + +import pytest + +from audio_separator.separator import Separator + +MIRROR = "https://github.com/nomadkaraoke/python-audio-separator/releases/download/model-configs" +UVR = "https://github.com/TRvlvr/model_repo/releases/download/all_public_uvr_models" +VIP = "https://github.com/Anjok0109/ai_magic/releases/download/v5" + +GOOD = b"genuine model bytes" +BAD = b"tampered model bytes" + + +@pytest.fixture +def separator(tmp_path, monkeypatch): + # The env var would redirect downloads into a real model directory. + monkeypatch.delenv("AUDIO_SEPARATOR_MODEL_DIR", raising=False) + model_dir = tmp_path / "models" + model_dir.mkdir() + sep = Separator(model_file_dir=str(model_dir), output_dir=str(tmp_path / "output"), info_only=True) + sep._model_checksums = {"model.ckpt": hashlib.sha256(GOOD).hexdigest(), "config.yaml": hashlib.sha256(b"cfg").hexdigest()} + return sep + + +def _serve(content_by_url): + """Fake requests.get: URLs in content_by_url return 200 with those bytes, anything else 404.""" + + def fake_get(url, stream=True, timeout=None): + response = MagicMock() + if url in content_by_url: + response.status_code = 200 + response.headers = {"content-length": str(len(content_by_url[url]))} + response.iter_content.return_value = [content_by_url[url]] + else: + response.status_code = 404 + return response + + return patch("audio_separator.separator.separator.requests.get", side_effect=fake_get) + + +def _supported(model_type, friendly_name, download_files): + return {model_type: {friendly_name: {"filename": download_files[0], "download_files": download_files}}} + + +def test_model_is_fetched_from_mirror_before_upstream(separator): + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDXC", "Roformer Model: X", ["model.ckpt"])), \ + _serve({f"{MIRROR}/model.ckpt": GOOD, f"{UVR}/model.ckpt": GOOD}) as get: + separator.download_model_files("model.ckpt") + + assert get.call_args_list[0].args[0] == f"{MIRROR}/model.ckpt" + assert get.call_count == 1 + + +def test_upstream_is_used_when_mirror_lacks_the_file(separator, tmp_path): + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDXC", "Roformer Model: X", ["model.ckpt"])), \ + _serve({f"{UVR}/model.ckpt": GOOD}): + separator.download_model_files("model.ckpt") + + assert (tmp_path / "models" / "model.ckpt").read_bytes() == GOOD + + +def test_vip_models_still_come_from_vip_repo_first(separator): + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDX", "MDX-Net Model VIP: X", ["model.ckpt"])), \ + _serve({f"{VIP}/model.ckpt": GOOD, f"{MIRROR}/model.ckpt": GOOD}) as get: + separator.download_model_files("model.ckpt") + + assert get.call_args_list[0].args[0] == f"{VIP}/model.ckpt" + + +def test_mdxc_yaml_config_comes_from_mirror_first(separator): + files = ["model.ckpt", "config.yaml"] + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDXC", "Roformer Model: X", files)), \ + _serve({f"{MIRROR}/model.ckpt": GOOD, f"{MIRROR}/config.yaml": b"cfg"}) as get: + result = separator.download_model_files("model.ckpt") + + assert [c.args[0] for c in get.call_args_list] == [f"{MIRROR}/model.ckpt", f"{MIRROR}/config.yaml"] + assert result[4] == "config.yaml" + + +def test_tampered_download_is_rejected_and_next_source_used(separator, tmp_path): + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDXC", "Roformer Model: X", ["model.ckpt"])), \ + _serve({f"{MIRROR}/model.ckpt": BAD, f"{UVR}/model.ckpt": GOOD}): + separator.download_model_files("model.ckpt") + + assert (tmp_path / "models" / "model.ckpt").read_bytes() == GOOD + assert not (tmp_path / "models" / "model.ckpt.partial").exists() + + +def test_download_fails_when_every_source_is_tampered(separator, tmp_path): + with patch.object(separator, "list_supported_model_files", return_value=_supported("MDXC", "Roformer Model: X", ["model.ckpt"])), \ + _serve({f"{MIRROR}/model.ckpt": BAD, f"{UVR}/model.ckpt": BAD}): + with pytest.raises(RuntimeError, match="checksum"): + separator.download_model_files("model.ckpt") + + assert list((tmp_path / "models").iterdir()) == [] + + +def test_tampered_cached_file_is_replaced(separator, tmp_path): + (tmp_path / "models" / "model.ckpt").write_bytes(BAD) + + with _serve({f"{MIRROR}/model.ckpt": GOOD}) as get: + separator.download_file_from_first_available([f"{MIRROR}/model.ckpt"], str(tmp_path / "models" / "model.ckpt")) + + assert get.call_count == 1 + assert (tmp_path / "models" / "model.ckpt").read_bytes() == GOOD + + +def test_verified_cached_file_is_not_downloaded_or_rehashed(separator, tmp_path): + path = tmp_path / "models" / "model.ckpt" + path.write_bytes(GOOD) + + with _serve({}) as get: + separator.download_file_if_not_exists(f"{MIRROR}/model.ckpt", str(path), verify_checksum=True) + with patch("audio_separator.separator.separator.hashlib.sha256") as sha256: + separator.download_file_if_not_exists(f"{MIRROR}/model.ckpt", str(path), verify_checksum=True) + + get.assert_not_called() + sha256.assert_not_called() + + +def test_files_without_a_published_checksum_are_accepted(separator, tmp_path): + with _serve({f"{MIRROR}/new_model.ckpt": b"anything"}): + separator.download_file_if_not_exists(f"{MIRROR}/new_model.ckpt", str(tmp_path / "models" / "new_model.ckpt"), verify_checksum=True) + + assert (tmp_path / "models" / "new_model.ckpt").read_bytes() == b"anything" + + +def test_metadata_downloads_are_not_checked_against_model_checksums(separator, tmp_path): + # Mutable lists like download_checks.json share names with old snapshots in the mirror release. + separator._model_checksums["download_checks.json"] = hashlib.sha256(b"old snapshot").hexdigest() + + with _serve({"https://example.com/download_checks.json": b"current list"}): + separator.download_file_if_not_exists("https://example.com/download_checks.json", str(tmp_path / "models" / "download_checks.json")) + + assert (tmp_path / "models" / "download_checks.json").read_bytes() == b"current list" + + +def test_bundled_manifest_covers_default_models(): + with resources.open_text("audio_separator", "model-checksums.json") as f: + checksums = json.load(f)["sha256"] + + for filename in ["model_bs_roformer_ep_317_sdr_12.9755.ckpt", "mel_band_roformer_karaoke_aufr33_viperx_sdr_10.1956.ckpt", "htdemucs_ft.yaml", "f7e0c4bc-ba3fe64a.th"]: + assert len(checksums[filename]) == 64 diff --git a/tools/update-model-checksums.py b/tools/update-model-checksums.py new file mode 100644 index 00000000..b6cf0bd8 --- /dev/null +++ b/tools/update-model-checksums.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Regenerate audio_separator/model-checksums.json: the SHA-256 of every file published in the +release repos that Separator downloads models from. Separator refuses any download (and any +cached file) whose hash doesn't match, so rerun this after uploading new model files. + +Files are streamed and hashed, never written to disk, but this reads every asset (~110 GB for +a full run), so run it on the build host. Hashes already in the manifest are kept unless +--rehash is given, so routine runs only download new files. + +Usage: tools/update-model-checksums.py [--rehash] [--jobs N] +""" + +import argparse +import hashlib +import json +import os +import sys +from concurrent.futures import ThreadPoolExecutor + +import requests + +# Same order Separator tries them in: our mirror first, then the upstream repos. +RELEASES = [ + ("nomadkaraoke/python-audio-separator", "model-configs"), + ("TRvlvr/model_repo", "all_public_uvr_models"), + ("Anjok0109/ai_magic", "v5"), +] +# Demucs checkpoints are downloaded straight from Meta's CDN when the mirror doesn't have them. +EXTRA_URLS = [ + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/f7e0c4bc-ba3fe64a.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/d12395a8-e57c48e6.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/92cfc3b6-ef3bcb9c.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/04573f0d-f3cf25b2.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/5c90dfd2-34c22ccb.th", + "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/75fc33f5-1941ce65.th", +] +MANIFEST_PATH = os.path.join(os.path.dirname(__file__), "..", "audio_separator", "model-checksums.json") + + +def list_assets(): + headers = {"Accept": "application/vnd.github+json"} + if os.getenv("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + for repo, tag in RELEASES: + response = requests.get(f"https://api.github.com/repos/{repo}/releases/tags/{tag}", headers=headers, timeout=60) + response.raise_for_status() + for asset in response.json()["assets"]: + yield asset["name"], asset["browser_download_url"] + for url in EXTRA_URLS: + yield url.split("/")[-1], url + + +def sha256_of_url(url): + digest = hashlib.sha256() + with requests.get(url, stream=True, timeout=300) as response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--rehash", action="store_true", help="re-download and re-hash files already in the manifest") + parser.add_argument("--jobs", type=int, default=6, help="parallel downloads") + args = parser.parse_args() + + with open(MANIFEST_PATH, encoding="utf-8") as f: + manifest = json.load(f) + known = {} if args.rehash else manifest["sha256"] + + # The first source listing a filename is the one Separator tries first. + # .json assets are snapshots of mutable metadata lists, which Separator fetches live and doesn't verify. + sources = {} + for name, url in list_assets(): + if not name.endswith(".json"): + sources.setdefault(name, []).append(url) + + to_hash = [(name, url) for name, urls in sources.items() for url in (urls if args.rehash or name not in known else [])] + print(f"{len(sources)} published files, hashing {len(to_hash)} downloads") + + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + hashes = list(pool.map(lambda item: (item[0], item[1], sha256_of_url(item[1])), to_hash)) + + found = dict(known) + conflicts = [] + for name, url, digest in hashes: + if name in found and found[name] != digest: + conflicts.append(f"{name}: {url} has {digest}, expected {found[name]}") + continue + found[name] = digest + + if conflicts: + # Separator accepts one hash per filename, so a source serving different bytes would always fail verification. + print("Sources disagree on file contents:\n " + "\n ".join(conflicts), file=sys.stderr) + sys.exit(1) + + manifest["sha256"] = dict(sorted(found.items())) + with open(MANIFEST_PATH, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + f.write("\n") + print(f"Wrote {len(found)} checksums to {os.path.normpath(MANIFEST_PATH)}") + + +if __name__ == "__main__": + main() From 185328e5b8621b0f427ff6042133cbdf23e4d386 Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Thu, 29 Oct 2026 12:13:34 -0400 Subject: [PATCH 2/2] fix: per-process partial download path; retry transient errors in checksum tool Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012RYLVTA9CYSnmncNv3GRHJ --- audio_separator/separator/separator.py | 3 ++- tests/unit/test_model_download_integrity.py | 1 + tools/update-model-checksums.py | 22 ++++++++++++++------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/audio_separator/separator/separator.py b/audio_separator/separator/separator.py index cfea2720..a71f3f8e 100644 --- a/audio_separator/separator/separator.py +++ b/audio_separator/separator/separator.py @@ -577,7 +577,8 @@ def download_file_if_not_exists(self, url, output_path, verify_checksum=False): # Download beside the target and rename once complete, so an interrupted # download never leaves a truncated file that later runs treat as present. - partial_path = f"{output_path}.partial" + # The pid keeps processes sharing a model directory from writing the same file. + partial_path = f"{output_path}.{os.getpid()}.partial" try: with open(partial_path, "wb") as f: for chunk in response.iter_content(chunk_size=8192): diff --git a/tests/unit/test_model_download_integrity.py b/tests/unit/test_model_download_integrity.py index f76f86e3..6be79488 100644 --- a/tests/unit/test_model_download_integrity.py +++ b/tests/unit/test_model_download_integrity.py @@ -90,6 +90,7 @@ def test_tampered_download_is_rejected_and_next_source_used(separator, tmp_path) assert (tmp_path / "models" / "model.ckpt").read_bytes() == GOOD assert not (tmp_path / "models" / "model.ckpt.partial").exists() + assert not list((tmp_path / "models").glob("*.partial")) def test_download_fails_when_every_source_is_tampered(separator, tmp_path): diff --git a/tools/update-model-checksums.py b/tools/update-model-checksums.py index b6cf0bd8..c835f3cd 100644 --- a/tools/update-model-checksums.py +++ b/tools/update-model-checksums.py @@ -16,6 +16,7 @@ import json import os import sys +import time from concurrent.futures import ThreadPoolExecutor import requests @@ -52,13 +53,20 @@ def list_assets(): yield url.split("/")[-1], url -def sha256_of_url(url): - digest = hashlib.sha256() - with requests.get(url, stream=True, timeout=300) as response: - response.raise_for_status() - for chunk in response.iter_content(chunk_size=1024 * 1024): - digest.update(chunk) - return digest.hexdigest() +def sha256_of_url(url, attempts=5): + for attempt in range(1, attempts + 1): + digest = hashlib.sha256() + try: + with requests.get(url, stream=True, timeout=300) as response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + except requests.RequestException as e: + if attempt == attempts: + raise + print(f"Retrying {url} after error: {e}", file=sys.stderr) + time.sleep(5 * attempt) def main():