Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions scripts/upload_hf.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,20 @@ def _fix(url, endpoint):
}


def files_to_upload(local_dir: str) -> list[str]:
"""Every regular file in ``local_dir`` worth uploading, sorted.

Excludes dotfiles and ``.bak_vision`` backups left by
scripts/strip_vision_weights.py; everything else — including
README.md — is uploaded.
"""
names = sorted(
f for f in os.listdir(local_dir)
if not f.startswith(".") and ".bak_vision" not in f
)
return [n for n in names if os.path.isfile(os.path.join(local_dir, n))]


def main():
tier = sys.argv[1] if len(sys.argv) > 1 else "edge0-8b"
env_name, repo_id = REPOS[tier]
Expand All @@ -89,12 +103,7 @@ def main():
raise SystemExit(f"checkpoint directory not found: {local_dir}")
api = HfApi()
print(f"uploading {local_dir} -> {repo_id} via {os.environ['HF_ENDPOINT']} (no xet)", flush=True)
names = sorted(
f for f in os.listdir(local_dir)
if not f.startswith(".") and ".bak_vision" not in f and f != "README.md"
)
# README usually already uploaded; skip leftovers handled below
files = [n for n in names if os.path.isfile(os.path.join(local_dir, n))]
files = files_to_upload(local_dir)
for i, name in enumerate(files):
path = os.path.join(local_dir, name)
print(f"[{i+1}/{len(files)}] {name} ({os.path.getsize(path)} bytes)", flush=True)
Expand Down
42 changes: 42 additions & 0 deletions tests/test_upload_hf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""scripts/upload_hf.py file-selection logic.

Pure filesystem logic, no MLX and no network — the upload itself
(``HfApi.upload_file``) is out of scope for unit tests.
"""

from __future__ import annotations

import importlib.util
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]


def _load_upload_hf():
"""Import scripts/upload_hf.py without running main() or touching
the network — the module patches httpx at import time, which is
fine (pure monkeypatch, no I/O)."""
spec = importlib.util.spec_from_file_location(
"upload_hf", ROOT / "scripts" / "upload_hf.py")
mod = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = mod
spec.loader.exec_module(mod)
return mod


def test_files_to_upload_includes_readme(tmp_path):
for name in ("README.md", "config.json", "model.safetensors"):
(tmp_path / name).write_text("x")
mod = _load_upload_hf()
assert mod.files_to_upload(str(tmp_path)) == [
"README.md", "config.json", "model.safetensors"]


def test_files_to_upload_excludes_dotfiles_and_vision_backups(tmp_path):
for name in ("README.md", "model.safetensors",
"model.safetensors.bak_vision", ".DS_Store"):
(tmp_path / name).write_text("x")
mod = _load_upload_hf()
assert mod.files_to_upload(str(tmp_path)) == [
"README.md", "model.safetensors"]
Loading