From 5e44e83aa78c4a4967f17b378db9771c0ccf722b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Wed, 19 Aug 2026 17:41:33 +0800 Subject: [PATCH 1/5] bump version --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 42ac468..99e6186 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.3.0+main" +__version__ = "0.3.0" From cf1180f3c97cc594197a7013c2d1fb65a21b61c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Mon, 31 Aug 2026 17:47:51 +0800 Subject: [PATCH 2/5] refactor(upload): centralize upload config --- src/modelscope_hub/_legacy_api.py | 13 +- src/modelscope_hub/_upload.py | 120 ++++----- src/modelscope_hub/api.py | 6 +- src/modelscope_hub/cli/upload.py | 4 +- src/modelscope_hub/compat/constants.py | 64 ++++- src/modelscope_hub/compat/hub_api.py | 86 +++++-- src/modelscope_hub/constants.py | 334 +++++++++++++++++++++---- src/modelscope_hub/version.py | 2 +- tests/cli/test_upload.py | 4 +- tests/test_compat_get_model_files.py | 121 +++++++++ tests/test_upload_config.py | 310 +++++++++++++++++++++++ tests/test_upload_lfs_gate.py | 48 ++++ 12 files changed, 971 insertions(+), 141 deletions(-) create mode 100644 tests/test_upload_config.py diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index ae8a95d..539fc28 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -30,9 +30,9 @@ REPO_FILES_TRUNCATION_LIMIT, REPO_TREE_MAX_REQUESTS, REPO_TREE_WALK_WORKERS, - UPLOAD_BLOB_CONNECT_TIMEOUT, - UPLOAD_BLOB_READ_TIMEOUT, - UPLOAD_RETRY_ALLOWED_METHODS, + UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, + UPLOAD_BLOB_READ_TIMEOUT_SECONDS, + UPLOAD_HTTP_RETRY_ALLOWED_METHODS, RepoType, ) from .errors import InvalidParameter, NetworkError, RequestTimeoutError, ServerError, raise_for_status @@ -108,7 +108,7 @@ def __init__( total=max_retries, backoff_factor=0.5, status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=UPLOAD_RETRY_ALLOWED_METHODS, + allowed_methods=UPLOAD_HTTP_RETRY_ALLOWED_METHODS, ) adapter = HTTPAdapter(max_retries=retry) self._session.mount("https://", adapter) @@ -778,7 +778,7 @@ def upload_blob( size: int, *, headers: dict[str, str] | None = None, - timeout: int | None = None, + timeout: int | tuple[int, int] | None = None, ) -> dict: """Upload a blob to the presigned URL returned by :meth:`validate_blobs`. @@ -805,7 +805,8 @@ def upload_blob( upload_url, data=data, headers=upload_headers, - timeout=timeout or (UPLOAD_BLOB_CONNECT_TIMEOUT, UPLOAD_BLOB_READ_TIMEOUT), + timeout=timeout + or (UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS), ) except requests.ConnectionError as exc: raise NetworkError(f"Blob upload connection failed: {exc}") from exc diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index 28bbd66..7cdac54 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -32,32 +32,32 @@ from .constants import ( DATASET_LFS_SUFFIX, DEFAULT_IGNORE_PATTERNS, - DEFAULT_MAX_WORKERS, MODEL_LFS_SUFFIX, - UPLOAD_ADAPTIVE_BATCH_SIZE, - UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT, - UPLOAD_BLOB_MAX_RETRIES, - UPLOAD_BLOB_RETRY_BACKOFF, - UPLOAD_BLOB_RETRY_MAX_WAIT, - UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD, + UPLOAD_ADAPTIVE_BATCHING_ENABLED, + UPLOAD_BLOB_MAX_ATTEMPTS, + UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES, + UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS, + UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS, + UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS, + UPLOAD_CACHE_ENABLED, UPLOAD_CACHE_FILE, - UPLOAD_COMMIT_BATCH_SIZE, - UPLOAD_COMMIT_MAX_RETRIES, - UPLOAD_COMMIT_MAX_TOTAL_WAIT, - UPLOAD_FAILED_FILE_MAX_RETRIES, + UPLOAD_COMMIT_BATCH_MAX_OPERATIONS, + UPLOAD_COMMIT_MAX_ATTEMPTS, + UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES, + UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS, + UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS, UPLOAD_LEGACY_PROGRESS_FILE, - UPLOAD_LFS_ENFORCE_THRESHOLD, + UPLOAD_LFS_FORCE_THRESHOLD_BYTES, + UPLOAD_MAX_CONCURRENT_WORKERS, UPLOAD_MAX_FILE_COUNT, - UPLOAD_MAX_FILE_COUNT_IN_DIR, - UPLOAD_MAX_FILE_SIZE, - UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT, - UPLOAD_REACT_BACKOFF_MAX_EXPONENT, - UPLOAD_REACT_ENABLED, - UPLOAD_REACT_MAX_DELAY, - UPLOAD_REACT_ROUND2_BASE_DELAY, - UPLOAD_REACT_ROUND3_FILE_DELAY, - UPLOAD_USE_CACHE, - UPLOAD_VALIDATE_BLOB_BATCH_SIZE, + UPLOAD_MAX_FILE_SIZE_BYTES, + UPLOAD_MAX_FILES_PER_DIRECTORY, + UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, + UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT, + UPLOAD_RECOVERY_ENABLED, + UPLOAD_RECOVERY_MAX_DELAY_SECONDS, + UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS, + UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS, ) from .errors import ( FileIntegrityError, @@ -121,14 +121,14 @@ def verify_complete(self) -> None: def _is_lfs(path: str | Path, size: int, repo_type: str) -> bool: """Determine if a file should use LFS upload mode (suffix + size threshold).""" - if size > UPLOAD_LFS_ENFORCE_THRESHOLD: + if size > UPLOAD_LFS_FORCE_THRESHOLD_BYTES: return True suffix = Path(path).suffix.lower() if isinstance(path, (str, Path)) else "" if repo_type == "model": return suffix in MODEL_LFS_SUFFIX if repo_type == "dataset": return suffix in DATASET_LFS_SUFFIX - return size > UPLOAD_LFS_ENFORCE_THRESHOLD + return size > UPLOAD_LFS_FORCE_THRESHOLD_BYTES def _upload_mode(path: str | Path, size: int, repo_type: str) -> str: @@ -651,7 +651,7 @@ def upload_folder( allow_patterns: list[str] | None = None, ignore_patterns: list[str] | None = None, max_workers: int | None = None, - use_cache: bool = UPLOAD_USE_CACHE, + use_cache: bool | None = None, disable_tqdm: bool = False, sync_remote_repo: bool = False, ) -> dict | list[dict] | None: @@ -664,7 +664,9 @@ def upload_folder( raise InvalidParameter("The arg `folder_path` cannot be None!") if max_workers is None: - max_workers = DEFAULT_MAX_WORKERS + max_workers = UPLOAD_MAX_CONCURRENT_WORKERS + if use_cache is None: + use_cache = UPLOAD_CACHE_ENABLED # Normalize patterns allow_patterns = allow_patterns or None @@ -723,7 +725,7 @@ def upload_folder( sorted_files = sorted(sorted_files, key=lambda x: x[0]) # Calculate batch size - if UPLOAD_ADAPTIVE_BATCH_SIZE: + if UPLOAD_ADAPTIVE_BATCHING_ENABLED: commit_batch_size = _calculate_adaptive_batch_size(len(sorted_files)) logger.info( "Adaptive batch size: %d (for %d files)", @@ -731,7 +733,11 @@ def upload_folder( len(sorted_files), ) else: - commit_batch_size = UPLOAD_COMMIT_BATCH_SIZE if UPLOAD_COMMIT_BATCH_SIZE > 0 else len(sorted_files) + commit_batch_size = ( + UPLOAD_COMMIT_BATCH_MAX_OPERATIONS + if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 + else len(sorted_files) + ) # Initialize tracker folder_path_resolved = Path(folder_path).resolve() @@ -944,7 +950,7 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - ) consecutive_failures += 1 - if consecutive_failures >= UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT: + if consecutive_failures >= UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES: raise RuntimeError( f"Upload aborted: {consecutive_failures} consecutive batch commits failed. " f"Last error: {e}" @@ -953,7 +959,7 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - tracker.save() # ReAct progressive retry fallback - if total_failed_files and UPLOAD_REACT_ENABLED: + if total_failed_files and UPLOAD_RECOVERY_ENABLED: total_failed_files, react_commits, react_results = self._retry_failed_files_react( failed_files=total_failed_files, tracker=tracker, @@ -1177,7 +1183,7 @@ def _upload_single_file( if upload_mode == "lfs": # Retry loop for transient blob upload failures last_error = None - for attempt in range(UPLOAD_BLOB_MAX_RETRIES): + for attempt in range(UPLOAD_BLOB_MAX_ATTEMPTS): try: if isinstance(file_path, (str, os.PathLike)): current_size = os.path.getsize(str(file_path)) @@ -1193,7 +1199,7 @@ def _upload_single_file( sha256=file_hash, size=file_size, data=file_path, - disable_tqdm=(disable_tqdm or file_size <= UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD), + disable_tqdm=(disable_tqdm or file_size <= UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES), tqdm_desc=f"[Uploading {file_path_in_repo}]", pre_validated=pre_validated, ) @@ -1202,15 +1208,15 @@ def _upload_single_file( if isinstance(e, HubError) and not e.retryable: raise last_error = e - if attempt < UPLOAD_BLOB_MAX_RETRIES - 1: + if attempt < UPLOAD_BLOB_MAX_ATTEMPTS - 1: wait = min( - UPLOAD_BLOB_RETRY_BACKOFF**attempt, - UPLOAD_BLOB_RETRY_MAX_WAIT, + UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS**attempt, + UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS, ) logger.warning( "Blob upload attempt %d/%d failed for %s: %s, retrying in %ds ...", attempt + 1, - UPLOAD_BLOB_MAX_RETRIES, + UPLOAD_BLOB_MAX_ATTEMPTS, file_path_in_repo, e, wait, @@ -1218,7 +1224,9 @@ def _upload_single_file( time.sleep(wait) else: raise StorageError( - f"Blob upload failed after {UPLOAD_BLOB_MAX_RETRIES} attempts for {file_path_in_repo}: {last_error}" + "Blob upload failed after " + f"{UPLOAD_BLOB_MAX_ATTEMPTS} attempts for " + f"{file_path_in_repo}: {last_error}" ) from last_error else: if isinstance(file_path, (str, os.PathLike)): @@ -1331,7 +1339,7 @@ def _validate_blobs_batch( objects: list[dict], ) -> dict[str, str | None]: result: dict[str, str | None] = {} - batch_size = UPLOAD_VALIDATE_BLOB_BATCH_SIZE + batch_size = UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS for i in range(0, len(objects), batch_size): chunk = objects[i : i + batch_size] @@ -1355,11 +1363,11 @@ def _commit_with_retry( operations: list[dict], commit_message: str, revision: str = "master", - max_retries: int = UPLOAD_COMMIT_MAX_RETRIES, + max_attempts: int = UPLOAD_COMMIT_MAX_ATTEMPTS, ) -> dict: last_error: Exception | None = None start_time = time.monotonic() - for attempt in range(max_retries): + for attempt in range(max_attempts): try: return self._client.create_commit( repo_id=repo_id, @@ -1382,17 +1390,17 @@ def _commit_with_retry( wait = min(2**attempt, 60) elapsed = time.monotonic() - start_time - if elapsed + wait > UPLOAD_COMMIT_MAX_TOTAL_WAIT: + if elapsed + wait > UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS: logger.error( "Commit total wait time would exceed %ds (already %.1fs elapsed), aborting retries.", - UPLOAD_COMMIT_MAX_TOTAL_WAIT, + UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS, elapsed, ) break logger.warning( "Commit attempt %d/%d failed: %s, retrying in %ds ...", attempt + 1, - max_retries, + max_attempts, last_error, wait, ) @@ -1400,7 +1408,7 @@ def _commit_with_retry( if isinstance(last_error, HubError): raise last_error - raise NetworkError(f"Commit failed after {max_retries} attempts: {last_error}") from last_error + raise NetworkError(f"Commit failed after {max_attempts} attempts: {last_error}") from last_error # ------------------------------------------------------------------ # Internal: build operations @@ -1510,10 +1518,10 @@ def _prepare_upload_folder( parent = str(path.parent) dir_counts[parent] = dir_counts.get(parent, 0) + 1 for dir_path, count in dir_counts.items(): - if count > UPLOAD_MAX_FILE_COUNT_IN_DIR: + if count > UPLOAD_MAX_FILES_PER_DIRECTORY: raise InvalidParameter( f"Too many files ({count}) in directory {dir_path}, " - f"max allowed per directory: {UPLOAD_MAX_FILE_COUNT_IN_DIR}" + f"max allowed per directory: {UPLOAD_MAX_FILES_PER_DIRECTORY}" ) # File size checks @@ -1521,21 +1529,21 @@ def _prepare_upload_folder( normal_size = 0 for path in all_files: fsize = path.stat().st_size - if fsize > UPLOAD_MAX_FILE_SIZE: + if fsize > UPLOAD_MAX_FILE_SIZE_BYTES: raise InvalidParameter( f"File too large: {path} ({fsize / 1024 / 1024:.1f} MB), " - f"max allowed: {UPLOAD_MAX_FILE_SIZE / 1024 / 1024:.0f} MB" + f"max allowed: {UPLOAD_MAX_FILE_SIZE_BYTES / 1024 / 1024:.0f} MB" ) total_size += fsize if not _is_lfs(str(path), fsize, repo_type): normal_size += fsize - if normal_size > UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT: + if normal_size > UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: logger.warning( "Total normal (non-LFS) file size %d bytes exceeds soft " "limit %d bytes. Consider using LFS for large files.", normal_size, - UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT, + UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, ) relpath_to_abspath = {path.relative_to(folder).as_posix(): str(path) for path in all_files} @@ -1611,14 +1619,14 @@ def _retry_failed_files_react( "parallel": False, "workers": 1, "batch_size": 8, - "delay": UPLOAD_REACT_ROUND2_BASE_DELAY, + "delay": UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS, }, { "name": "Round 3 (single-file)", "parallel": False, "workers": 1, "batch_size": 1, - "delay": UPLOAD_REACT_ROUND3_FILE_DELAY, + "delay": UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS, }, ] @@ -1661,11 +1669,11 @@ def _retry_failed_files_react( for i, ((path_in_repo_r, file_path_r), _err) in enumerate(retryable): if cfg["delay"] > 0 and i > 0: delay = ( - cfg["delay"] * (2 ** min(i, UPLOAD_REACT_BACKOFF_MAX_EXPONENT)) + cfg["delay"] * (2 ** min(i, UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT)) if round_idx == 1 else cfg["delay"] ) - delay = min(delay, UPLOAD_REACT_MAX_DELAY) + delay = min(delay, UPLOAD_RECOVERY_MAX_DELAY_SECONDS) logger.info( "[ReAct] Waiting %ds before retrying %s ...", delay, @@ -1812,13 +1820,13 @@ def _retry_failed_simple( disable_tqdm: bool = False, ) -> list[tuple]: total_failed_files = list(failed_files) - for retry_round in range(UPLOAD_FAILED_FILE_MAX_RETRIES): + for retry_round in range(UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS): if not total_failed_files: break logger.info( "Retry round %d/%d: re-uploading %d failed file(s) ...", retry_round + 1, - UPLOAD_FAILED_FILE_MAX_RETRIES, + UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS, len(total_failed_files), ) retry_failures: list[tuple] = [] diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 61cf5fe..2dca2f8 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -1224,7 +1224,7 @@ def upload_folder( allow_patterns: list[str] | None = None, ignore_patterns: list[str] | None = None, max_workers: int | None = None, - use_cache: bool = True, + use_cache: bool | None = None, disable_tqdm: bool = False, sync_remote_repo: bool = False, ) -> dict | list[dict] | None: @@ -1257,7 +1257,9 @@ def upload_folder( max_workers : int, optional Concurrency for parallel uploads. Defaults to adaptive. use_cache : bool, optional - Use ``.ms_upload_cache`` for resumable uploads. Default True. + Use resumable upload caching. When omitted, read + ``MODELSCOPE_UPLOAD_CACHE_ENABLED``. Explicit ``True`` or ``False`` + overrides the environment configuration. disable_tqdm : bool, optional Disable progress bars. Default False. sync_remote_repo : bool, optional diff --git a/src/modelscope_hub/cli/upload.py b/src/modelscope_hub/cli/upload.py index 3486eed..628e3a0 100644 --- a/src/modelscope_hub/cli/upload.py +++ b/src/modelscope_hub/cli/upload.py @@ -78,8 +78,8 @@ def register(subparsers: SubParsers) -> None: "--use-cache", dest="use_cache", action="store_true", - default=True, - help="Use .ms_upload_cache for resumable folder uploads (default).", + default=None, + help="Enable upload cache / resume support.", ) cache_group.add_argument( "--no-cache", diff --git a/src/modelscope_hub/compat/constants.py b/src/modelscope_hub/compat/constants.py index 7c8ad72..db57ebb 100644 --- a/src/modelscope_hub/compat/constants.py +++ b/src/modelscope_hub/compat/constants.py @@ -1,6 +1,37 @@ """Legacy constant mappings for backward compatibility with modelscope SDK.""" -from ..constants import RepoType, Visibility +from ..constants import ( + UPLOAD_ADAPTIVE_BATCHING_ENABLED, + UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, + UPLOAD_BLOB_MAX_ATTEMPTS, + UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES, + UPLOAD_BLOB_READ_TIMEOUT_SECONDS, + UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS, + UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS, + UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS, + UPLOAD_CACHE_ENABLED, + UPLOAD_COMMIT_BATCH_MAX_OPERATIONS, + UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS, + UPLOAD_HTTP_RETRY_ALLOWED_METHODS, + UPLOAD_LFS_FORCE_THRESHOLD_BYTES, + UPLOAD_MAX_CONCURRENT_WORKERS, + UPLOAD_MAX_FILE_SIZE_BYTES, + UPLOAD_MAX_FILES_PER_DIRECTORY, + UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, + UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT, + UPLOAD_RECOVERY_ENABLED, + UPLOAD_RECOVERY_MAX_DELAY_SECONDS, + UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS, + UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS, + RepoType, + Visibility, +) +from ..constants import ( + UPLOAD_LFS_THRESHOLD as _UPLOAD_LFS_THRESHOLD, +) +from ..constants import ( + UPLOAD_MAX_FILE_COUNT as _UPLOAD_MAX_FILE_COUNT, +) REPO_TYPE_MODEL: str = RepoType.MODEL.value REPO_TYPE_DATASET: str = RepoType.DATASET.value @@ -8,7 +39,36 @@ REPO_TYPE_SUPPORT: list[str] = [REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_STUDIO] DEFAULT_DATASET_REVISION: str = "master" -DEFAULT_MAX_WORKERS: int = 4 +DEFAULT_MAX_WORKERS: int = UPLOAD_MAX_CONCURRENT_WORKERS + +# Legacy upload constants. Values come from the modelscope-hub runtime config. +UPLOAD_BLOB_CONNECT_TIMEOUT: int = UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS +UPLOAD_BLOB_READ_TIMEOUT: int = UPLOAD_BLOB_READ_TIMEOUT_SECONDS +UPLOAD_BLOB_MAX_RETRIES: int = UPLOAD_BLOB_MAX_ATTEMPTS +UPLOAD_BLOB_RETRY_BACKOFF: int = UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS +UPLOAD_BLOB_RETRY_MAX_WAIT: int = UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS +UPLOAD_FAILED_FILE_MAX_RETRIES: int = UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS +UPLOAD_BLOB_TIMEOUT: tuple[int, int] = ( + UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, + UPLOAD_BLOB_READ_TIMEOUT_SECONDS, +) +UPLOAD_RETRY_ALLOWED_METHODS: frozenset[str] = UPLOAD_HTTP_RETRY_ALLOWED_METHODS +UPLOAD_MAX_FILE_SIZE: int = UPLOAD_MAX_FILE_SIZE_BYTES +UPLOAD_MAX_FILE_COUNT: int = _UPLOAD_MAX_FILE_COUNT +UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS: int = UPLOAD_LFS_FORCE_THRESHOLD_BYTES +UPLOAD_LFS_THRESHOLD: int = _UPLOAD_LFS_THRESHOLD +UPLOAD_MAX_FILE_COUNT_IN_DIR: int = UPLOAD_MAX_FILES_PER_DIRECTORY +UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT: int = UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES +UPLOAD_COMMIT_BATCH_SIZE: int = UPLOAD_COMMIT_BATCH_MAX_OPERATIONS +UPLOAD_VALIDATE_BLOB_BATCH_SIZE: int = UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS +UPLOAD_ADAPTIVE_BATCH_SIZE: bool = UPLOAD_ADAPTIVE_BATCHING_ENABLED +UPLOAD_REACT_ENABLED: bool = UPLOAD_RECOVERY_ENABLED +UPLOAD_REACT_ROUND2_BASE_DELAY: int = UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS +UPLOAD_REACT_ROUND3_FILE_DELAY: int = UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS +UPLOAD_REACT_BACKOFF_MAX_EXPONENT: int = UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT +UPLOAD_REACT_MAX_DELAY: int = UPLOAD_RECOVERY_MAX_DELAY_SECONDS +UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD: int = UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES +UPLOAD_USE_CACHE: bool = UPLOAD_CACHE_ENABLED # Visibility integer constants matching the old SDK ModelVisibility_PUBLIC: int = int(Visibility.PUBLIC) diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index c9e6fe0..792be6f 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -19,8 +19,10 @@ AlreadyExistsError, AuthenticationError, InvalidParameter, + NetworkError, NotExistError, PermissionDeniedError, + RequestTimeoutError, is_repo_exists_error, ) from ..utils.logger import get_logger @@ -61,6 +63,26 @@ def upload_folder(self, *, repo_id: str, folder_path: Any, **kwargs: Any) -> dic ) +def _resolve_aigc_readme_content(aigc_model: Any, override: str | None = None) -> str | None: + content = override if override is not None else getattr(aigc_model, "readme_content", None) + if content is not None and not isinstance(content, str): + raise InvalidParameter("readme_content must be a string or None") + return content + + +def _upload_aigc_readme(api: HubApi, model_id: str, content: str | None, version: str) -> None: + if content is None: + return + api.upload_file( + repo_id=model_id, + repo_type=RepoType.MODEL, + path_or_fileobj=content.encode("utf-8"), + path_in_repo="README.md", + revision="master", + commit_message=f"Update README.md for AIGC version {version}", + ) + + class LegacyHubApi: """Drop-in replacement for the old ``modelscope.hub.api.HubApi``. @@ -218,6 +240,9 @@ def _create_aigc_model(self, model_id: str, aigc_model: Any, kwargs: dict[str, A chinese_name = kwargs.pop("chinese_name", None) original_model_id = kwargs.pop("original_model_id", "") gated_mode = kwargs.pop("gated_mode", None) + readme_content = _resolve_aigc_readme_content( + aigc_model, kwargs.pop("readme_content", None) + ) if kwargs: unexpected = ", ".join(sorted(kwargs)) raise TypeError(f"create_model() got unexpected keyword argument(s): {unexpected}") @@ -268,6 +293,7 @@ def _create_aigc_model(self, model_id: str, aigc_model: Any, kwargs: dict[str, A ) api.legacy.create_aigc_model(body) aigc_model.upload_to_repo(_AigcUploadAdapter(api), model_id, token) + _upload_aigc_readme(api, model_id, readme_content, aigc_model.tag) return f"{api._config.endpoint}/models/{model_id}" def create_model_tag( @@ -277,8 +303,14 @@ def create_model_tag( endpoint: str | None = None, token: str | None = None, aigc_model: Any = None, + readme_content: str | None = None, ) -> str: - """Create a model tag while preserving the AIGC-specific endpoint.""" + """Create a model tag while preserving the AIGC-specific endpoint. + + For AIGC models, ``readme_content`` is committed to ``master`` before + the immutable version snapshot is created. When omitted, the value on + ``aigc_model.readme_content`` is used. + """ if not model_id: raise InvalidParameter("model_id is required!") if not tag_name: @@ -294,29 +326,49 @@ def create_model_tag( if token or endpoint: api = HubApi(token=token, endpoint=endpoint or self._endpoint) if aigc_model is None: + if readme_content is not None: + raise InvalidParameter("readme_content is only supported for AIGC model tags") api.create_repo_tag(model_id, RepoType.MODEL, tag_name, revision="master") else: owner, name = api._parse_repo_id(model_id) + content = _resolve_aigc_readme_content(aigc_model, readme_content) cookies = api.get_cookies(access_token=token, cookies_required=True) aigc_model.preupload_weights( cookies=cookies, headers={}, endpoint=api._config.endpoint, ) - api.legacy.create_aigc_model_tag( - { - "CoverImages": aigc_model.cover_images, - "Name": name, - "Path": owner, - "TagShowName": tag_name, - "WeightsName": aigc_model.weight_filename, - "WeightsSha256": aigc_model.weight_sha256, - "WeightsSize": aigc_model.weight_size, - "TriggerWords": aigc_model.trigger_words, - "AigcType": aigc_model.aigc_type, - "VisionFoundation": aigc_model.base_model_type, - } - ) + _upload_aigc_readme(api, model_id, content, tag_name) + try: + api.legacy.create_aigc_model_tag( + { + "CoverImages": aigc_model.cover_images, + "Name": name, + "Path": owner, + "TagShowName": tag_name, + "WeightsName": aigc_model.weight_filename, + "WeightsSha256": aigc_model.weight_sha256, + "WeightsSize": aigc_model.weight_size, + "TriggerWords": aigc_model.trigger_words, + "AigcType": aigc_model.aigc_type, + "VisionFoundation": aigc_model.base_model_type, + } + ) + except (NetworkError, RequestTimeoutError): + for attempt in range(5): + try: + _, tags = api.legacy.list_revisions_detail(model_id, "model") + if any( + item.get("ShowName") == tag_name or item.get("Revision") == tag_name + for item in tags + ): + break + except (NetworkError, RequestTimeoutError): + pass + if attempt < 4: + time.sleep(2) + else: + raise return f"{api._config.endpoint}/models/{model_id}/tags/{tag_name}" def push_model(self, model_id: str, model_dir: str, **kwargs: Any) -> None: @@ -363,8 +415,8 @@ def push_model(self, model_id: str, model_dir: str, **kwargs: Any) -> None: revision=kwargs.get("revision"), allow_patterns=kwargs.get("allow_patterns"), ignore_patterns=kwargs.get("ignore_patterns"), - max_workers=kwargs.get("max_workers", 4), - use_cache=kwargs.get("use_cache", True), + max_workers=kwargs.get("max_workers"), + use_cache=kwargs.get("use_cache"), ) except (AuthenticationError, InvalidParameter) as e: if _is_auth_related(e): diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index 203d98b..a1b3457 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -223,6 +223,75 @@ def _env_int_mb( return default_mb * 1024 * 1024 +def _env_int_mb_with_deprecated_units( + name: str, + default_mb: int, + description: str, + category: str, + *, + deprecated_mb_names: tuple[str, ...] = (), + deprecated_byte_names: tuple[str, ...] = (), +) -> int: + """Read an MB setting while preserving aliases with explicit units.""" + deprecated_names = deprecated_mb_names + deprecated_byte_names + _env_register(name, str(default_mb), description, category, deprecated_names=deprecated_names) + default_bytes = default_mb * 1024 * 1024 + + raw = os.environ.get(name) + if raw is not None and raw.strip(): + try: + value = int(raw) + except ValueError: + return default_bytes + return value * 1024 * 1024 if value > 0 else default_bytes + + import warnings + + for old in deprecated_mb_names: + raw = os.environ.get(old) + if raw is not None and raw.strip(): + warnings.warn( + f"Environment variable {old!r} is deprecated, use {name!r} instead.", + FutureWarning, + stacklevel=2, + ) + try: + value = int(raw) + except ValueError: + return default_bytes + return value * 1024 * 1024 if value > 0 else default_bytes + + for old in deprecated_byte_names: + raw = os.environ.get(old) + if raw is not None and raw.strip(): + warnings.warn( + f"Environment variable {old!r} is deprecated, use {name!r} instead. " + f"Note: {name!r} expects a value in MB.", + FutureWarning, + stacklevel=2, + ) + try: + value = int(raw) + except ValueError: + return default_bytes + return value if value > 0 else default_bytes + + return default_bytes + + +def _env_csv_frozenset( + name: str, + default: str, + description: str, + category: str, + *deprecated_names: str, +) -> frozenset[str]: + """Read a comma-separated, case-normalised set from the environment.""" + _env_register(name, default, description, category, deprecated_names=deprecated_names) + raw = _env(name, *deprecated_names) or default + return frozenset(item.strip().upper() for item in raw.split(",") if item.strip()) + + def _env_bool( name: str, default: bool, @@ -424,103 +493,237 @@ def _env_register( ENV_INTER_CLOUD_REGIONS, "", "Comma-separated peer regions for cross-region internal acceleration", "Download" ) -UPLOAD_LFS_THRESHOLD: int = _env_int("UPLOAD_LFS_THRESHOLD", 5 * 1024 * 1024) -UPLOAD_LFS_ENFORCE_THRESHOLD: int = _env_int("UPLOAD_LFS_ENFORCE_THRESHOLD", 1 * 1024 * 1024) - -# Upload: blob retry -UPLOAD_BLOB_MAX_RETRIES: int = _env_int("UPLOAD_BLOB_MAX_RETRIES", 5) -UPLOAD_BLOB_RETRY_BACKOFF: int = _env_int("UPLOAD_BLOB_RETRY_BACKOFF", 2) -UPLOAD_BLOB_RETRY_MAX_WAIT: int = _env_int("UPLOAD_BLOB_RETRY_MAX_WAIT", 60) -UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD: int = _env_int("UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD", 5 * 1024 * 1024) - -# Upload: blob timeout -UPLOAD_BLOB_CONNECT_TIMEOUT: int = _env_int( - "MODELSCOPE_UPLOAD_CONNECT_TIMEOUT", +# Upload: blob transport and retries +UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", 30, - "Upload connect timeout (seconds)", + "Blob upload connection timeout (seconds)", "Upload", + "MODELSCOPE_UPLOAD_CONNECT_TIMEOUT", "UPLOAD_BLOB_CONNECT_TIMEOUT", ) -UPLOAD_BLOB_READ_TIMEOUT: int = _env_int( - "MODELSCOPE_UPLOAD_READ_TIMEOUT", +UPLOAD_BLOB_READ_TIMEOUT_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_READ_TIMEOUT_SECONDS", 3600, - "Upload read timeout (seconds)", + "Blob upload socket read idle timeout (seconds)", "Upload", + "MODELSCOPE_UPLOAD_READ_TIMEOUT", "UPLOAD_BLOB_READ_TIMEOUT", + "UPLOAD_BLOB_TIMEOUT_SECONDS", ) - -# Upload: urllib3 retry -UPLOAD_RETRY_ALLOWED_METHODS: frozenset[str] = frozenset( - os.environ.get("UPLOAD_RETRY_ALLOWED_METHODS", "GET,HEAD,DELETE,OPTIONS,TRACE").split(",") +UPLOAD_BLOB_MAX_ATTEMPTS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_MAX_ATTEMPTS", + 5, + "Maximum total attempts for one blob upload", + "Upload", + "UPLOAD_BLOB_MAX_RETRIES", +) +UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS", + 2, + "Exponential backoff base for blob retries (seconds)", + "Upload", + "UPLOAD_BLOB_RETRY_BACKOFF", +) +UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS", + 60, + "Maximum delay between blob attempts (seconds)", + "Upload", + "UPLOAD_BLOB_RETRY_MAX_WAIT", +) +UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES: int = _env_int_mb_with_deprecated_units( + "MODELSCOPE_UPLOAD_BLOB_PROGRESS_THRESHOLD_MB", + 5, + "Minimum blob size for displaying upload progress (MB)", + "Upload", + deprecated_byte_names=("UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD",), ) -# Upload: batching -UPLOAD_COMMIT_BATCH_SIZE: int = _env_int("UPLOAD_COMMIT_BATCH_SIZE", 256) -UPLOAD_ADAPTIVE_BATCH_SIZE: bool = _env_bool("UPLOAD_ADAPTIVE_BATCH_SIZE", True) -UPLOAD_VALIDATE_BLOB_BATCH_SIZE: int = _env_int("UPLOAD_VALIDATE_BLOB_BATCH_SIZE", 64) +# Upload: HTTP transport retries +UPLOAD_HTTP_RETRY_ALLOWED_METHODS: frozenset[str] = _env_csv_frozenset( + "MODELSCOPE_UPLOAD_HTTP_RETRY_ALLOWED_METHODS", + "GET,HEAD,DELETE,OPTIONS,TRACE", + "HTTP methods eligible for automatic transport retries", + "Upload", + "UPLOAD_RETRY_ALLOWED_METHODS", +) -# Upload: commit retry -UPLOAD_COMMIT_MAX_RETRIES: int = _env_int("UPLOAD_COMMIT_MAX_RETRIES", 5) -UPLOAD_COMMIT_MAX_TOTAL_WAIT: int = _env_int( - "MODELSCOPE_UPLOAD_COMMIT_MAX_TOTAL_WAIT", +# Upload: batching and commit retries +UPLOAD_COMMIT_BATCH_MAX_OPERATIONS: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", + 256, + "Maximum operations in one upload commit batch", + "Upload", + "UPLOAD_COMMIT_BATCH_SIZE", +) +UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS: int = _env_int( + "MODELSCOPE_UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", + 64, + "Maximum objects in one blob validation request", + "Upload", + "UPLOAD_VALIDATE_BLOB_BATCH_SIZE", +) +UPLOAD_ADAPTIVE_BATCHING_ENABLED: bool = _env_bool( + "MODELSCOPE_UPLOAD_ADAPTIVE_BATCHING_ENABLED", + True, + "Enable adaptive upload commit batch sizing", + "Upload", + "UPLOAD_ADAPTIVE_BATCH_SIZE", +) +UPLOAD_COMMIT_MAX_ATTEMPTS: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_MAX_ATTEMPTS", + 5, + "Maximum total attempts for one upload commit", + "Upload", + "UPLOAD_COMMIT_MAX_RETRIES", +) +UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS", 300, - "Maximum total wait time (seconds) for commit retries in _commit_with_retry", + "Maximum total wait across upload commit retries (seconds)", "Upload", + "MODELSCOPE_UPLOAD_COMMIT_MAX_TOTAL_WAIT", ) - -# Upload: consecutive batch failure limit -UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT: int = _env_int( +UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES", + 3, + "Maximum consecutive failed upload commit batches", + "Upload", "MODELSCOPE_UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT", +) +UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS: int = _env_int( + "MODELSCOPE_UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS", 3, - "Maximum consecutive batch commit failures before aborting upload_folder", + "Maximum retry rounds for failed upload files", "Upload", + "UPLOAD_FAILED_FILE_MAX_RETRIES", +) +# Upload: progressive recovery +UPLOAD_RECOVERY_ENABLED: bool = _env_bool( + "MODELSCOPE_UPLOAD_RECOVERY_ENABLED", + True, + "Enable progressive upload recovery", + "Upload", + "UPLOAD_REACT_ENABLED", +) +UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS", + 2, + "Backoff base for serial upload recovery (seconds)", + "Upload", + "UPLOAD_REACT_ROUND2_BASE_DELAY", +) +UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS", + 5, + "Delay between single-file recovery attempts (seconds)", + "Upload", + "UPLOAD_REACT_ROUND3_FILE_DELAY", +) +UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT: int = _env_int( + "MODELSCOPE_UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT", + 5, + "Maximum exponent for progressive upload recovery backoff", + "Upload", + "UPLOAD_REACT_BACKOFF_MAX_EXPONENT", +) +UPLOAD_RECOVERY_MAX_DELAY_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_RECOVERY_MAX_DELAY_SECONDS", + 120, + "Maximum progressive upload recovery delay (seconds)", + "Upload", + "UPLOAD_REACT_MAX_DELAY", ) - -# Upload: failed file retry & ReAct -UPLOAD_FAILED_FILE_MAX_RETRIES: int = _env_int("UPLOAD_FAILED_FILE_MAX_RETRIES", 3) -UPLOAD_REACT_ENABLED: bool = _env_bool("UPLOAD_REACT_ENABLED", True) -UPLOAD_REACT_ROUND2_BASE_DELAY: int = _env_int("UPLOAD_REACT_ROUND2_BASE_DELAY", 2) -UPLOAD_REACT_ROUND3_FILE_DELAY: int = _env_int("UPLOAD_REACT_ROUND3_FILE_DELAY", 5) -UPLOAD_REACT_BACKOFF_MAX_EXPONENT: int = _env_int("UPLOAD_REACT_BACKOFF_MAX_EXPONENT", 5) -UPLOAD_REACT_MAX_DELAY: int = _env_int("UPLOAD_REACT_MAX_DELAY", 120) # Upload: workers -DEFAULT_MAX_WORKERS: int = _env_int( - "MODELSCOPE_UPLOAD_MAX_WORKERS", +UPLOAD_MAX_CONCURRENT_WORKERS: int = _env_int( + "MODELSCOPE_UPLOAD_MAX_CONCURRENT_WORKERS", min(8, (os.cpu_count() or 4) + 4), - "Default parallel worker threads (min(8, cpu+4))", + "Maximum concurrent upload workers", "Upload", + "MODELSCOPE_UPLOAD_MAX_WORKERS", "DEFAULT_MAX_WORKERS", ) # Upload: cache / tracker -UPLOAD_USE_CACHE: bool = _env_bool( - "MODELSCOPE_UPLOAD_CACHE", +UPLOAD_CACHE_ENABLED: bool = _env_bool( + "MODELSCOPE_UPLOAD_CACHE_ENABLED", True, "Enable resumable upload cache", "Upload", + "MODELSCOPE_UPLOAD_CACHE", "UPLOAD_USE_CACHE", ) UPLOAD_CACHE_FILE: str = ".ms_upload_cache" UPLOAD_LEGACY_PROGRESS_FILE: str = ".ms_upload_progress" # Upload: limits -UPLOAD_MAX_FILE_SIZE: int = _env_int_mb( +UPLOAD_LFS_FORCE_THRESHOLD_BYTES: int = _env_int_mb_with_deprecated_units( + "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB", + 1, + "File-size threshold that forces LFS mode (MB)", + "Upload", + deprecated_byte_names=("UPLOAD_LFS_ENFORCE_THRESHOLD", "UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS"), +) +UPLOAD_MAX_FILE_SIZE_BYTES: int = _env_int_mb_with_deprecated_units( "MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB", 100 * 1024, - "Max single file size (MB, default 100 GB)", + "Maximum single upload file size (MB, default 100 GB)", "Upload", - "UPLOAD_MAX_FILE_SIZE", + deprecated_mb_names=("UPLOAD_MAX_FILE_SIZE_MB",), + deprecated_byte_names=("UPLOAD_MAX_FILE_SIZE",), ) UPLOAD_MAX_FILE_COUNT: int = _env_int( "MODELSCOPE_UPLOAD_MAX_FILE_COUNT", 100_000, - "Max total files per upload", + "Maximum total files per upload", "Upload", "UPLOAD_MAX_FILE_COUNT", ) -UPLOAD_MAX_FILE_COUNT_IN_DIR: int = _env_int("UPLOAD_MAX_FILE_COUNT_IN_DIR", 50_000) -UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT: int = _env_int("UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT", 500 * 1024 * 1024) +UPLOAD_MAX_FILES_PER_DIRECTORY: int = _env_int( + "MODELSCOPE_UPLOAD_MAX_FILES_PER_DIRECTORY", + 50_000, + "Maximum files in one uploaded directory", + "Upload", + "UPLOAD_MAX_FILE_COUNT_IN_DIR", +) +UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: int = _env_int_mb_with_deprecated_units( + "MODELSCOPE_UPLOAD_NORMAL_FILES_TOTAL_SIZE_MB", + 500, + "Maximum total size of normal (non-LFS) files (MB)", + "Upload", + deprecated_byte_names=("UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT",), +) + +# Deprecated Python aliases. Runtime code must use the explicit names above. +UPLOAD_BLOB_CONNECT_TIMEOUT = UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS +UPLOAD_BLOB_READ_TIMEOUT = UPLOAD_BLOB_READ_TIMEOUT_SECONDS +UPLOAD_BLOB_MAX_RETRIES = UPLOAD_BLOB_MAX_ATTEMPTS +UPLOAD_BLOB_RETRY_BACKOFF = UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS +UPLOAD_BLOB_RETRY_MAX_WAIT = UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS +UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD = UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES +UPLOAD_RETRY_ALLOWED_METHODS = UPLOAD_HTTP_RETRY_ALLOWED_METHODS +UPLOAD_COMMIT_BATCH_SIZE = UPLOAD_COMMIT_BATCH_MAX_OPERATIONS +UPLOAD_VALIDATE_BLOB_BATCH_SIZE = UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS +UPLOAD_ADAPTIVE_BATCH_SIZE = UPLOAD_ADAPTIVE_BATCHING_ENABLED +UPLOAD_COMMIT_MAX_RETRIES = UPLOAD_COMMIT_MAX_ATTEMPTS +UPLOAD_COMMIT_MAX_TOTAL_WAIT = UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS +UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT = UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES +UPLOAD_FAILED_FILE_MAX_RETRIES = UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS +UPLOAD_REACT_ENABLED = UPLOAD_RECOVERY_ENABLED +UPLOAD_REACT_ROUND2_BASE_DELAY = UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS +UPLOAD_REACT_ROUND3_FILE_DELAY = UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS +UPLOAD_REACT_BACKOFF_MAX_EXPONENT = UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT +UPLOAD_REACT_MAX_DELAY = UPLOAD_RECOVERY_MAX_DELAY_SECONDS +DEFAULT_MAX_WORKERS = UPLOAD_MAX_CONCURRENT_WORKERS +UPLOAD_USE_CACHE = UPLOAD_CACHE_ENABLED +UPLOAD_LFS_ENFORCE_THRESHOLD = UPLOAD_LFS_FORCE_THRESHOLD_BYTES +UPLOAD_MAX_FILE_COUNT_IN_DIR = UPLOAD_MAX_FILES_PER_DIRECTORY +UPLOAD_MAX_FILE_SIZE = UPLOAD_MAX_FILE_SIZE_BYTES +UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT = UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES +# This setting never affected runtime upload selection; keep only the import. +UPLOAD_LFS_THRESHOLD: int = 5 * 1024 * 1024 # LFS suffix lists (from old SDK — determines upload mode regardless of size) MODEL_LFS_SUFFIX: list[str] = [ @@ -692,31 +895,56 @@ def _env_register( "StrEnum", "SESSION_FILE_NAME", "TEMPORARY_FOLDER_NAME", + "UPLOAD_ADAPTIVE_BATCHING_ENABLED", "UPLOAD_ADAPTIVE_BATCH_SIZE", + "UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT", "UPLOAD_BLOB_CONNECT_TIMEOUT", + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "UPLOAD_BLOB_MAX_ATTEMPTS", "UPLOAD_BLOB_MAX_RETRIES", + "UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES", "UPLOAD_BLOB_READ_TIMEOUT", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", "UPLOAD_BLOB_RETRY_BACKOFF", + "UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS", + "UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS", "UPLOAD_BLOB_RETRY_MAX_WAIT", "UPLOAD_BLOB_TQDM_DISABLE_THRESHOLD", + "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", + "UPLOAD_CACHE_ENABLED", "UPLOAD_CACHE_FILE", + "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", "UPLOAD_COMMIT_BATCH_SIZE", - "UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT", + "UPLOAD_COMMIT_MAX_ATTEMPTS", + "UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES", "UPLOAD_COMMIT_MAX_RETRIES", "UPLOAD_COMMIT_MAX_TOTAL_WAIT", + "UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS", "UPLOAD_FAILED_FILE_MAX_RETRIES", + "UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS", + "UPLOAD_HTTP_RETRY_ALLOWED_METHODS", "UPLOAD_LEGACY_PROGRESS_FILE", "UPLOAD_LFS_ENFORCE_THRESHOLD", + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", "UPLOAD_LFS_THRESHOLD", + "UPLOAD_MAX_CONCURRENT_WORKERS", "UPLOAD_MAX_FILE_COUNT", "UPLOAD_MAX_FILE_COUNT_IN_DIR", "UPLOAD_MAX_FILE_SIZE", + "UPLOAD_MAX_FILE_SIZE_BYTES", + "UPLOAD_MAX_FILES_PER_DIRECTORY", "UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT", + "UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES", "UPLOAD_REACT_BACKOFF_MAX_EXPONENT", "UPLOAD_REACT_ENABLED", "UPLOAD_REACT_MAX_DELAY", "UPLOAD_REACT_ROUND2_BASE_DELAY", "UPLOAD_REACT_ROUND3_FILE_DELAY", + "UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT", + "UPLOAD_RECOVERY_ENABLED", + "UPLOAD_RECOVERY_MAX_DELAY_SECONDS", + "UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS", + "UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS", "UPLOAD_RETRY_ALLOWED_METHODS", "UPLOAD_USE_CACHE", "UPLOAD_VALIDATE_BLOB_BATCH_SIZE", diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 99e6186..88db203 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index 3d8f544..8582d85 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -91,9 +91,9 @@ def test_max_workers_default_none(self, parser): args = parser.parse_args(["upload", "o/r", "."]) assert args.max_workers is None - def test_use_cache_default_true(self, parser): + def test_use_cache_default_uses_environment(self, parser): args = parser.parse_args(["upload", "o/r", "."]) - assert args.use_cache is True + assert args.use_cache is None def test_no_cache(self, parser): args = parser.parse_args(["upload", "o/r", ".", "--no-cache"]) diff --git a/tests/test_compat_get_model_files.py b/tests/test_compat_get_model_files.py index 7d4023b..6f16696 100644 --- a/tests/test_compat_get_model_files.py +++ b/tests/test_compat_get_model_files.py @@ -9,7 +9,10 @@ from types import SimpleNamespace from unittest import mock +import pytest + from modelscope_hub.compat import LegacyHubApi +from modelscope_hub.errors import InvalidParameter, RequestTimeoutError class _FakeAigcModel: @@ -27,6 +30,7 @@ class _FakeAigcModel: model_source = "USER_UPLOAD" base_model_sub_type = "SD_XL" official_tags = ["photography"] + readme_content = None def __init__(self): self.preupload_weights = mock.MagicMock() @@ -141,6 +145,31 @@ def test_aigc_model_uses_dedicated_endpoint_and_payload(self): assert model_id == "owner/aigc-model" assert token is None + def test_aigc_model_uploads_optional_readme_after_creation(self): + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + aigc_model = _FakeAigcModel() + + with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + api.create_model( + "owner/aigc-model", + aigc_model=aigc_model, + readme_content="# Initial AIGC README\n", + ) + + assert request.call_count == 3 + create_call, ensure_repo_call, readme_commit_call = request.call_args_list + assert create_call.args == ("POST", "models/aigc") + assert ensure_repo_call.args == ("POST", "models") + assert readme_commit_call.args == ( + "POST", + "repos/models/owner/aigc-model/commit/master", + ) + commit_body = readme_commit_call.kwargs["json_body"] + assert commit_body["commit_message"] == "Update README.md for AIGC version v1.0" + operation = commit_body["actions"][0] + assert operation["path"] == "README.md" + assert operation["content"] == "IyBJbml0aWFsIEFJR0MgUkVBRE1FCg==" + def test_plain_model_keeps_generic_create_repo_path(self): api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") @@ -192,6 +221,83 @@ def test_aigc_model_tag_uses_dedicated_endpoint(self): aigc_model.preupload_weights.assert_called_once() assert url == "https://modelscope.cn/models/owner/aigc-model/tags/v1.1" + def test_aigc_model_tag_reconciles_timeout_with_remote_state(self): + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + aigc_model = _FakeAigcModel() + revisions = _response( + { + "RevisionMap": { + "Branches": [{"Revision": "master", "ShowName": ""}], + "Tags": [{"Revision": "20260829162320", "ShowName": "v1.3"}], + } + } + ) + + with mock.patch.object( + api._api.legacy, + "_request", + side_effect=[RequestTimeoutError("timed out"), revisions], + ) as request: + url = api.create_model_tag( + "owner/aigc-model", + "v1.3", + aigc_model=aigc_model, + ) + + assert request.call_count == 2 + assert request.call_args_list[0].args == ("POST", "models/aigc/repo/tag") + assert request.call_args_list[1].args == ( + "GET", + "models/owner/aigc-model/revisions", + ) + assert url == "https://modelscope.cn/models/owner/aigc-model/tags/v1.3" + + def test_aigc_model_tag_uploads_readme_before_snapshot(self): + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + aigc_model = _FakeAigcModel() + aigc_model.readme_content = "# Version v1.3\n\nCustom README content.\n" + + with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + api.create_model_tag( + "owner/aigc-model", + "v1.3", + aigc_model=aigc_model, + ) + + assert request.call_count == 3 + create_repo_call, readme_commit_call, tag_call = request.call_args_list + assert create_repo_call.args == ("POST", "models") + assert readme_commit_call.args == ( + "POST", + "repos/models/owner/aigc-model/commit/master", + ) + commit_body = readme_commit_call.kwargs["json_body"] + assert commit_body["commit_message"] == "Update README.md for AIGC version v1.3" + operation = commit_body["actions"][0] + assert operation["path"] == "README.md" + assert operation["type"] == "normal" + assert operation["encoding"] == "base64" + assert operation["content"] == "IyBWZXJzaW9uIHYxLjMKCkN1c3RvbSBSRUFETUUgY29udGVudC4K" + assert tag_call.args == ("POST", "models/aigc/repo/tag") + assert tag_call.kwargs["json_body"]["TagShowName"] == "v1.3" + + def test_explicit_readme_content_overrides_aigc_model_value(self): + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + aigc_model = _FakeAigcModel() + aigc_model.readme_content = "model value" + + with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + api.create_model_tag( + "owner/aigc-model", + "v1.3", + aigc_model=aigc_model, + readme_content="explicit value", + ) + + commit_body = request.call_args_list[1].kwargs["json_body"] + operation = commit_body["actions"][0] + assert operation["content"] == "ZXhwbGljaXQgdmFsdWU=" + def test_plain_model_tag_keeps_generic_endpoint(self): api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") @@ -206,3 +312,18 @@ def test_plain_model_tag_keeps_generic_endpoint(self): "Ref": "master", } assert url == "https://modelscope.cn/models/owner/plain-model/tags/v1.1" + + def test_plain_model_tag_rejects_aigc_readme_content(self): + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with mock.patch.object(api._api.legacy, "_request") as request: + with pytest.raises( + InvalidParameter, + match="readme_content is only supported for AIGC model tags", + ): + api.create_model_tag( + "owner/plain-model", + "v1.1", + readme_content="# AIGC only", + ) + request.assert_not_called() diff --git a/tests/test_upload_config.py b/tests/test_upload_config.py new file mode 100644 index 0000000..d73ada0 --- /dev/null +++ b/tests/test_upload_config.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub.config import HubConfig + +_SRC = str(Path(__file__).parents[1] / "src") + + +def _run_constants(*names: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + child_env = os.environ.copy() + for name in list(child_env): + if name.startswith(("MODELSCOPE_UPLOAD_", "UPLOAD_")): + child_env.pop(name) + child_env.update(env or {}) + child_env["PYTHONPATH"] = _SRC + script = ( + "from dataclasses import asdict, is_dataclass; import json; " + "import modelscope_hub.constants as c; " + f"print(json.dumps({{name: getattr(c, name) for name in {names!r}}}, " + "default=lambda value: asdict(value) if is_dataclass(value) else sorted(value)))" + ) + return subprocess.run( + [sys.executable, "-W", "always::FutureWarning", "-c", script], + check=True, + capture_output=True, + env=child_env, + text=True, + ) + + +def test_upload_timeout_defaults_are_explicit() -> None: + result = _run_constants( + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", + ) + values = json.loads(result.stdout) + assert values == { + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS": 30, + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS": 3600, + } + + +def test_canonical_timeout_env_wins_over_deprecated_aliases() -> None: + result = _run_constants( + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", + env={ + "MODELSCOPE_UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS": "11", + "MODELSCOPE_UPLOAD_BLOB_READ_TIMEOUT_SECONDS": "22", + "MODELSCOPE_UPLOAD_CONNECT_TIMEOUT": "33", + "UPLOAD_BLOB_TIMEOUT_SECONDS": "44", + }, + ) + values = json.loads(result.stdout) + assert values == { + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS": 11, + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS": 22, + } + assert "FutureWarning" not in result.stderr + + +def test_legacy_timeout_seconds_only_controls_read_timeout() -> None: + result = _run_constants( + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", + env={"UPLOAD_BLOB_TIMEOUT_SECONDS": "123"}, + ) + values = json.loads(result.stdout) + assert values == { + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS": 30, + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS": 123, + } + assert "UPLOAD_BLOB_TIMEOUT_SECONDS" in result.stderr + assert "FutureWarning" in result.stderr + + +def test_size_units_and_http_method_normalization() -> None: + result = _run_constants( + "UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES", + "UPLOAD_MAX_FILE_SIZE_BYTES", + "UPLOAD_HTTP_RETRY_ALLOWED_METHODS", + env={ + "MODELSCOPE_UPLOAD_BLOB_PROGRESS_THRESHOLD_MB": "7", + "UPLOAD_MAX_FILE_SIZE_MB": "12", + "MODELSCOPE_UPLOAD_HTTP_RETRY_ALLOWED_METHODS": " get, post, ,HEAD ", + }, + ) + values = json.loads(result.stdout) + assert values["UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES"] == 7 * 1024 * 1024 + assert values["UPLOAD_MAX_FILE_SIZE_BYTES"] == 12 * 1024 * 1024 + assert values["UPLOAD_HTTP_RETRY_ALLOWED_METHODS"] == ["GET", "HEAD", "POST"] + assert "UPLOAD_MAX_FILE_SIZE_MB" in result.stderr + + +@pytest.mark.parametrize( + ("legacy_env", "constant", "raw", "expected"), + [ + ( + "MODELSCOPE_UPLOAD_CONNECT_TIMEOUT", + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "15", + 15, + ), + ( + "UPLOAD_BLOB_CONNECT_TIMEOUT", + "UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS", + "16", + 16, + ), + ( + "MODELSCOPE_UPLOAD_READ_TIMEOUT", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", + "17", + 17, + ), + ( + "UPLOAD_BLOB_READ_TIMEOUT", + "UPLOAD_BLOB_READ_TIMEOUT_SECONDS", + "18", + 18, + ), + ("UPLOAD_BLOB_MAX_RETRIES", "UPLOAD_BLOB_MAX_ATTEMPTS", "7", 7), + ("UPLOAD_BLOB_RETRY_BACKOFF", "UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS", "3", 3), + ("UPLOAD_BLOB_RETRY_MAX_WAIT", "UPLOAD_BLOB_RETRY_MAX_DELAY_SECONDS", "9", 9), + ("UPLOAD_COMMIT_BATCH_SIZE", "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", "12", 12), + ( + "UPLOAD_VALIDATE_BLOB_BATCH_SIZE", + "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", + "13", + 13, + ), + ("UPLOAD_ADAPTIVE_BATCH_SIZE", "UPLOAD_ADAPTIVE_BATCHING_ENABLED", "false", False), + ("UPLOAD_COMMIT_MAX_RETRIES", "UPLOAD_COMMIT_MAX_ATTEMPTS", "4", 4), + ( + "MODELSCOPE_UPLOAD_COMMIT_MAX_TOTAL_WAIT", + "UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS", + "21", + 21, + ), + ( + "MODELSCOPE_UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT", + "UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES", + "6", + 6, + ), + ( + "UPLOAD_FAILED_FILE_MAX_RETRIES", + "UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS", + "8", + 8, + ), + ("UPLOAD_REACT_ENABLED", "UPLOAD_RECOVERY_ENABLED", "false", False), + ( + "UPLOAD_REACT_ROUND2_BASE_DELAY", + "UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS", + "10", + 10, + ), + ( + "UPLOAD_REACT_ROUND3_FILE_DELAY", + "UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS", + "11", + 11, + ), + ( + "UPLOAD_REACT_BACKOFF_MAX_EXPONENT", + "UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT", + "12", + 12, + ), + ("UPLOAD_REACT_MAX_DELAY", "UPLOAD_RECOVERY_MAX_DELAY_SECONDS", "14", 14), + ("DEFAULT_MAX_WORKERS", "UPLOAD_MAX_CONCURRENT_WORKERS", "2", 2), + ( + "MODELSCOPE_UPLOAD_MAX_WORKERS", + "UPLOAD_MAX_CONCURRENT_WORKERS", + "3", + 3, + ), + ("UPLOAD_USE_CACHE", "UPLOAD_CACHE_ENABLED", "false", False), + ("MODELSCOPE_UPLOAD_CACHE", "UPLOAD_CACHE_ENABLED", "false", False), + ( + "UPLOAD_LFS_ENFORCE_THRESHOLD", + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + "1233", + 1233, + ), + ( + "UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS", + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + "1234", + 1234, + ), + ("UPLOAD_MAX_FILE_COUNT", "UPLOAD_MAX_FILE_COUNT", "42", 42), + ( + "UPLOAD_MAX_FILE_SIZE", + "UPLOAD_MAX_FILE_SIZE_BYTES", + "9876", + 9876, + ), + ( + "UPLOAD_MAX_FILE_COUNT_IN_DIR", + "UPLOAD_MAX_FILES_PER_DIRECTORY", + "43", + 43, + ), + ( + "UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT", + "UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES", + "4321", + 4321, + ), + ], +) +def test_deprecated_upload_env_aliases( + legacy_env: str, + constant: str, + raw: str, + expected: int | bool, +) -> None: + result = _run_constants(constant, env={legacy_env: raw}) + assert json.loads(result.stdout)[constant] == expected + assert legacy_env in result.stderr + assert "FutureWarning" in result.stderr + + +def test_deprecated_http_retry_methods_are_normalized() -> None: + result = _run_constants( + "UPLOAD_HTTP_RETRY_ALLOWED_METHODS", + env={"UPLOAD_RETRY_ALLOWED_METHODS": " get, patch,HEAD "}, + ) + assert json.loads(result.stdout)["UPLOAD_HTTP_RETRY_ALLOWED_METHODS"] == [ + "GET", + "HEAD", + "PATCH", + ] + assert "UPLOAD_RETRY_ALLOWED_METHODS" in result.stderr + + +def test_dead_lfs_threshold_is_not_registered() -> None: + result = _run_constants("ENV_REGISTRY") + names = { + item["name"] + for item in json.loads(result.stdout)["ENV_REGISTRY"] + } + assert "UPLOAD_LFS_THRESHOLD" not in names + + +def _response(data: dict | None = None) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.headers = {} + response.request = None + response.url = "https://modelscope.cn/test" + response.json.return_value = {"Data": data or {}} + return response + + +def test_hubapi_blob_put_receives_explicit_connect_and_read_timeouts() -> None: + config = HubConfig(token="ms-test") + api = HubApi(config=config) + api.uploader._create_repo_fn = None + digest = hashlib.sha256(b"weights").hexdigest() + + def request(method: str, path: str, **kwargs): + if path.endswith("info/lfs/objects/batch"): + return _response( + { + "objects": [ + { + "oid": digest, + "actions": {"upload": {"href": "https://storage/upload"}}, + } + ] + } + ) + return _response() + + put_response = _response() + put_response.json.side_effect = ValueError + + def consume_upload(*args, **kwargs): + data = kwargs["data"] + while data.read(1024): + pass + return put_response + + with ( + patch.object(api.legacy, "_request", side_effect=request), + patch.object(api.legacy._session, "put", side_effect=consume_upload) as put, + ): + api.upload_file( + "owner/repo", + "model", + b"weights", + "model.bin", + disable_tqdm=True, + ) + + put.assert_called_once() + assert put.call_args.kwargs["timeout"] == (30, 3600) diff --git a/tests/test_upload_lfs_gate.py b/tests/test_upload_lfs_gate.py index 225790e..7728cef 100644 --- a/tests/test_upload_lfs_gate.py +++ b/tests/test_upload_lfs_gate.py @@ -6,7 +6,9 @@ from pathlib import Path from unittest.mock import MagicMock +import modelscope_hub._upload as upload_module from modelscope_hub._upload import UploadManager +from modelscope_hub.errors import NetworkError def _make_manager() -> tuple[UploadManager, MagicMock]: @@ -86,6 +88,52 @@ def test_upload_folder_mixed_files_only_uploads_lfs_blob(tmp_path: Path) -> None } +def test_upload_folder_retries_blob_with_configured_max_attempts(tmp_path: Path, monkeypatch) -> None: + manager, client = _make_manager() + (tmp_path / "model.bin").write_bytes(b"weights") + attempts = 0 + + def upload_blob(*, upload_url: str, data, size: int) -> None: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise NetworkError("temporary upload failure") + while data.read(1024 * 1024): + pass + + client.upload_blob.side_effect = upload_blob + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_MAX_ATTEMPTS", 3) + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_RETRY_BACKOFF_BASE_SECONDS", 1) + monkeypatch.setattr(upload_module.time, "sleep", lambda _seconds: None) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + assert client.upload_blob.call_count == 3 + + +def test_upload_folder_uses_configured_cache_default(tmp_path: Path, monkeypatch) -> None: + manager, _ = _make_manager() + (tmp_path / "README.md").write_bytes(b"hello") + monkeypatch.setattr(upload_module, "UPLOAD_CACHE_ENABLED", False) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + disable_tqdm=True, + ) + + assert not (tmp_path / ".ms_upload_cache").exists() + + def test_upload_folder_cached_normal_hash_skips_batch_blob_validation(tmp_path: Path) -> None: manager, client = _make_manager() file_path = tmp_path / "README.md" From f04f2d35223137868143e8892311424d06802110 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Mon, 31 Aug 2026 19:59:07 +0800 Subject: [PATCH 3/5] fix(mcp): allow readonly server discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 班扬 --- src/modelscope_hub/_openapi.py | 56 ++++++++++++++++++++++++++++++++-- src/modelscope_hub/api.py | 4 +++ tests/cli/test_openapi.py | 54 ++++++++++++++++++++++++++------ 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 2f44df4..7f917b5 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -29,9 +29,11 @@ from .config import HubConfig, get_default_config from .constants import API_CONNECT_TIMEOUT, API_MAX_RETRIES, API_TIMEOUT, OPENAPI_PREFIX from .errors import ( + APIError, AuthenticationError, InvalidParameter, NetworkError, + PermissionDeniedError, RateLimitError, RequestTimeoutError, ServerError, @@ -262,6 +264,7 @@ def _request( files: Any | None = None, headers: Mapping[str, str] | None = None, require_token: bool = True, + anonymous: bool = False, unwrap: bool = True, timeout: float | None = None, ) -> Any: @@ -277,7 +280,7 @@ def _request( # Only attach our credentials when the target is our own host. Absolute # URLs to a foreign host (e.g. signed OSS upload URLs) must not receive # the Authorization header or session cookies, which carry the token. - if self._same_host_as_endpoint(final_url): + if self._same_host_as_endpoint(final_url) and not anonymous: merged_headers = dict(self._auth_headers(require_token=require_token)) request_cookies = self._auth_cookies() else: @@ -678,6 +681,29 @@ def update_studio_settings( # ================================================================== # MCP (Model Context Protocol) servers # ================================================================== + @staticmethod + def _flatten_mcp_list_params(body: Mapping[str, Any]) -> QueryParams: + params: QueryParams = [] + for key, value in body.items(): + if key == "filter" and isinstance(value, Mapping): + for filter_key, filter_value in value.items(): + params.append( + ( + f"filter.{filter_key}", + str(filter_value).lower() + if isinstance(filter_value, bool) else str(filter_value), + ) + ) + else: + params.append( + (key, str(value).lower() if isinstance(value, bool) else str(value)) + ) + return params + + @staticmethod + def _is_method_or_route_unsupported(exc: APIError) -> bool: + return exc.status_code in (404, 405, 501) + def list_mcp_servers( self, *, @@ -687,7 +713,12 @@ def list_mcp_servers( filter: Mapping[str, Any] | None = None, extra: Mapping[str, Any] | None = None, ) -> JSON: - """``PUT /mcp/servers`` — discover MCP servers (JSON body, not query). + """``GET /mcp/servers`` — discover MCP servers. + + Falls back to the historical ``PUT /mcp/servers`` endpoint while the + service rolls out GET support. If an optional token is rejected by the + legacy PUT list endpoint, retry anonymously so readonly or stale tokens + do not block public MCP discovery. Parameters ---------- @@ -708,7 +739,26 @@ def list_mcp_servers( if extra: body.update(extra) body = {k: v for k, v in body.items() if v is not None} - return self._request("PUT", "/mcp/servers", json_body=body, require_token=False) + params = self._flatten_mcp_list_params(body) + has_token = self._resolve_token() is not None + try: + return self._request("GET", "/mcp/servers", params=params, require_token=False) + except APIError as exc: + if not self._is_method_or_route_unsupported(exc): + raise + + try: + return self._request("PUT", "/mcp/servers", json_body=body, require_token=False) + except (AuthenticationError, PermissionDeniedError): + if not has_token: + raise + return self._request( + "PUT", + "/mcp/servers", + json_body=body, + require_token=False, + anonymous=True, + ) def list_operational_mcp_servers(self) -> JSON: """``GET /mcp/servers/operational`` — list servers deployed by the caller.""" diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 2dca2f8..226953e 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -1903,6 +1903,7 @@ def list_mcp_servers( search: str | None = None, page_number: int = 1, page_size: int = 10, + filter: Mapping[str, Any] | None = None, **extra: Any, ) -> PagedResult[dict]: """List MCP servers via the OpenAPI surface. @@ -1915,6 +1916,8 @@ def list_mcp_servers( 1-based page index. Default is 1. page_size : int, optional Items per page. Default is 10. + filter : Mapping, optional + Nested MCP filter object. **extra : Any Additional filter fields. ``None`` values are dropped. @@ -1933,6 +1936,7 @@ def list_mcp_servers( search=search, page_number=page_number, page_size=page_size, + filter=filter, extra={k: v for k, v in extra.items() if v is not None} or None, ) items, total, page, size = self._extract_paged(payload) diff --git a/tests/cli/test_openapi.py b/tests/cli/test_openapi.py index 2d0a295..43c3798 100644 --- a/tests/cli/test_openapi.py +++ b/tests/cli/test_openapi.py @@ -13,7 +13,7 @@ from modelscope_hub._openapi import _RETRYABLE_POST_PATHS, OpenAPIClient from modelscope_hub.api import HubApi from modelscope_hub.config import HubConfig -from modelscope_hub.errors import InvalidParameter, RateLimitError, ServerError +from modelscope_hub.errors import InvalidParameter, PermissionDeniedError, RateLimitError, ServerError @pytest.fixture @@ -48,21 +48,57 @@ def _mock_response(status_code=200, json_data=None): # Item 2: list_mcp_servers filter param # ================================================================== class TestListMcpServersFilter: - def test_filter_included_in_body(self, client): + def test_filter_included_in_get_params(self, client): resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers(filter={"category": "tools", "is_hosted": True}) call_kwargs = mock_req.call_args.kwargs - body = call_kwargs["json"] - assert body["filter"] == {"category": "tools", "is_hosted": True} + params = dict(call_kwargs["params"]) + assert call_kwargs["method"] == "GET" + assert call_kwargs["json"] is None + assert params["filter.category"] == "tools" + assert params["filter.is_hosted"] == "true" - def test_filter_none_not_in_body(self, client): + def test_filter_none_not_in_get_params(self, client): resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers() call_kwargs = mock_req.call_args.kwargs - body = call_kwargs["json"] - assert "filter" not in body + params = dict(call_kwargs["params"]) + assert call_kwargs["method"] == "GET" + assert "filter.category" not in params + assert "filter" not in params + + def test_get_route_unsupported_falls_back_to_legacy_put(self, client): + not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) + with patch.object(client._session, "request", side_effect=[not_found, success]) as mock_req: + client.list_mcp_servers(search="weather", page_number=1, page_size=2) + first, second = mock_req.call_args_list + assert first.kwargs["method"] == "GET" + assert dict(first.kwargs["params"])["search"] == "weather" + assert second.kwargs["method"] == "PUT" + assert second.kwargs["json"] == {"search": "weather", "page_number": 1, "page_size": 2} + + def test_get_permission_denied_does_not_fallback_to_put(self, client): + denied = _mock_response(status_code=403, json_data={"message": "denied"}) + with patch.object(client._session, "request", return_value=denied) as mock_req: + with pytest.raises(PermissionDeniedError): + client.list_mcp_servers() + assert mock_req.call_count == 1 + assert mock_req.call_args.kwargs["method"] == "GET" + + def test_put_auth_failure_falls_back_to_anonymous_put(self, client): + not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + denied = _mock_response(status_code=403, json_data={"message": "denied"}) + success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) + with patch.object(client._session, "request", side_effect=[not_found, denied, success]) as mock_req: + client.list_mcp_servers(page_number=1, page_size=2) + assert [call.kwargs["method"] for call in mock_req.call_args_list] == ["GET", "PUT", "PUT"] + assert "Authorization" in mock_req.call_args_list[1].kwargs["headers"] + assert mock_req.call_args_list[1].kwargs["cookies"] + assert "Authorization" not in mock_req.call_args_list[2].kwargs["headers"] + assert mock_req.call_args_list[2].kwargs["cookies"] == {} # ================================================================== @@ -202,8 +238,8 @@ def test_list_mcp_servers_default_page_size(self, client): with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers() call_kwargs = mock_req.call_args.kwargs - body = call_kwargs["json"] - assert body["page_size"] == 10 + params = dict(call_kwargs["params"]) + assert params["page_size"] == "10" # ================================================================== From 9f8f0bd93150b91fa4d75a7f67634fbc474b4ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 14:10:34 +0800 Subject: [PATCH 4/5] fix deprecated warning for envs --- README.md | 14 +++--- src/modelscope_hub/compat/constants.py | 1 + src/modelscope_hub/constants.py | 64 +++++++++++++++----------- tests/test_upload_config.py | 37 ++++++++++++--- 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 3455325..7c27b19 100644 --- a/README.md +++ b/README.md @@ -710,12 +710,13 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se | Variable | Default | Description | |----------|---------|-------------| -| `MODELSCOPE_UPLOAD_MAX_WORKERS` | `min(8, cpu+4)` | Default parallel worker threads | -| `MODELSCOPE_UPLOAD_CACHE` | `true` | Enable resumable upload cache | +| `MODELSCOPE_UPLOAD_MAX_CONCURRENT_WORKERS` | `min(8, cpu+4)` | Default parallel worker threads | +| `MODELSCOPE_UPLOAD_CACHE_ENABLED` | `true` | Enable resumable upload cache | +| `MODELSCOPE_UPLOAD_IGNORE_FILE_PATTERN` | — | File pattern excluded by legacy `push_to_hub` uploads | | `MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB` | `102400` | Max single file size (MB, default 100 GB) | | `MODELSCOPE_UPLOAD_MAX_FILE_COUNT` | `100000` | Max total files per upload | -| `MODELSCOPE_UPLOAD_CONNECT_TIMEOUT` | `30` | Upload connect timeout (seconds) | -| `MODELSCOPE_UPLOAD_READ_TIMEOUT` | `3600` | Upload read timeout (seconds) | +| `MODELSCOPE_UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS` | `30` | Blob upload connect timeout (seconds) | +| `MODELSCOPE_UPLOAD_BLOB_READ_TIMEOUT_SECONDS` | `3600` | Blob upload read timeout (seconds) | **Logging:** @@ -725,8 +726,9 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se | `MODELSCOPE_NO_DEPRECATION_WARNINGS` | — | Suppress deprecation warnings | > Old variable names (e.g. `API_TIMEOUT`, `DOWNLOAD_RETRY_TIMES`, `UPLOAD_USE_CACHE`) are -> still accepted but emit a `FutureWarning`. Run `ms-hub list --envs` to see which deprecated -> names are active in your environment. +> deprecated and remain temporarily supported. They emit a `FutureWarning` and will be removed +> in a future version. Run `ms-hub list --envs` to see which deprecated names are active in your +> environment. diff --git a/src/modelscope_hub/compat/constants.py b/src/modelscope_hub/compat/constants.py index db57ebb..75baec2 100644 --- a/src/modelscope_hub/compat/constants.py +++ b/src/modelscope_hub/compat/constants.py @@ -25,6 +25,7 @@ UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS, RepoType, Visibility, + get_upload_ignore_file_pattern, # noqa: F401 ) from ..constants import ( UPLOAD_LFS_THRESHOLD as _UPLOAD_LFS_THRESHOLD, diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index a1b3457..0a244c8 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -9,6 +9,7 @@ import os import sys +import warnings from dataclasses import dataclass from enum import Enum, IntEnum @@ -139,6 +140,22 @@ class License(StrEnum): _DEPRECATED_LOOKUP: dict[str, tuple[str, ...]] = {} +def _warn_deprecated_env( + old: str, + name: str, + *, + expects_mb: bool = False, + stacklevel: int = 3, +) -> None: + """Warn that a legacy environment variable remains temporarily supported.""" + message = ( + f"Environment variable {old!r} is deprecated and will be removed in a future version. Use {name!r} instead." + ) + if expects_mb: + message += f" {name!r} expects a value in MB." + warnings.warn(message, FutureWarning, stacklevel=stacklevel) + + def _env(name: str, *deprecated_names: str) -> str | None: """Read an env var, falling back to deprecated names with a warning.""" value = os.environ.get(name) @@ -147,13 +164,7 @@ def _env(name: str, *deprecated_names: str) -> str | None: for old in deprecated_names: value = os.environ.get(old) if value is not None: - import warnings - - warnings.warn( - f"Environment variable {old!r} is deprecated, use {name!r} instead.", - FutureWarning, - stacklevel=4, - ) + _warn_deprecated_env(old, name, stacklevel=4) return value return None @@ -207,14 +218,7 @@ def _env_int_mb( for old in all_deprecated: raw = os.environ.get(old) if raw is not None and raw.strip(): - import warnings - - warnings.warn( - f"Environment variable {old!r} is deprecated, " - f"use {name!r} instead. Note: {name!r} expects a value in MB.", - FutureWarning, - stacklevel=2, - ) + _warn_deprecated_env(old, name, expects_mb=True, stacklevel=2) try: value = int(raw) except ValueError: @@ -245,16 +249,10 @@ def _env_int_mb_with_deprecated_units( return default_bytes return value * 1024 * 1024 if value > 0 else default_bytes - import warnings - for old in deprecated_mb_names: raw = os.environ.get(old) if raw is not None and raw.strip(): - warnings.warn( - f"Environment variable {old!r} is deprecated, use {name!r} instead.", - FutureWarning, - stacklevel=2, - ) + _warn_deprecated_env(old, name, stacklevel=2) try: value = int(raw) except ValueError: @@ -264,12 +262,7 @@ def _env_int_mb_with_deprecated_units( for old in deprecated_byte_names: raw = os.environ.get(old) if raw is not None and raw.strip(): - warnings.warn( - f"Environment variable {old!r} is deprecated, use {name!r} instead. " - f"Note: {name!r} expects a value in MB.", - FutureWarning, - stacklevel=2, - ) + _warn_deprecated_env(old, name, expects_mb=True, stacklevel=2) try: value = int(raw) except ValueError: @@ -655,6 +648,20 @@ def _env_register( "MODELSCOPE_UPLOAD_CACHE", "UPLOAD_USE_CACHE", ) +_env_register( + "MODELSCOPE_UPLOAD_IGNORE_FILE_PATTERN", + "-", + "File pattern excluded by legacy push_to_hub uploads", + "Upload", + deprecated_names=("UPLOAD_IGNORE_FILE_PATTERN",), +) + + +def get_upload_ignore_file_pattern() -> str | None: + """Return the optional ignore pattern used by legacy ``push_to_hub`` calls.""" + return _env("MODELSCOPE_UPLOAD_IGNORE_FILE_PATTERN", "UPLOAD_IGNORE_FILE_PATTERN") + + UPLOAD_CACHE_FILE: str = ".ms_upload_cache" UPLOAD_LEGACY_PROGRESS_FILE: str = ".ms_upload_progress" @@ -887,6 +894,7 @@ def _env_register( "ENV_REGISTRY", "EnvVar", "FILE_HASH_FIELD", + "get_upload_ignore_file_pattern", "LEGACY_API_PREFIX", "License", "MODEL_LFS_SUFFIX", diff --git a/tests/test_upload_config.py b/tests/test_upload_config.py index d73ada0..6176727 100644 --- a/tests/test_upload_config.py +++ b/tests/test_upload_config.py @@ -16,17 +16,22 @@ _SRC = str(Path(__file__).parents[1] / "src") -def _run_constants(*names: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: +def _run_constants( + *names: str, + env: dict[str, str] | None = None, + call: str | None = None, +) -> subprocess.CompletedProcess[str]: child_env = os.environ.copy() for name in list(child_env): if name.startswith(("MODELSCOPE_UPLOAD_", "UPLOAD_")): child_env.pop(name) child_env.update(env or {}) child_env["PYTHONPATH"] = _SRC + result = f"getattr(c, {call!r})()" if call is not None else f"{{name: getattr(c, name) for name in {names!r}}}" script = ( "from dataclasses import asdict, is_dataclass; import json; " "import modelscope_hub.constants as c; " - f"print(json.dumps({{name: getattr(c, name) for name in {names!r}}}, " + f"print(json.dumps({result}, " "default=lambda value: asdict(value) if is_dataclass(value) else sorted(value)))" ) return subprocess.run( @@ -82,6 +87,27 @@ def test_legacy_timeout_seconds_only_controls_read_timeout() -> None: } assert "UPLOAD_BLOB_TIMEOUT_SECONDS" in result.stderr assert "FutureWarning" in result.stderr + assert "will be removed in a future version" in result.stderr + + +def test_upload_ignore_file_pattern_env_migration() -> None: + legacy = _run_constants( + env={"UPLOAD_IGNORE_FILE_PATTERN": "*.legacy"}, + call="get_upload_ignore_file_pattern", + ) + assert json.loads(legacy.stdout) == "*.legacy" + assert "UPLOAD_IGNORE_FILE_PATTERN" in legacy.stderr + assert "will be removed in a future version" in legacy.stderr + + canonical = _run_constants( + env={ + "MODELSCOPE_UPLOAD_IGNORE_FILE_PATTERN": "*.canonical", + "UPLOAD_IGNORE_FILE_PATTERN": "*.legacy", + }, + call="get_upload_ignore_file_pattern", + ) + assert json.loads(canonical.stdout) == "*.canonical" + assert "FutureWarning" not in canonical.stderr def test_size_units_and_http_method_normalization() -> None: @@ -231,6 +257,7 @@ def test_deprecated_upload_env_aliases( assert json.loads(result.stdout)[constant] == expected assert legacy_env in result.stderr assert "FutureWarning" in result.stderr + assert "will be removed in a future version" in result.stderr def test_deprecated_http_retry_methods_are_normalized() -> None: @@ -244,14 +271,12 @@ def test_deprecated_http_retry_methods_are_normalized() -> None: "PATCH", ] assert "UPLOAD_RETRY_ALLOWED_METHODS" in result.stderr + assert "will be removed in a future version" in result.stderr def test_dead_lfs_threshold_is_not_registered() -> None: result = _run_constants("ENV_REGISTRY") - names = { - item["name"] - for item in json.loads(result.stdout)["ENV_REGISTRY"] - } + names = {item["name"] for item in json.loads(result.stdout)["ENV_REGISTRY"]} assert "UPLOAD_LFS_THRESHOLD" not in names From 709eecf3832489515218de03107125df23ca293c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 14:56:11 +0800 Subject: [PATCH 5/5] fix ruff lint --- src/modelscope_hub/_legacy_api.py | 3 +-- src/modelscope_hub/_openapi.py | 7 ++----- src/modelscope_hub/_upload.py | 4 +--- src/modelscope_hub/compat/hub_api.py | 11 +++-------- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index 539fc28..2e70d52 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -805,8 +805,7 @@ def upload_blob( upload_url, data=data, headers=upload_headers, - timeout=timeout - or (UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS), + timeout=timeout or (UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS), ) except requests.ConnectionError as exc: raise NetworkError(f"Blob upload connection failed: {exc}") from exc diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 7f917b5..198007e 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -690,14 +690,11 @@ def _flatten_mcp_list_params(body: Mapping[str, Any]) -> QueryParams: params.append( ( f"filter.{filter_key}", - str(filter_value).lower() - if isinstance(filter_value, bool) else str(filter_value), + str(filter_value).lower() if isinstance(filter_value, bool) else str(filter_value), ) ) else: - params.append( - (key, str(value).lower() if isinstance(value, bool) else str(value)) - ) + params.append((key, str(value).lower() if isinstance(value, bool) else str(value))) return params @staticmethod diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index 7cdac54..43d49f8 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -734,9 +734,7 @@ def upload_folder( ) else: commit_batch_size = ( - UPLOAD_COMMIT_BATCH_MAX_OPERATIONS - if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 - else len(sorted_files) + UPLOAD_COMMIT_BATCH_MAX_OPERATIONS if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 else len(sorted_files) ) # Initialize tracker diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 792be6f..21b4b6d 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -240,9 +240,7 @@ def _create_aigc_model(self, model_id: str, aigc_model: Any, kwargs: dict[str, A chinese_name = kwargs.pop("chinese_name", None) original_model_id = kwargs.pop("original_model_id", "") gated_mode = kwargs.pop("gated_mode", None) - readme_content = _resolve_aigc_readme_content( - aigc_model, kwargs.pop("readme_content", None) - ) + readme_content = _resolve_aigc_readme_content(aigc_model, kwargs.pop("readme_content", None)) if kwargs: unexpected = ", ".join(sorted(kwargs)) raise TypeError(f"create_model() got unexpected keyword argument(s): {unexpected}") @@ -319,7 +317,7 @@ def create_model_tag( raise InvalidParameter( f'tag_name "{tag_name}" is not allowed. ' 'Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). ' - 'Reserved names: main, master' + "Reserved names: main, master" ) api = self._api @@ -358,10 +356,7 @@ def create_model_tag( for attempt in range(5): try: _, tags = api.legacy.list_revisions_detail(model_id, "model") - if any( - item.get("ShowName") == tag_name or item.get("Revision") == tag_name - for item in tags - ): + if any(item.get("ShowName") == tag_name or item.get("Revision") == tag_name for item in tags): break except (NetworkError, RequestTimeoutError): pass