From b286a716533e468b41ef88a911e2d7f474327cf6 Mon Sep 17 00:00:00 2001 From: Lion Summerbell Date: Thu, 18 Jun 2026 12:17:03 +0200 Subject: [PATCH] feature[hunyuan translation]: translate chinese dialects with tencent hunyuan + shared resources --- .../datashare_python/dependencies.py | 16 ++ datashare-python/datashare_python/objects.py | 26 ++- datashare-python/tests/test_dependencies.py | 67 +++++- datashare-python/uv.lock | 3 +- workers/translation-worker/pyproject.toml | 8 +- workers/translation-worker/tests/conftest.py | 46 +++- .../tests/test_activities.py | 10 +- .../tests/test_hunyuan_translator.py | 191 +++++++++++++++++ .../translation-worker/tests/test_objects.py | 15 +- .../tests/test_workflows.py | 59 ++++- .../translation_worker/__init__.py | 5 +- .../translation_worker/activities.py | 10 +- .../translation_worker/config.py | 101 ++++++++- .../translation_worker/constants.py | 42 ++++ .../translation_worker/dependencies.py | 35 ++- .../translation_worker/objects.py | 75 +------ .../translation_worker/processors.py | 12 +- .../translation_worker/sentence_splitters.py | 16 +- .../translators/__init__.py | 5 + .../translation_worker/translators/argos.py | 3 +- .../translation_worker/translators/hunyuan.py | 105 +++++++++ .../translation_worker/workflows.py | 34 ++- workers/translation-worker/uv.lock | 202 +++++++++++++++++- 23 files changed, 968 insertions(+), 118 deletions(-) create mode 100644 workers/translation-worker/tests/test_hunyuan_translator.py create mode 100644 workers/translation-worker/translation_worker/translators/hunyuan.py diff --git a/datashare-python/datashare_python/dependencies.py b/datashare-python/datashare_python/dependencies.py index 505e0244..afc44a55 100644 --- a/datashare-python/datashare_python/dependencies.py +++ b/datashare-python/datashare_python/dependencies.py @@ -12,6 +12,7 @@ from .config import LogLevel, WorkerConfig from .exceptions import DependencyInjectionError from .logging_ import setup_worker_loggers +from .objects import Shared from .task_client import DatashareTaskClient from .types_ import ContextManagerFactory, TemporalClient @@ -23,6 +24,7 @@ TASK_CLIENT: ContextVar[DatashareTaskClient] = ContextVar("task_client") TEMPORAL_CLIENT: ContextVar[TemporalClient] = ContextVar("temporal_client") WORKER_CONFIG: ContextVar[WorkerConfig] = ContextVar("worker_config") +SHARED: ContextVar[Shared] = ContextVar("shared") def set_event_loop(event_loop: AbstractEventLoop) -> None: @@ -99,6 +101,20 @@ def lifespan_temporal_client() -> TemporalClient: raise DependencyInjectionError("temporal client") from e +# Setup shared resources +async def set_shared_resources(shared: Shared) -> Shared: + SHARED.set(shared) + return shared + + +# Return shared resources +def shared_resources() -> Shared: + try: + return SHARED.get() + except LookupError as e: + raise DependencyInjectionError("shared resources") from e + + @asynccontextmanager async def with_dependencies( dependencies: list[ContextManagerFactory], **kwargs diff --git a/datashare-python/datashare_python/objects.py b/datashare-python/datashare_python/objects.py index da23f8ad..72a6d0cc 100644 --- a/datashare-python/datashare_python/objects.py +++ b/datashare-python/datashare_python/objects.py @@ -1,7 +1,8 @@ import logging import os +from asyncio import Lock from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from enum import StrEnum, unique from io import BytesIO @@ -367,3 +368,26 @@ class TaskGroup: @classmethod def python(cls) -> Self: return cls(name="PYTHON") + + +@dataclass(frozen=True) +class Shared: + _resources: dict[str, Any] = field(default_factory=dict) + _lock: Lock = field(default_factory=Lock) + + def get_resource(self, key: str, default: Any = None) -> Any: + return self._resources.get(key, default) + + def set_resource(self, key: str, value: Any) -> None: + self._resources[key] = value + + def pop_resource(self, key: str, default: Any = None) -> Any: + return self._resources.pop(key, default) + + async def async_set_resource(self, key: str, value: Any) -> None: + async with self._lock: + self._resources[key] = value + + async def async_pop_resource(self, key: str, default: Any = None) -> Any: + async with self._lock: + return self._resources.pop(key, default) diff --git a/datashare-python/tests/test_dependencies.py b/datashare-python/tests/test_dependencies.py index e8d16934..bd7f0c7b 100644 --- a/datashare-python/tests/test_dependencies.py +++ b/datashare-python/tests/test_dependencies.py @@ -1,7 +1,72 @@ from typing import Any import pytest -from datashare_python.dependencies import add_missing_args +from datashare_python.dependencies import ( + add_missing_args, + set_shared_resources, + shared_resources, +) +from datashare_python.exceptions import DependencyInjectionError +from datashare_python.objects import Shared + + +async def test_set_shared_resources_and_get() -> None: + shared = Shared() + shared.set_resource("k", "v") + returned = await set_shared_resources(shared) + assert returned is shared + assert shared_resources() is shared + + +async def test_set_shared_resources_overwrites() -> None: + first = Shared() + second = Shared() + first.set_resource("k", "v1") + second.set_resource("k", "v2") + await set_shared_resources(first) + await set_shared_resources(second) + assert shared_resources() is second + + +def test_shared_resources_raises_before_set() -> None: + with pytest.raises(DependencyInjectionError): + shared_resources() + + +def test_shared_resources_field_is_frozen() -> None: + shared = Shared() + shared.set_resource("k", "v") + with pytest.raises(Exception): # noqa: B017, PT011 + shared._resources = {} + + +def test_get_resource_existing_key() -> None: + shared = Shared() + shared.set_resource("k", "v") + assert shared.get_resource("k") == "v" + + +def test_get_resource_missing_key_with_default() -> None: + shared = Shared() + assert shared.get_resource("missing", "default") == "default" + + +async def test_async_set_resource() -> None: + shared = Shared() + await shared.async_set_resource("k", "v") + assert shared.get_resource("k") == "v" + + +async def test_async_set_resource_overwrites() -> None: + shared = Shared() + shared.set_resource("k", "old") + await shared.async_set_resource("k", "new") + assert shared.get_resource("k") == "new" + + +async def test_async_pop_resource_missing_key_with_default() -> None: + shared = Shared() + assert await shared.async_pop_resource("missing", None) is None @pytest.mark.parametrize( diff --git a/datashare-python/uv.lock b/datashare-python/uv.lock index 5adba0d6..889c4635 100644 --- a/datashare-python/uv.lock +++ b/datashare-python/uv.lock @@ -440,7 +440,7 @@ wheels = [ [[package]] name = "datashare-python" -version = "0.8.19" +version = "0.8.23" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -544,7 +544,6 @@ dev = [ { name = "pytest", specifier = "~=8.1" }, { name = "pytest-asyncio", specifier = "~=0.24" }, { name = "pytest-timeout", specifier = "==2.4.0" }, - { name = "redis", extras = ["hiredis"], specifier = ">=5.2.1" }, { name = "ruff", specifier = "==0.15.2" }, { name = "typing-extensions", specifier = ">=4.15.0" }, ] diff --git a/workers/translation-worker/pyproject.toml b/workers/translation-worker/pyproject.toml index aef67c0b..58ee3e03 100644 --- a/workers/translation-worker/pyproject.toml +++ b/workers/translation-worker/pyproject.toml @@ -12,6 +12,7 @@ requires-python = ">=3.11.0, <3.13" dependencies = [ "datashare-python~=0.8.20", + "language-data>=1.4.0", "pydantic-extra-types[pycountry]==2.11.1", ] @@ -28,9 +29,14 @@ dependencies = "translation_worker.dependencies:REGISTRY" worker_config_cls = "translation_worker.config:WORKER_CONFIG_CLS" [project.optional-dependencies] -inference = [ +argos_inference = [ "argostranslate==1.11.0", ] +hunyuan_inference = [ + "torch>=2.11.0", + "transformers>=5.12.1", + "accelerate>=1.14.0", +] [tool.uv.sources] datashare-python = { path = "../../datashare-python", editable = true } diff --git a/workers/translation-worker/tests/conftest.py b/workers/translation-worker/tests/conftest.py index a7fa9216..f00f9f1a 100644 --- a/workers/translation-worker/tests/conftest.py +++ b/workers/translation-worker/tests/conftest.py @@ -24,21 +24,31 @@ test_worker_config, worker_lifetime_deps, ) -from datashare_python.objects import DatashareLanguage, Document +from datashare_python.dependencies import SHARED, shared_resources +from datashare_python.objects import DatashareLanguage, Document, Shared from datashare_python.types_ import ContextManagerFactory, TemporalClient from datashare_python.worker import worker_context from icij_common.es import ESClient from translation_worker.activities import TranslationActivities from translation_worker.config import TranslationWorkerConfig -from translation_worker.dependencies import set_es_client, set_worker_config +from translation_worker.dependencies import ( + set_es_client, + set_hunyuan_translator, + set_worker_config, +) from translation_worker.workflows import TaskQueue, TranslationWorkflow @pytest.fixture(scope="session") -def test_deps() -> list[ContextManagerFactory]: +def test_io_deps() -> list[ContextManagerFactory]: return [set_worker_config, set_es_client] +@pytest.fixture(scope="session") +def test_inference_deps() -> list[ContextManagerFactory]: + return [set_worker_config, set_es_client, set_hunyuan_translator] + + @pytest.fixture(scope="session") def test_worker_config(tmp_path_factory: TempPathFactory) -> TranslationWorkerConfig: # noqa: ANN001, ARG001, F811 tmp_path = tmp_path_factory.mktemp("test-") @@ -67,6 +77,7 @@ def test_worker_config(tmp_path_factory: TempPathFactory) -> TranslationWorkerCo ENGLISH = "ENGLISH" FRENCH = "FRENCH" SPANISH = "SPANISH" +CHINESE = "CHINESE" DOC_ID_1 = "doc_id_1" DOC_ID_2 = "doc_id_2" ROOT_DOCUMENT_1 = "root_document_1" @@ -74,6 +85,7 @@ def test_worker_config(tmp_path_factory: TempPathFactory) -> TranslationWorkerCo DS_ENGLISH = DatashareLanguage(ENGLISH) DS_FRENCH = DatashareLanguage(FRENCH) DS_SPANISH = DatashareLanguage(SPANISH) +DS_CHINESE = DatashareLanguage(CHINESE) FRENCH_TEXT = ( @@ -87,6 +99,10 @@ def test_worker_config(tmp_path_factory: TempPathFactory) -> TranslationWorkerCo "vez. Besame.... Besame mucho.... Que tengo miedo perderte, " "perderte otra vez." ) +CHINESE_TEXT = ( + "北京是中国的首都。这是一个美丽的城市，有着悠久的历史和文化。" + "每年都有数以百万计的游客来这里参观故宫和长城。" +) def _create_doc( @@ -110,6 +126,16 @@ async def index_translation_documents( return docs +@pytest.fixture +async def index_chinese_translation_documents( + test_es_client: ESClient, # noqa: F811 +) -> list[Document]: + docs = [_create_doc(DOC_ID_1, ROOT_DOCUMENT_1, CHINESE_TEXT, DS_CHINESE)] + async for _ in index_docs(test_es_client, docs=docs, index_name=TEST_PROJECT): + pass + return docs + + @pytest.fixture(scope="session") def translation_worker_config() -> TranslationWorkerConfig: return TranslationWorkerConfig() @@ -142,7 +168,7 @@ async def io_worker( test_worker_config: TranslationWorkerConfig, # noqa: F811 test_temporal_client_session: TemporalClient, # noqa: F811 event_loop: asyncio.AbstractEventLoop, # noqa: F811 - test_deps: list[ContextManagerFactory], # noqa: F811 + test_io_deps: list[ContextManagerFactory], # noqa: F811 ) -> AsyncGenerator[None, None]: client = test_temporal_client_session worker_id = f"test-translation-io-worker-{uuid.uuid4()}" @@ -163,7 +189,7 @@ async def io_worker( client=client, event_loop=event_loop, task_queue=task_queue, - dependencies=test_deps, + dependencies=test_io_deps, ) async with worker_ctx: yield @@ -174,7 +200,7 @@ async def translation_inference_worker( test_worker_config: TranslationWorkerConfig, # noqa: F811 test_temporal_client_session: TemporalClient, # noqa: F811 event_loop: asyncio.AbstractEventLoop, # noqa: F811 - test_deps: list[ContextManagerFactory], # noqa: F811 + test_inference_deps: list[ContextManagerFactory], # noqa: F811 ) -> AsyncGenerator[None, None]: client = test_temporal_client_session worker_id = f"test-translation-cpu-worker-{uuid.uuid4()}" @@ -190,7 +216,13 @@ async def translation_inference_worker( client=client, event_loop=event_loop, task_queue=task_queue, - dependencies=test_deps, + dependencies=test_inference_deps, ) async with worker_ctx: yield + + +@pytest.fixture(scope="session") +def test_shared_resources() -> Shared: + SHARED.set(Shared()) + return shared_resources() diff --git a/workers/translation-worker/tests/test_activities.py b/workers/translation-worker/tests/test_activities.py index 98106be5..8ee38ff4 100644 --- a/workers/translation-worker/tests/test_activities.py +++ b/workers/translation-worker/tests/test_activities.py @@ -26,9 +26,13 @@ create_translation_batches_act, translate_docs_act, ) -from translation_worker.config import TranslationWorkerConfig +from translation_worker.config import ( + ArgosTranslatorConfig, + TranslationModel, + TranslationWorkerConfig, +) from translation_worker.constants import DOC_CONTENT_TEXT_LENGTH -from translation_worker.objects import TranslationModel, untranslated_query +from translation_worker.objects import untranslated_query from translation_worker.processors import SentenceSplitter, Translator from tests.conftest import ( @@ -80,7 +84,7 @@ class MockTranslator(Translator): registered_name = TranslationModel.ARGOS def __init__(self, translations: list[str]): - super().__init__() + super().__init__(ArgosTranslatorConfig()) self._translations = translations def translate(self, texts: Iterable[str]) -> list[str]: # noqa: ARG002 diff --git a/workers/translation-worker/tests/test_hunyuan_translator.py b/workers/translation-worker/tests/test_hunyuan_translator.py new file mode 100644 index 00000000..0bd45d32 --- /dev/null +++ b/workers/translation-worker/tests/test_hunyuan_translator.py @@ -0,0 +1,191 @@ +# ruff: noqa: ANN001, ANN202 +import sys +from unittest.mock import MagicMock, patch + +from datashare_python.objects import Shared +from translation_worker.constants import DEFAULT_HUNYUAN_MODEL_REF, HUNYUAN_SHARED_KEY +from translation_worker.translators.hunyuan import HunyuanMtTranslator + +from tests.conftest import DS_CHINESE, DS_ENGLISH + +TEXT_TO_TRANSLATE = "text to translate" +MORE_TEXT_TO_TRANSLATE = "more text to translate" +TRANSLATED_TEXT = "translated text" + + +def _make_config(max_new_tokens: int = 2048, torch_dtype: str = "float32") -> MagicMock: + config = MagicMock() + + config.max_new_tokens = max_new_tokens + config.torch_dtype = torch_dtype + + return config + + +def _translator(device: str = "cpu", **config_kwargs) -> HunyuanMtTranslator: + translator = HunyuanMtTranslator(_make_config(**config_kwargs), device=device) + + translator._source = DS_CHINESE + translator._target = DS_ENGLISH + + return translator + + +def _dummy_transformers() -> MagicMock: + mock = MagicMock() + mock.AutoTokenizer = MagicMock() + mock.AutoModelForCausalLM = MagicMock() + return mock + + +def test__translate__wraps_each_text_in_prompt() -> None: + translator = _translator() + translator._tokenizer = MagicMock() + translator._translator = MagicMock() + translator.translate([TEXT_TO_TRANSLATE, MORE_TEXT_TO_TRANSLATE]) + translator._tokenizer.apply_chat_template.assert_called_once_with( + [ + [ + { + "role": "user", + "content": f"Translate into English, " + f"without additional explanation: " + f"{TEXT_TO_TRANSLATE}", + } + ], + [ + { + "role": "user", + "content": f"Translate into English, " + f"without additional explanation: " + f"{MORE_TEXT_TO_TRANSLATE}", + } + ], + ], + tokenize=True, + add_generation_prompt=False, + return_tensors="pt", + padding=True, + ) + + +def test__translate__moves_tokenized_input_to_device_before_generation() -> None: + translator = _translator(device="cpu") + mock_tokenizer = MagicMock() + mock_model = MagicMock() + translator._tokenizer = mock_tokenizer + translator._translator = mock_model + tokenized = MagicMock() + mock_tokenizer.apply_chat_template.return_value = tokenized + translator.translate([TEXT_TO_TRANSLATE]) + input_ids = tokenized["input_ids"] + input_ids.to.assert_called_once_with("cpu") + mock_model.generate.assert_called_once_with( + input_ids.to.return_value, max_new_tokens=2048 + ) + + +def test__translate__decodes_first_element_of_generate_output() -> None: + translator = _translator() + mock_tokenizer = MagicMock() + mock_model = MagicMock() + translator._tokenizer = mock_tokenizer + translator._translator = mock_model + first_output = MagicMock() + mock_model.generate.return_value = [first_output] + mock_tokenizer.decode.return_value = TRANSLATED_TEXT + result = translator.translate([TEXT_TO_TRANSLATE]) + mock_tokenizer.decode.assert_called_once_with( + first_output, skip_special_tokens=True + ) + assert result == [TRANSLATED_TEXT] + + +def test__load__initialises_tokenizer_and_model_from_config_model_ref_with_device_map_and_dtype() -> ( # noqa: E501 + None +): + translator = _translator() + translator._config.model_ref = DEFAULT_HUNYUAN_MODEL_REF + translator._config.torch_dtype = "bfloat16" + translator._config.device_map = "cuda:0" + dummy_tf = _dummy_transformers() + with ( + patch.dict(sys.modules, {"transformers": dummy_tf}), + ): + translator._load(MagicMock(), target=MagicMock(), worker_config=MagicMock()) + + dummy_tf.AutoTokenizer.from_pretrained.assert_called_once_with( + DEFAULT_HUNYUAN_MODEL_REF + ) + + dummy_tf.AutoModelForCausalLM.from_pretrained.assert_called_once_with( + DEFAULT_HUNYUAN_MODEL_REF, + device_map="cuda:0", + torch_dtype="bfloat16", + ) + + +def test__load__assigns_loaded_tokenizer_and_model_to_instance_without_shared_resources() -> ( # noqa: E501 + None +): + translator = _translator() + dummy_tf = _dummy_transformers() + mock_tokenizer = MagicMock() + mock_model = MagicMock() + mock_model.to.return_value = mock_model + dummy_tf.AutoTokenizer.from_pretrained.return_value = mock_tokenizer + dummy_tf.AutoModelForCausalLM.from_pretrained.return_value = mock_model + + with ( + patch.dict(sys.modules, {"transformers": dummy_tf}), + ): + translator._load(MagicMock(), target=MagicMock(), worker_config=MagicMock()) + + assert translator._tokenizer is mock_tokenizer + assert translator._translator is mock_model + + +def test__load__assigns_loaded_tokenizer_and_model_to_instance_with_shared_resources( + test_shared_resources: Shared, +) -> None: + translator = _translator() + dummy_tf = _dummy_transformers() + mock_tokenizer = MagicMock() + mock_model = MagicMock() + mock_model.to.return_value = mock_model + dummy_tf.AutoTokenizer.from_pretrained.return_value = mock_tokenizer + dummy_tf.AutoModelForCausalLM.from_pretrained.return_value = mock_model + + test_shared_resources.set_resource(HUNYUAN_SHARED_KEY, None) + + with ( + patch.dict(sys.modules, {"transformers": dummy_tf}), + ): + translator._load(MagicMock(), target=MagicMock(), worker_config=MagicMock()) + + assert translator._tokenizer is mock_tokenizer + assert translator._translator is mock_model + assert test_shared_resources.get_resource(HUNYUAN_SHARED_KEY) is mock_model + + +# @pytest.mark.parametrize( +# ("source", "initial_translator", "expected_translator_type"), +# [ +# # should route to hunyuan +# (DS_CHINESE, ArgosTranslatorConfig(), HunyuanMtTranslatorConfig), +# # should remain with argos +# (DS_FRENCH, ArgosTranslatorConfig(), ArgosTranslatorConfig), +# # should remain with hunyuan +# (DS_CHINESE, HunyuanMtTranslatorConfig(), HunyuanMtTranslatorConfig), +# # should route to argos +# (DS_FRENCH, HunyuanMtTranslatorConfig(), ArgosTranslatorConfig), +# ], +# ) +# def test__set_config_translator( +# source: DatashareLanguage, +# initial_translator: ArgosTranslatorConfig | HunyuanMtTranslatorConfig, +# expected_translator_type: type, +# ) -> None: +# config = TranslationConfig(translator=initial_translator) +# result = _set_config_translator(source, config) +# assert isinstance(result.translator, expected_translator_type) diff --git a/workers/translation-worker/tests/test_objects.py b/workers/translation-worker/tests/test_objects.py index f30631e9..26eaf380 100644 --- a/workers/translation-worker/tests/test_objects.py +++ b/workers/translation-worker/tests/test_objects.py @@ -2,14 +2,16 @@ import pytest from datashare_python.conftest import TEST_PROJECT -from translation_worker.objects import ( +from translation_worker.config import ( ArgosSentenceSplitterConfig, - ArgosSentencizer, ArgosTranslatorConfig, + TranslationConfig, +) +from translation_worker.objects import ( + ArgosSentencizer, DocId, DocumentSearchQuery, TranslationArgs, - TranslationConfig, ) from .conftest import DS_ENGLISH @@ -55,7 +57,12 @@ def test_translation_args_as_query( docs: list[DocId] | DocumentSearchQuery | None, expected_query: dict[str, Any] ) -> None: # Given - args = TranslationArgs(project=TEST_PROJECT, docs=docs, target_language=DS_ENGLISH) + args = TranslationArgs( + project=TEST_PROJECT, + docs=docs, + target_language=DS_ENGLISH, + config=TranslationConfig(), + ) # When query = args.as_query() # Then diff --git a/workers/translation-worker/tests/test_workflows.py b/workers/translation-worker/tests/test_workflows.py index cb5e2aa3..234a8a87 100644 --- a/workers/translation-worker/tests/test_workflows.py +++ b/workers/translation-worker/tests/test_workflows.py @@ -1,13 +1,18 @@ import uuid import pytest -from datashare_python.conftest import TEST_PROJECT +from datashare_python.conftest import TEST_PROJECT, test_deps # noqa from datashare_python.objects import DatashareLanguage, Document from icij_common.es import HITS, ESClient, has_type from temporalio.client import Client as TemporalClient from temporalio.worker import Worker +from translation_worker.config import ( + TranslationConfig, +) from translation_worker.constants import TaskQueue -from translation_worker.objects import TranslationArgs +from translation_worker.objects import ( + TranslationArgs, +) from translation_worker.workflows import TranslationWorkflow from .conftest import DS_ENGLISH @@ -24,7 +29,9 @@ async def test_translation_workflow( ) -> None: # Given args = TranslationArgs( - project=TEST_PROJECT, target_language=DatashareLanguage("ENGLISH") + project=TEST_PROJECT, + target_language=DatashareLanguage("ENGLISH"), + config=TranslationConfig(), ) workflow_id = f"translation-{uuid.uuid4().hex}" @@ -48,3 +55,49 @@ async def test_translation_workflow( any(ct.target_language == DS_ENGLISH for ct in doc.content_translated) for doc in index_docs ) + + +# @pytest.mark.e2e +# async def test_translation_workflow_routes_chinese_to_hunyuan( +# test_temporal_client_session: TemporalClient, # noqa: ARG001 +# index_chinese_translation_documents: list[Document], # noqa: ARG001 +# test_es_client: ESClient, +# workflows_worker: Worker, # noqa: ARG001 +# io_worker: Worker, # noqa: ARG001 +# translation_inference_worker: Worker, # noqa: ARG001 +# ) -> None: +# # Given +# args = TranslationArgs( +# project=TEST_PROJECT, +# target_language=DatashareLanguage("ENGLISH"), +# config=TranslationConfig( +# translator=HunyuanMtTranslatorConfig(), +# sentence_splitter=DefaultSentenceSplitterConfig(), +# ), +# ) +# workflow_id = f"translation-{uuid.uuid4().hex}" +# +# # When +# res = await test_temporal_client_session.execute_workflow( +# TranslationWorkflow.run, args, id=workflow_id, task_queue=TaskQueue.WORKFLOWS +# ) +# +# assert res.n_translations == 1 +# +# body = {"query": has_type(type_field="type", type_value="Document")} +# sort = "_doc:asc" +# index_docs = [] +# async for hits in test_es_client.poll_search_pages( +# index=TEST_PROJECT, body=body, sort=sort +# ): +# index_docs += hits[HITS][HITS] +# assert len(index_docs) == 1 +# index_docs = [Document.from_es(doc) for doc in index_docs] +# assert all( +# any( +# ct.target_language == DS_ENGLISH +# and ct.translator == TranslationModel.HUNYUAN +# for ct in doc.content_translated +# ) +# for doc in index_docs +# ) diff --git a/workers/translation-worker/translation_worker/__init__.py b/workers/translation-worker/translation_worker/__init__.py index 0519c120..b3e15aff 100644 --- a/workers/translation-worker/translation_worker/__init__.py +++ b/workers/translation-worker/translation_worker/__init__.py @@ -2,7 +2,8 @@ with workflow.unsafe.imports_passed_through(): try: - from .sentence_splitters import ArgosSentenceSplitter + from .sentence_splitters import ArgosSentenceSplitter, DefaultSentenceSplitter except ImportError: ArgosSentenceSplitter = None - from .translators import ArgosTranslator # noqa: F401 + DefaultSentenceSplitter = None + from .translators import ArgosTranslator, HunyuanMtTranslator # noqa: F401 diff --git a/workers/translation-worker/translation_worker/activities.py b/workers/translation-worker/translation_worker/activities.py index e58cad21..22d15a37 100644 --- a/workers/translation-worker/translation_worker/activities.py +++ b/workers/translation-worker/translation_worker/activities.py @@ -24,13 +24,12 @@ has_type, ) from icij_common.iter_utils import async_batches, before_and_after, once -from pydantic import TypeAdapter from translation_worker.constants import DOC_CONTENT_TEXT_LENGTH -from .config import TranslationWorkerConfig +from .config import TranslationConfig, TranslationWorkerConfig from .constants import BATCHING_DOC_SOURCES, TRANSLATION_DOC_SOURCES -from .objects import Language, TranslationConfig +from .objects import _LANGUAGE_TYPE_ADAPTER, Language from .processors import SentenceSplitter, Translator logger = logging.getLogger(__name__) @@ -38,8 +37,6 @@ DocId = str Batch = list[DocId] -_LANGUAGE_TYPE_ADAPTER = TypeAdapter(Language) - class TranslationActivities(ActivityWithProgress): @activity_defn(name="translation.worker-config") @@ -90,7 +87,8 @@ async def translate_docs( # TODO: perform some caching here to avoid reloading translator = config.to_translator() sentence_splitter = config.to_sentence_splitter() - # Load the translator first to install the SBD, then load the splitter + # Load the translator first to install the + # SBD, then load the splitter logger.debug("loading %s -> %s translator...", source, target) with translator.load(source, target=target, worker_config=worker_config): logger.debug("loading %s sentence splitter...", source) diff --git a/workers/translation-worker/translation_worker/config.py b/workers/translation-worker/translation_worker/config.py index 1de76b46..9c38e2ba 100644 --- a/workers/translation-worker/translation_worker/config.py +++ b/workers/translation-worker/translation_worker/config.py @@ -1,8 +1,25 @@ +from abc import ABC +from typing import TYPE_CHECKING, ClassVar + from datashare_python.config import WorkerConfig -from datashare_python.objects import DatashareModel -from pydantic import Field +from datashare_python.objects import BaseModel, DatashareModel +from icij_common.pydantic_utils import make_enum_discriminator, tagged_union +from icij_common.registrable import RegistrableConfig +from pydantic import Discriminator, Field + +from translation_worker.objects import ( + ArgosSentencizer, + SentenceSplitterModel, + TranslationModel, +) + +from .constants import DEFAULT_HUNYUAN_MODEL_REF, TorchDevice + +if TYPE_CHECKING: + from translation_worker.processors import SentenceSplitter, Translator + -from .constants import TorchDevice +class _BaseProcessorConfig(BaseModel, RegistrableConfig, ABC): ... class C2TranslateConfig(DatashareModel): @@ -23,4 +40,82 @@ class TranslationWorkerConfig(WorkerConfig): c2_translate: C2TranslateConfig = Field(default_factory=C2TranslateConfig) +class SentenceSplitterConfig(_BaseProcessorConfig): + registry_key: ClassVar[str] = Field(frozen=True, default="model") + model: ClassVar[SentenceSplitterModel] + + +class DefaultSentenceSplitterConfig(SentenceSplitterConfig): + model: ClassVar[SentenceSplitterModel] = SentenceSplitterModel.DEFAULT + + +class ArgosSentenceSplitterConfig(SentenceSplitterConfig): + model: ClassVar[SentenceSplitterModel] = SentenceSplitterModel.ARGOS + + sentencizer: ArgosSentencizer = ArgosSentencizer.MINI_SBD + + +class TranslatorConfig(_BaseProcessorConfig): + registry_key: ClassVar[str] = Field(frozen=True, default="model") + model: ClassVar[TranslationModel] + + +_SentenceSplitterConfig = tagged_union( + SentenceSplitterConfig.__subclasses__(), lambda t: t.model +) +splitter_discriminator = make_enum_discriminator("model", SentenceSplitterModel) + + +class ArgosTranslatorConfig(TranslatorConfig): + model: ClassVar[TranslationModel] = TranslationModel.ARGOS + + beam_size: int = 2 + length_penalty: float = 0.2 + + +class HunyuanMtTranslatorConfig(TranslatorConfig): + model_config = {"arbitrary_types_allowed": True} + model: ClassVar[TranslationModel] = TranslationModel.HUNYUAN + + model_ref: str = DEFAULT_HUNYUAN_MODEL_REF + max_new_tokens: int = 2048 + top_k: int = 20 + top_p: float = 0.6 + repetition_penalty: float = 1.05 + temperature: float = 0.7 + torch_dtype: str = "float32" + device_map: str = "auto" + + +_TranslatorConfig = tagged_union(TranslatorConfig.__subclasses__(), lambda t: t.model) +translator_discriminator = make_enum_discriminator("model", TranslationModel) + + +class TranslationConfig(DatashareModel): + sentence_splitter: _SentenceSplitterConfig = Field( + discriminator=Discriminator(splitter_discriminator), + default_factory=DefaultSentenceSplitterConfig, + ) + translator: _TranslatorConfig = Field( + discriminator=Discriminator(translator_discriminator), + default_factory=ArgosTranslatorConfig, + ) + + def to_sentence_splitter(self) -> "SentenceSplitter": + from .processors import SentenceSplitter # noqa: PLC0415 + + return SentenceSplitter.from_config(self.sentence_splitter) + + def to_translator(self) -> "Translator": + from .processors import Translator # noqa: PLC0415 + + return Translator.from_config(self.translator) + + WORKER_CONFIG_CLS = TranslationWorkerConfig + +# Allows us to get around a circular import with objects.py + +from .objects import TranslationArgs # noqa: E402 + +TranslationArgs.model_rebuild(_types_namespace={"TranslationConfig": TranslationConfig}) diff --git a/workers/translation-worker/translation_worker/constants.py b/workers/translation-worker/translation_worker/constants.py index 79e88738..e4f1a588 100644 --- a/workers/translation-worker/translation_worker/constants.py +++ b/workers/translation-worker/translation_worker/constants.py @@ -22,3 +22,45 @@ class TorchDevice(StrEnum): DOC_CONTENT_TEXT_LENGTH = "contentTextLength" TRANSLATION_DOC_SOURCES = [DOC_CONTENT, DOC_ROOT_ID, DOC_LANGUAGE] BATCHING_DOC_SOURCES = TRANSLATION_DOC_SOURCES[1:] + [DOC_CONTENT_TEXT_LENGTH] + +DEFAULT_HUNYUAN_MODEL_REF = "tencent/Hunyuan-MT-Chimera-7B" +HUNYUAN_SHARED_KEY = "hunyuan_translator" + +HUNYUAN_LANGUAGES = [ + # "en", + "zh", + # "fr", + # "pt", + # "es", + # "ja", + # "tr", + # "ru", + # "ar", + # "ko", + # "th", + # "it", + # "de", + # "vi", + # "ms", + # "id", + # "tl", + # "hi", + # "pl", + # "cs", + # "nl", + # "km", + # "my", + # "fa", + # "gu", + # "ur", + # "te", + # "mr", + # "he", + # "bn", + # "ta", + # "uk", + # "bo", + # "kk", + # "mn", + # "ug", +] diff --git a/workers/translation-worker/translation_worker/dependencies.py b/workers/translation-worker/translation_worker/dependencies.py index 33602649..89daa4bf 100644 --- a/workers/translation-worker/translation_worker/dependencies.py +++ b/workers/translation-worker/translation_worker/dependencies.py @@ -1,16 +1,49 @@ +import gc import logging +from contextlib import asynccontextmanager +from typing import Any from datashare_python.dependencies import ( # noqa: F401 lifespan_es_client, lifespan_worker_config, set_es_client, + set_shared_resources, set_worker_config, + shared_resources, ) +from datashare_python.exceptions import DependencyInjectionError +from datashare_python.objects import Shared + +from .constants import HUNYUAN_SHARED_KEY logger = logging.getLogger(__name__) +@asynccontextmanager +async def set_hunyuan_translator(**kwargs) -> None: # noqa: ARG001 + shared = Shared() + await set_shared_resources(shared) + try: + yield + finally: + import torch # noqa: PLC0415 + + model = await shared.async_pop_resource(HUNYUAN_SHARED_KEY, None) + if model is not None: + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + gc.collect() + + +def lifespan_hunyuan_translator() -> Any: + try: + return shared_resources().get_resource(HUNYUAN_SHARED_KEY) + except (LookupError, KeyError) as e: + raise DependencyInjectionError("hunyuan translator") from e + + REGISTRY = { - "translation.inference": [set_worker_config, set_es_client], + "translation.inference": [set_worker_config, set_es_client, set_hunyuan_translator], "translation.io": [set_worker_config, set_es_client], } diff --git a/workers/translation-worker/translation_worker/objects.py b/workers/translation-worker/translation_worker/objects.py index 4f2f65ea..0716d830 100644 --- a/workers/translation-worker/translation_worker/objects.py +++ b/workers/translation-worker/translation_worker/objects.py @@ -1,9 +1,7 @@ -from abc import ABC from enum import StrEnum -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any from datashare_python.objects import ( - BaseModel, DatashareModel, Language, ) @@ -15,29 +13,24 @@ has_id, must_not, ) -from icij_common.registrable import RegistrableConfig -from pydantic import Field +from pydantic import TypeAdapter -from .processors import SentenceSplitter, Translator +from .constants import HUNYUAN_LANGUAGES if TYPE_CHECKING: from argostranslate.sbd import ISentenceBoundaryDetectionModel - -class _BaseProcessorConfig(BaseModel, RegistrableConfig, ABC): ... + from translation_worker.objects import TranslationConfig class SentenceSplitterModel(StrEnum): ARGOS = "ARGOS" + DEFAULT = "DEFAULT" class TranslationModel(StrEnum): ARGOS = "ARGOS" - - -class SentenceSplitterConfig(_BaseProcessorConfig): - registry_key: ClassVar[str] = Field(frozen=True, default="model") - model: ClassVar[SentenceSplitterModel] + HUNYUAN = "HUNYUAN" class ArgosSentencizer(StrEnum): @@ -60,57 +53,11 @@ def sentencizer_cls(self) -> type["ISentenceBoundaryDetectionModel"]: raise NotImplementedError() -class ArgosSentenceSplitterConfig(SentenceSplitterConfig): - model: ClassVar[SentenceSplitterModel] = SentenceSplitterModel.ARGOS - - sentencizer: ArgosSentencizer = ArgosSentencizer.MINI_SBD - - -class TranslatorConfig(_BaseProcessorConfig): - registry_key: ClassVar[str] = Field(frozen=True, default="model") - model: ClassVar[TranslationModel] - - -class ArgosTranslatorConfig(TranslatorConfig): - model: ClassVar[TranslationModel] = TranslationModel.ARGOS - - beam_size: int = 2 - length_penalty: float = 0.2 - - -# TODO: uncomment when adding more implems -# _SentenceSplitterConfig = tagged_union( -# SentenceSplitterConfig.__subclasses__(), lambda t: t.model.default.value -# ) -# splitter_discriminator = make_enum_discriminator("model", SentenceSplitterModel) - -# TODO: uncomment when adding more implems -# _TranslatorConfig = tagged_union( -# TranslatorConfig.__subclasses__(), lambda t: t.model.default.value -# ) -# translator_discriminator = make_enum_discriminator("model", TranslationModel) - - -class TranslationConfig(DatashareModel): - sentence_splitter: ArgosSentenceSplitterConfig = Field( - # TODO: uncomment when adding more implem - # discriminator=Discriminator(model_discriminator=splitter_discriminator), - default_factory=ArgosSentenceSplitterConfig, - ) - translator: ArgosTranslatorConfig = Field( - # discriminator=Discriminator(model_discriminator=splitter_discriminator), - default_factory=ArgosTranslatorConfig, - ) - - def to_sentence_splitter(self) -> "SentenceSplitter": - from .processors import SentenceSplitter # noqa: PLC0415 - - return SentenceSplitter.from_config(self.sentence_splitter) - - def to_translator(self) -> "Translator": - from .processors import Translator # noqa: PLC0415 +_LANGUAGE_TYPE_ADAPTER = TypeAdapter(Language) - return Translator.from_config(self.translator) +_VALIDATED_HUNYUAN_LANGUAGES = { + _LANGUAGE_TYPE_ADAPTER.validate_python(lang) for lang in HUNYUAN_LANGUAGES +} DocumentSearchQuery = dict[str, Any] @@ -127,7 +74,7 @@ def untranslated_query(target: Language) -> dict: class TranslationArgs(DatashareModel): project: str docs: list[DocId] | DocumentSearchQuery | None = None - config: TranslationConfig = Field(default_factory=TranslationConfig) + config: "TranslationConfig" target_language: Language def as_query(self) -> dict[str, Any]: diff --git a/workers/translation-worker/translation_worker/processors.py b/workers/translation-worker/translation_worker/processors.py index 941176f4..2b791606 100644 --- a/workers/translation-worker/translation_worker/processors.py +++ b/workers/translation-worker/translation_worker/processors.py @@ -9,7 +9,9 @@ from .config import TranslationWorkerConfig if TYPE_CHECKING: + from .config import TranslatorConfig from .objects import Language + logger = logging.getLogger(__name__) @@ -37,7 +39,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): ... # noqa: ANN001 class Translator(RegistrableFromConfig): - def __init__(self): + def __init__(self, config: "TranslatorConfig"): + self._config = config + self._source: Language | None = None self._target: Language | None = None @@ -63,9 +67,9 @@ def load( source: "Language", *, target: "Language", - worker_config: "TranslationWorkerConfig | None" = None, + worker_config: TranslationWorkerConfig | None = None, ) -> Self: - if worker_config is not None: + if worker_config is None: worker_config = TranslationWorkerConfig() with self: self._load(source, target=target, worker_config=worker_config) @@ -76,7 +80,7 @@ def _load( source: "Language", *, target: "Language", - worker_config: "TranslationWorkerConfig", # noqa: ARG002 + worker_config: TranslationWorkerConfig, # noqa: ARG002 ) -> None: self._source = source self._target = target diff --git a/workers/translation-worker/translation_worker/sentence_splitters.py b/workers/translation-worker/translation_worker/sentence_splitters.py index 2078d642..ac661152 100644 --- a/workers/translation-worker/translation_worker/sentence_splitters.py +++ b/workers/translation-worker/translation_worker/sentence_splitters.py @@ -4,10 +4,24 @@ from datashare_python.objects import DatashareLanguage from pydantic_extra_types.language_code import LanguageAlpha2 -from .objects import ArgosSentenceSplitterConfig, Language, SentenceSplitterModel +from .config import ArgosSentenceSplitterConfig, DefaultSentenceSplitterConfig +from .objects import Language, SentenceSplitterModel from .processors import SentenceSplitter +@SentenceSplitter.register(SentenceSplitterModel.DEFAULT) +class DefaultSentenceSplitter(SentenceSplitter): + def __init__(self, config: DefaultSentenceSplitterConfig): + self._config = config + + @classmethod + def _from_config(cls, config: DefaultSentenceSplitterConfig) -> Self: + return cls(config) + + def split_sentences(self, text: str) -> list[str]: + return [text] + + @SentenceSplitter.register(SentenceSplitterModel.ARGOS) class ArgosSentenceSplitter(SentenceSplitter): def __init__(self, config: ArgosSentenceSplitterConfig): diff --git a/workers/translation-worker/translation_worker/translators/__init__.py b/workers/translation-worker/translation_worker/translators/__init__.py index cb403819..d78b4f89 100644 --- a/workers/translation-worker/translation_worker/translators/__init__.py +++ b/workers/translation-worker/translation_worker/translators/__init__.py @@ -2,3 +2,8 @@ from .argos import ArgosTranslator except ImportError: ArgosTranslator = None + +try: + from .hunyuan import HunyuanMtTranslator +except ImportError: + HunyuanMtTranslator = None diff --git a/workers/translation-worker/translation_worker/translators/argos.py b/workers/translation-worker/translation_worker/translators/argos.py index b2d9f8ed..e9a71285 100644 --- a/workers/translation-worker/translation_worker/translators/argos.py +++ b/workers/translation-worker/translation_worker/translators/argos.py @@ -25,11 +25,10 @@ @Translator.register(TranslationModel.ARGOS) class ArgosTranslator(Translator): def __init__(self, config: "ArgosTranslatorConfig"): - super().__init__() + super().__init__(config) self._translator = None self._tokenizer = None self._target_prefix = None - self._config = config @classmethod def _from_config(cls, config: "ArgosTranslatorConfig", **extras) -> Self: # noqa: ARG003 diff --git a/workers/translation-worker/translation_worker/translators/hunyuan.py b/workers/translation-worker/translation_worker/translators/hunyuan.py new file mode 100644 index 00000000..1a4cb0db --- /dev/null +++ b/workers/translation-worker/translation_worker/translators/hunyuan.py @@ -0,0 +1,105 @@ +import gc +from collections.abc import Iterable +from typing import TYPE_CHECKING, Self + +from datashare_python.objects import Language + +from ..config import HunyuanMtTranslatorConfig, TranslationWorkerConfig +from ..constants import HUNYUAN_SHARED_KEY +from ..objects import TranslationModel +from ..processors import Translator + +if TYPE_CHECKING: + import torch + + +def _message_template(text: str, target_lang: str) -> dict[str, str]: + context = f"Translate into {target_lang}, without additional explanation: {text}" + return {"role": "user", "content": context} + + +@Translator.register(TranslationModel.HUNYUAN) +class HunyuanMtTranslator(Translator): + def __init__( + self, config: HunyuanMtTranslatorConfig, device: "torch.device | str" = "cpu" + ): + super().__init__(config) + + self._tokenizer = None + self._translator = None + self._device = device + + @classmethod + def _from_config( + cls, + config: HunyuanMtTranslatorConfig, + device: "torch.device | str" = "cpu", + **extras, # noqa: ARG003 + ) -> Self: + return cls(config, device=device) + + def _load( + self, + source: Language, + *, + target: Language, + worker_config: TranslationWorkerConfig, + ) -> None: + from datashare_python.dependencies import shared_resources # noqa: PLC0415 + from datashare_python.exceptions import ( # noqa: PLC0415 + DependencyInjectionError, + ) + from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: PLC0415 + + super()._load(source, target=target, worker_config=worker_config) + + try: + shared = shared_resources() + except DependencyInjectionError: + shared = None + + translator = ( + shared.get_resource(HUNYUAN_SHARED_KEY) if shared is not None else None + ) + + if translator is None: + translator = AutoModelForCausalLM.from_pretrained( + self._config.model_ref, + device_map=self._config.device_map, + torch_dtype=self._config.torch_dtype, + ) + if shared is not None: + shared.set_resource(HUNYUAN_SHARED_KEY, translator) + + self._translator = translator + self._tokenizer = AutoTokenizer.from_pretrained(self._config.model_ref) + self._device = next(self._translator.parameters()).device + + def translate(self, texts: Iterable[str]) -> list[str]: + target_lang = self._target.title() + conversations = [[_message_template(text, target_lang)] for text in texts] + tokenized = self._tokenizer.apply_chat_template( + conversations, + tokenize=True, + add_generation_prompt=False, + return_tensors="pt", + padding=True, + ) + input_ids = tokenized["input_ids"] if hasattr(tokenized, "keys") else tokenized + outputs = self._translator.generate( + input_ids.to(self._device), max_new_tokens=self._config.max_new_tokens + ) + return [ + self._tokenizer.decode(out, skip_special_tokens=True) for out in outputs + ] + + def __exit__(self, exc_type, exc_val, exc_tb): # noqa: ANN001 + import torch # noqa: PLC0415 + + self._tokenizer = None + self._translator = None + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + gc.collect() diff --git a/workers/translation-worker/translation_worker/workflows.py b/workers/translation-worker/translation_worker/workflows.py index 4973fad1..64fb8daf 100644 --- a/workers/translation-worker/translation_worker/workflows.py +++ b/workers/translation-worker/translation_worker/workflows.py @@ -1,6 +1,7 @@ import asyncio from datetime import timedelta +from datashare_python.objects import DatashareLanguage from icij_common.iter_utils import batches from temporalio import workflow @@ -8,9 +9,30 @@ from datashare_python.utils import WorkflowWithProgress, execute_activity from .activities import TranslationActivities - from .config import TranslationWorkerConfig + from .config import ( + ArgosTranslatorConfig, + HunyuanMtTranslatorConfig, + TranslationConfig, + TranslationWorkerConfig, + ) from .constants import TRANSLATION_WORKFLOW_NAME, TaskQueue - from .objects import TranslationArgs, TranslationResponse + from .objects import ( + _VALIDATED_HUNYUAN_LANGUAGES, + TranslationArgs, + TranslationResponse, + ) + + +def _set_config_translator( + source: DatashareLanguage, config: "TranslationConfig" +) -> "TranslationConfig": + source_code = source.alpha2 if hasattr(source, "alpha2") else str(source) + if source_code in _VALIDATED_HUNYUAN_LANGUAGES: + if not isinstance(config.translator, HunyuanMtTranslatorConfig): + return config.model_copy(update={"translator": HunyuanMtTranslatorConfig()}) + elif isinstance(config.translator, HunyuanMtTranslatorConfig): + return config.model_copy(update={"translator": ArgosTranslatorConfig()}) + return config @workflow.defn(name=TRANSLATION_WORKFLOW_NAME) @@ -37,7 +59,13 @@ async def run(self, args: TranslationArgs) -> TranslationResponse: # Translate translation_args = [ - (b, source, target, args.config, args.project) + ( + b, + source, + target, + args.config, + args.project, + ) for source, languages_batches in per_language_batches for b in batches(languages_batches, batch_size=batches_per_worker) ] diff --git a/workers/translation-worker/uv.lock b/workers/translation-worker/uv.lock index 36554b7e..3881e399 100644 --- a/workers/translation-worker/uv.lock +++ b/workers/translation-worker/uv.lock @@ -11,6 +11,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/cd/7ee00d6aa023b1d0551da0da5fee3bc23c3eeea632fbfc5126d1fec52b7e/about_time-4.2.1-py3-none-any.whl", hash = "sha256:8bbf4c75fe13cbd3d72f49a03b02c5c7dca32169b6d49117c257e7eb3eaee341", size = 13295, upload-time = "2022-12-21T04:15:53.613Z" }, ] +[[package]] +name = "accelerate" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8d/75/94cd5d389649578aca399e5aa822637eec18319a1dadc400ffe2f9a7493f/accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d", size = 412167, upload-time = "2026-06-11T13:45:52.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/db/253133d7e7cb40d3af384bb2f5c0b4a2b7fdcffbc95c688cc67a20a3c103/accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6", size = 389246, upload-time = "2026-06-11T13:45:50.477Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -107,6 +125,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/39/cade3a5a97fffa3ae84f298208237b3a9f7112d6b0ed57e8ff4b755e44b4/alive_progress-3.2.0-py3-none-any.whl", hash = "sha256:0677929f8d3202572e9d142f08170b34dbbe256cc6d2afbf75ef187c7da964a8", size = 77106, upload-time = "2024-10-26T04:22:29.103Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -260,14 +287,14 @@ wheels = [ [[package]] name = "click" -version = "8.1.8" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] @@ -465,13 +492,19 @@ name = "datashare-translation-worker" source = { editable = "." } dependencies = [ { name = "datashare-python" }, + { name = "language-data" }, { name = "pydantic-extra-types", extra = ["pycountry"] }, ] [package.optional-dependencies] -inference = [ +argos-inference = [ { name = "argostranslate" }, ] +hunyuan-inference = [ + { name = "accelerate" }, + { name = "torch" }, + { name = "transformers" }, +] [package.dev-dependencies] dev = [ @@ -489,11 +522,15 @@ dev = [ [package.metadata] requires-dist = [ - { name = "argostranslate", marker = "extra == 'inference'", specifier = "==1.11.0" }, + { name = "accelerate", marker = "extra == 'hunyuan-inference'", specifier = ">=1.14.0" }, + { name = "argostranslate", marker = "extra == 'argos-inference'", specifier = "==1.11.0" }, { name = "datashare-python", editable = "../../datashare-python" }, + { name = "language-data", specifier = ">=1.4.0" }, { name = "pydantic-extra-types", extras = ["pycountry"], specifier = "==2.11.1" }, + { name = "torch", marker = "extra == 'hunyuan-inference'", specifier = ">=2.11.0" }, + { name = "transformers", marker = "extra == 'hunyuan-inference'", specifier = ">=5.12.1" }, ] -provides-extras = ["inference"] +provides-extras = ["argos-inference", "hunyuan-inference"] [package.metadata.requires-dev] dev = [ @@ -642,6 +679,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/e7/ae38d7a6dfba0533684e0b2136817d667588ae3ec984c1a4e5df5eb88482/hatchling-1.27.0-py3-none-any.whl", hash = "sha256:d3a2f3567c4f926ea39849cdf924c7e99e6686c9c8e288ae1037c8fa2a5d937b", size = 75794, upload-time = "2024-12-15T17:08:10.364Z" }, ] +[[package]] +name = "hf-xet" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" }, + { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" }, + { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" }, + { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" }, +] + [[package]] name = "hiredis" version = "3.3.0" @@ -704,6 +757,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/27/629cfe58c582f92ded066c4a07d1a057ff617118ab7973200f770bd853cb/huggingface_hub-1.19.0.tar.gz", hash = "sha256:fd771622182d40977272a923953ee3b1b13538f9f8a7f5d78398f10af0f1c0bd", size = 824721, upload-time = "2026-06-11T12:33:18.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a5/558da89f66464d8d0229ff497e8b8666977de2d8cf48c28a2862ecf1250f/huggingface_hub-1.19.0-py3-none-any.whl", hash = "sha256:1dc72e1f6b4d6df6b30eb72e57d00514ef453d660f04af2b87f0e67267f31ee0", size = 693398, upload-time = "2026-06-11T12:33:16.695Z" }, +] + [[package]] name = "icij-common" version = "0.8.2" @@ -781,6 +855,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/c1/d10b371bcba7abce05e2b33910e39c33cfa496a53f13640b7b8e10bb4d2b/langcodes-3.5.1-py3-none-any.whl", hash = "sha256:b6a9c25c603804e2d169165091d0cdb23934610524a21d226e4f463e8e958a72", size = 183050, upload-time = "2025-12-02T16:21:59.954Z" }, ] +[[package]] +name = "language-data" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marisa-trie" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/50/2518b4d0805f4d1f10166837829ad0bd71dcee3ec33fa84aa8c0db23c13c/language_data-1.4.0.tar.gz", hash = "sha256:800e6457e7beda781c156e02d7707e38db2ded026472e07e2c055dc8446ee574", size = 5309660, upload-time = "2025-11-28T13:45:25.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/d1/68e2bcca94c9bbdc122a71e504e0b6a6c3e31541b1bad33fee0205996006/language_data-1.4.0-py3-none-any.whl", hash = "sha256:f741927c24ab14cbed2a57bc2bfe82b00cff266c427179597e8b14123364f084", size = 5572678, upload-time = "2025-11-28T13:45:23.381Z" }, +] + +[[package]] +name = "marisa-trie" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/5d/e235921b5b74818cb65b557fa05cc6201c2c1612d4866ff75c835bcf808d/marisa_trie-1.4.1.tar.gz", hash = "sha256:44ce3bdbeb7c950d463e460184fc3e18702df9ef0edb826bac672fd789fb1d20", size = 261581, upload-time = "2026-04-08T07:17:52.991Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/94/1ad851729ba0cdbc269bdd72b4b725cfbf5a25e4186fd12e56836d2d52ba/marisa_trie-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:579d1498e6b9e8f139b36601d2ea35e9239e849ff1615f3c3fc8df8ce4d3a936", size = 208644, upload-time = "2026-04-08T07:16:25.926Z" }, + { url = "https://files.pythonhosted.org/packages/11/67/2a8870ef1c42412ace3d656830902893fad6239434cde749f6654f907b41/marisa_trie-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a6404610eca835cf179c4407bcaa00d7acfbf3fd7aafcc1413d3adc262b554c", size = 192740, upload-time = "2026-04-08T07:16:26.971Z" }, + { url = "https://files.pythonhosted.org/packages/b7/3f/d1d67e1ae5058dfcbb3756f75a26ea203acae875584636d360fd4a38b248/marisa_trie-1.4.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:284ff4b2a63f00e175c7fe88d18c23a556c988ce705eb8e15a65e60ad7f86a98", size = 1506335, upload-time = "2026-04-08T07:16:28.378Z" }, + { url = "https://files.pythonhosted.org/packages/da/23/086b81133baccbc05feb2fbc4113c70775352f3b9851dc8a20d69b2db44f/marisa_trie-1.4.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29eb718078431518d13037830c50023333721b146ca58eb78889aabfa60f4c33", size = 1529146, upload-time = "2026-04-08T07:16:30.061Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a0/c717ffb7697f059cde03b75c9facf6fd4ee18ce720513156bb03e875d215/marisa_trie-1.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ac478766ff9381f1bc18f39f694388f64c20cfa8cb2b308e41ece2b4ce05467", size = 2436516, upload-time = "2026-04-08T07:16:31.452Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d7/c5f5cd9eb32a34447971ee3430410b19087564137d62da7bd348b40e684a/marisa_trie-1.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f7024c2cb001442fe04b9720be105bbe01ff2d7f357b70fa44d42272abf7da1f", size = 2532453, upload-time = "2026-04-08T07:16:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/32/56/84fe4788b29a81e00efb4c97f3c1b7cae0dbda14369dc7ee2659823bfd83/marisa_trie-1.4.1-cp311-cp311-win32.whl", hash = "sha256:c059562d5aea86bf623a2c440b8595a86c0de553ca96986e4d36f25d07570d5b", size = 140594, upload-time = "2026-04-08T07:16:33.883Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b0/3f793c5f7727ca1d36f9d53b9fc48804942918a9f460b6d46988770677b2/marisa_trie-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:c74606bd7e0066f20cf7187de44c955ba3b4ce85159a43f8cd8b0ee982ea4c4c", size = 175072, upload-time = "2026-04-08T07:16:35.254Z" }, + { url = "https://files.pythonhosted.org/packages/fe/8c/ac1851b7c9ce881343bf0639c902aaa2b767a31dc79d6af5950413a6ba66/marisa_trie-1.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:59a5c286329a5defa33c40cce1f16c9829e4128b57ecc851ac32a7d1071913d5", size = 142560, upload-time = "2026-04-08T07:16:36.425Z" }, + { url = "https://files.pythonhosted.org/packages/b3/b7/89811f7eba6e92386279376df81cfa281ab99e30f7e4f5a5e04d8dba6b99/marisa_trie-1.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:63964dedbf49ef0d17cb32d368f13ec71ca0ec026976b1cc24cb6a993d05752a", size = 206731, upload-time = "2026-04-08T07:16:37.439Z" }, + { url = "https://files.pythonhosted.org/packages/f7/8b/cc34313149486dfc13e84303e12d61fd55788b37d92c3e082cc3d142e776/marisa_trie-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87e65dff37d1b9edea7bc7a8e935c851ec4934f2e56071a4501ce8db97b579a4", size = 190988, upload-time = "2026-04-08T07:16:38.686Z" }, + { url = "https://files.pythonhosted.org/packages/15/0c/376e21c62bd0e658a5e9f6b8912f3116591778c639857ad374c7639ceebe/marisa_trie-1.4.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7bed50d39ff1391a67b9383a7f3c458a1a0cb40fe8dd16952f813fbf8939eeff", size = 1471836, upload-time = "2026-04-08T07:16:39.88Z" }, + { url = "https://files.pythonhosted.org/packages/bb/95/cd6e73d0857608f2946f3bc5ccac86488073b96fb37bc1b45b0184268bed/marisa_trie-1.4.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d51bdd22a7238ef4d681effd7c224a267ddae054b64b1cec9ce95bbcd2b6a88", size = 1516414, upload-time = "2026-04-08T07:16:41.4Z" }, + { url = "https://files.pythonhosted.org/packages/51/73/339e8fab2e8cea88e9e0fd78aeb8ccdd3f8656d228dae2cb698f667a0fe7/marisa_trie-1.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8da4dea083209301430d80c8a33d0a5ecb6a270c904743505adceaae4fface2", size = 2394325, upload-time = "2026-04-08T07:16:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d7/0ba8bcaeee68a8e6cbc61b47825370a6c8a523ab16ea42e8728dec2213bc/marisa_trie-1.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e7f9603cc8a57dca847febf45349c51916c3e1340eb6ee064baabf181398dc79", size = 2510974, upload-time = "2026-04-08T07:16:44.848Z" }, + { url = "https://files.pythonhosted.org/packages/23/ef/fa342fdbc0c055030b93007dff5393675705071a14621e3f81bfb52eb970/marisa_trie-1.4.1-cp312-cp312-win32.whl", hash = "sha256:63cd2870f3890f2657610ed437110713e87972da0dc4d3e6303d370c9b28d215", size = 138890, upload-time = "2026-04-08T07:16:46.871Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/419114325f1bb4c2202c20f19f424f9dea1cc38de7cd1fae60d991e99b69/marisa_trie-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:fc9bc6de7197cdd1f32b72566cc7ac75c465d6f2191bba51d17edfae2b5ca8b0", size = 168513, upload-time = "2026-04-08T07:16:48.475Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f8/ae0dcbf79498b7aa00dae740982c9812fa95339bc6549ea63b4ad15eeb58/marisa_trie-1.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:59375ab1e4e4cee87d318b6b3dffa91c599c89afd920ef53428235f4326ba1d6", size = 139710, upload-time = "2026-04-08T07:16:49.863Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1703,6 +1815,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/f0/89ee2bc9da434bd78464f288fdb346bc2932f2ee80a90b2a4bbbac262c74/sacremoses-0.1.1-py3-none-any.whl", hash = "sha256:31e04c98b169bfd902144824d191825cd69220cdb4ae4bcf1ec58a7db5587b1a", size = 897476, upload-time = "2023-10-30T15:56:18.121Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + [[package]] name = "sentencepiece" version = "0.2.1" @@ -1942,6 +2078,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/8d/683703de021ffbe46833d722b70f49ffbbca8e5bd6876256977555d92d7d/thinc-8.3.13-cp312-cp312-win_arm64.whl", hash = "sha256:c6a049703a6011c8fe26ee41af7e70272145594140d82f79bb23de619c6a6525", size = 1645777, upload-time = "2026-03-23T07:22:08.104Z" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -1994,6 +2156,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "transformers" +version = "5.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/7c/8240f612819718100a9346dc28dea6a11370c3ca9c8c6eabadd3dea4ef29/transformers-5.12.1.tar.gz", hash = "sha256:679ee731c8225347889ad4fb3b2c926a62e9da3b7d284e9d12c791da7272466b", size = 8924054, upload-time = "2026-06-15T17:27:50.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/56/bbd60dd8668055803bf8ba55a81f9b8a8b31497f620109a9671d26a2076d/transformers-5.12.1-py3-none-any.whl", hash = "sha256:2a5e109d2021265df7098ffbb738295acaf5ad256f12cbc586db2ea4dcbb1a8a", size = 11150587, upload-time = "2026-06-15T17:27:46.679Z" }, +] + [[package]] name = "triton" version = "3.6.0" @@ -2016,17 +2198,17 @@ wheels = [ [[package]] name = "typer" -version = "0.15.4" +version = "0.25.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-doc" }, { name = "click" }, { name = "rich" }, { name = "shellingham" }, - { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/89/c527e6c848739be8ceb5c44eb8208c52ea3515c6cf6406aa61932887bf58/typer-0.15.4.tar.gz", hash = "sha256:89507b104f9b6a0730354f27c39fae5b63ccd0c95b1ce1f1a6ba0cfd329997c3", size = 101559, upload-time = "2025-05-14T16:34:57.704Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/27/ede8cec7596e0041ba7e7b80b47d132562f56ff454313a16f6084e555c9f/typer-0.25.0.tar.gz", hash = "sha256:123eaf9f19bb40fd268310e12a542c0c6b4fab9c98d9d23342a01ff95e3ce930", size = 120150, upload-time = "2026-04-26T08:46:14.767Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/62/d4ba7afe2096d5659ec3db8b15d8665bdcb92a3c6ff0b95e99895b335a9c/typer-0.15.4-py3-none-any.whl", hash = "sha256:eb0651654dcdea706780c466cf06d8f174405a659ffff8f163cfbfee98c0e173", size = 45258, upload-time = "2025-05-14T16:34:55.583Z" }, + { url = "https://files.pythonhosted.org/packages/9a/72/193d4e586ec5a4db834a36bbeb47641a62f951f114ffd0fe5b1b46e8d56f/typer-0.25.0-py3-none-any.whl", hash = "sha256:ac01b48823d3db9a83c9e164338057eadbb1c9957a2a6b4eeb486669c560b5dc", size = 55993, upload-time = "2026-04-26T08:46:15.889Z" }, ] [[package]]