From 9e69e9795aedc346ed6c36f8037a721cc0946908 Mon Sep 17 00:00:00 2001 From: m86hussein Date: Sun, 8 Nov 2026 11:55:48 +0000 Subject: [PATCH] Make ChatGPT connector timeout configurable The ChatGPT connector wrapped the completion call in asyncio.wait_for with a hard-coded 10s timeout, so long-running completions (e.g. reasoning models) failed with TimeoutError even when callers set a timeout in chatgpt_config. Read the timeout from chatgpt_config (defaulting to TIMEOUT_SECONDS) so the wait_for guard and the OpenAI request agree. Fixes flyteorg/flyte#8287 Signed-off-by: m86hussein --- .../openai/chatgpt/connector.py | 4 +- .../tests/chatgpt/test_connector.py | 69 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/connector.py b/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/connector.py index eaa0e478b5..e2a94b5a93 100644 --- a/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/connector.py +++ b/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/connector.py @@ -43,7 +43,9 @@ async def do( logger = logging.getLogger("httpx") logger.setLevel(logging.WARNING) - completion = await asyncio.wait_for(client.chat.completions.create(**custom["chatgpt_config"]), TIMEOUT_SECONDS) + # Honor a caller-supplied timeout; the OpenAI SDK also accepts this key, so wait_for and the request agree. + timeout = custom["chatgpt_config"].get("timeout", TIMEOUT_SECONDS) + completion = await asyncio.wait_for(client.chat.completions.create(**custom["chatgpt_config"]), timeout) message = completion.choices[0].message.content outputs = {"o0": message} diff --git a/plugins/flytekit-openai/tests/chatgpt/test_connector.py b/plugins/flytekit-openai/tests/chatgpt/test_connector.py index d18e1f6975..d9bb47449a 100644 --- a/plugins/flytekit-openai/tests/chatgpt/test_connector.py +++ b/plugins/flytekit-openai/tests/chatgpt/test_connector.py @@ -3,6 +3,7 @@ import pytest from flyteidl.core.execution_pb2 import TaskExecution +from flytekitplugins.openai.chatgpt.connector import TIMEOUT_SECONDS from flytekit.extend.backend.base_connector import ConnectorRegistry from flytekit.interfaces.cli_identifiers import Identifier @@ -74,3 +75,71 @@ async def test_chatgpt_connector(): assert response.phase == TaskExecution.SUCCEEDED assert response.outputs == {"o0": message} + + +def _build_chatgpt_template(chatgpt_config): + task_id = Identifier( + resource_type=ResourceType.TASK, project="project", domain="domain", name="name", version="version" + ) + task_config = { + "openai_organization": "test-openai-orgnization-id", + "chatgpt_config": chatgpt_config, + } + task_metadata = TaskMetadata( + True, + RuntimeMetadata(RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"), + timedelta(days=1), + literals.RetryStrategy(3), + True, + "0.1.1b0", + "This is deprecated!", + True, + "A", + (), + ) + return TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=None, + type="chatgpt", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chatgpt_config, expected_timeout", + [ + ({"model": "gpt-3.5-turbo", "temperature": 0.7}, TIMEOUT_SECONDS), + ({"model": "gpt-3.5-turbo", "temperature": 0.7, "timeout": 120}, 120), + ], +) +async def test_chatgpt_connector_timeout_from_config(chatgpt_config, expected_timeout): + connector = ConnectorRegistry.get_connector("chatgpt") + tmp = _build_chatgpt_template(chatgpt_config) + task_inputs = LiteralMap( + { + "message": literals.Literal( + scalar=literals.Scalar(primitive=literals.Primitive(string_value="Test ChatGPT Plugin")) + ), + }, + ) + + captured = {} + + async def fake_wait_for(coro, timeout): + captured["timeout"] = timeout + return await coro + + mock_context = mock.MagicMock() + mock_secrets = mock.MagicMock() + mock_secrets.get.return_value = "mocked_openai_api_key" + mock_context.secrets = mock_secrets + + with mock.patch("flytekit.FlyteContextManager.current_context", return_value=mock_context): + with mock.patch("openai.resources.chat.completions.AsyncCompletions.create", new=mock_acreate): + with mock.patch("asyncio.wait_for", new=fake_wait_for): + response = await connector.do(tmp, task_inputs) + + assert response.phase == TaskExecution.SUCCEEDED + assert captured["timeout"] == expected_timeout