Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ async def do(
logger = logging.getLogger("httpx")
logger.setLevel(logging.WARNING)

completion = await asyncio.wait_for(client.chat.completions.create(**custom["chatgpt_config"]), TIMEOUT_SECONDS)
# Honor a caller-supplied timeout; the OpenAI SDK also accepts this key, so wait_for and the request agree.
timeout = custom["chatgpt_config"].get("timeout", TIMEOUT_SECONDS)
completion = await asyncio.wait_for(client.chat.completions.create(**custom["chatgpt_config"]), timeout)
message = completion.choices[0].message.content
outputs = {"o0": message}

Expand Down
69 changes: 69 additions & 0 deletions plugins/flytekit-openai/tests/chatgpt/test_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import pytest
from flyteidl.core.execution_pb2 import TaskExecution
from flytekitplugins.openai.chatgpt.connector import TIMEOUT_SECONDS

from flytekit.extend.backend.base_connector import ConnectorRegistry
from flytekit.interfaces.cli_identifiers import Identifier
Expand Down Expand Up @@ -74,3 +75,71 @@ async def test_chatgpt_connector():

assert response.phase == TaskExecution.SUCCEEDED
assert response.outputs == {"o0": message}


def _build_chatgpt_template(chatgpt_config):
task_id = Identifier(
resource_type=ResourceType.TASK, project="project", domain="domain", name="name", version="version"
)
task_config = {
"openai_organization": "test-openai-orgnization-id",
"chatgpt_config": chatgpt_config,
}
task_metadata = TaskMetadata(
True,
RuntimeMetadata(RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"),
timedelta(days=1),
literals.RetryStrategy(3),
True,
"0.1.1b0",
"This is deprecated!",
True,
"A",
(),
)
return TaskTemplate(
id=task_id,
custom=task_config,
metadata=task_metadata,
interface=None,
type="chatgpt",
)


@pytest.mark.asyncio
@pytest.mark.parametrize(
"chatgpt_config, expected_timeout",
[
({"model": "gpt-3.5-turbo", "temperature": 0.7}, TIMEOUT_SECONDS),
({"model": "gpt-3.5-turbo", "temperature": 0.7, "timeout": 120}, 120),
],
)
async def test_chatgpt_connector_timeout_from_config(chatgpt_config, expected_timeout):
connector = ConnectorRegistry.get_connector("chatgpt")
tmp = _build_chatgpt_template(chatgpt_config)
task_inputs = LiteralMap(
{
"message": literals.Literal(
scalar=literals.Scalar(primitive=literals.Primitive(string_value="Test ChatGPT Plugin"))
),
},
)

captured = {}

async def fake_wait_for(coro, timeout):
captured["timeout"] = timeout
return await coro

mock_context = mock.MagicMock()
mock_secrets = mock.MagicMock()
mock_secrets.get.return_value = "mocked_openai_api_key"
mock_context.secrets = mock_secrets

with mock.patch("flytekit.FlyteContextManager.current_context", return_value=mock_context):
with mock.patch("openai.resources.chat.completions.AsyncCompletions.create", new=mock_acreate):
with mock.patch("asyncio.wait_for", new=fake_wait_for):
response = await connector.do(tmp, task_inputs)

assert response.phase == TaskExecution.SUCCEEDED
assert captured["timeout"] == expected_timeout