From d74accc78eb02ee1323a121bd9db53f8cc52be36 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 30 Oct 2026 10:28:19 +0000 Subject: [PATCH 1/4] Fix Lambda cold-start timeout caused by torch import Import ultralytics lazily on first evaluation so the IPC server opens its socket inside Lambda's 10 s init window, and install CPU-only torch/torchvision on Linux to drop the multi-GB CUDA libraries from the image. Also bump Poetry in the Dockerfile to 2.x to match poetry.lock. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 3 +- evaluation_function/evaluation.py | 6 +- poetry.lock | 152 ++++++++++++++++++++++++++---- pyproject.toml | 18 +++- 4 files changed, 158 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38ebf5e..11980b1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM ghcr.io/lambda-feedback/evaluation-function-base/python:3.12 AS builder -RUN pip install poetry==1.8.3 +# Must match the Poetry major version that generated poetry.lock +RUN pip install "poetry>=2,<3" ENV POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py index 8c48c90..00b8e1b 100755 --- a/evaluation_function/evaluation.py +++ b/evaluation_function/evaluation.py @@ -2,7 +2,6 @@ from typing import Any from lf_toolkit.evaluation import Result, Params from lf_toolkit.evaluation.image_upload import upload_image -from ultralytics import YOLO from PIL import Image, ImageOps import io import requests @@ -47,6 +46,11 @@ def evaluation_function( # Use a dict to cache models by name if model_name not in _model_cache: + # Imported here rather than at module level: importing ultralytics + # pulls in torch, which takes longer than Lambda's 10 s init window + # on a cold start. Deferring it lets the IPC server start first. + from ultralytics import YOLO + model_dir = os.path.dirname(os.path.abspath(__file__)) # Strip any directory components so `model_name` can't be used to # load a model file from outside the evaluation_function directory. diff --git a/poetry.lock b/poetry.lock index d48b033..9e28337 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1964,7 +1964,7 @@ description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb"}, {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668"}, @@ -1978,7 +1978,7 @@ description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc"}, {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.whl", hash = "sha256:358b4a1d35370353d52e12f0a7d1769fc01ff74a191689d3870b2123156184c4"}, @@ -1994,7 +1994,7 @@ description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13"}, {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53"}, @@ -2008,7 +2008,7 @@ description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd"}, {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e"}, @@ -2024,7 +2024,7 @@ description = "cuDNN runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def"}, {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2"}, @@ -2041,7 +2041,7 @@ description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6"}, {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8510990de9f96c803a051822618d42bf6cb8f069ff3f48d93a8486efdacb48fb"}, @@ -2060,7 +2060,7 @@ description = "cuFile GPUDirect libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db"}, @@ -2073,7 +2073,7 @@ description = "CURAND native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8"}, {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf"}, @@ -2089,7 +2089,7 @@ description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0"}, {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c"}, @@ -2110,7 +2110,7 @@ description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887"}, {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1"}, @@ -2129,7 +2129,7 @@ description = "NVIDIA cuSPARSELt" optional = false python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1"}, {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46"}, @@ -2155,7 +2155,7 @@ description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6"}, @@ -2168,7 +2168,7 @@ description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a"}, {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41"}, @@ -2182,7 +2182,7 @@ description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b"}, {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:adcaabb9d436c9761fca2b13959a2d237c5f9fd406c8e4b723c695409ff88059"}, @@ -2591,7 +2591,7 @@ description = "C parser in Python" optional = false python-versions = ">=3.11" groups = ["main"] -markers = "(sys_platform == \"linux\" and platform_python_implementation != \"PyPy\" or sys_platform == \"darwin\") and implementation_name != \"PyPy\"" +markers = "(platform_python_implementation != \"PyPy\" or sys_platform == \"darwin\") and implementation_name != \"PyPy\" and (sys_platform == \"linux\" or sys_platform == \"darwin\")" files = [ {file = "pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e"}, {file = "pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6"}, @@ -3286,7 +3286,7 @@ description = "Most extensible Python build backend with support for C/C++ exten optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version >= \"3.12\" or platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "python_version >= \"3.12\" or platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "setuptools-84.2.1-py3-none-any.whl", hash = "sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2"}, {file = "setuptools-84.2.1.tar.gz", hash = "sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc"}, @@ -3374,6 +3374,7 @@ description = "Tensors and Dynamic neural networks in Python with strong GPU acc optional = false python-versions = ">=3.9.0" groups = ["main"] +markers = "sys_platform != \"linux\"" files = [ {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f"}, {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d"}, @@ -3429,6 +3430,74 @@ typing-extensions = ">=4.10.0" opt-einsum = ["opt-einsum (>=3.3)"] optree = ["optree (>=0.13.0)"] +[[package]] +name = "torch" +version = "2.10.0+cpu" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "sys_platform == \"linux\"" +files = [ + {file = "torch-2.10.0+cpu-cp310-cp310-linux_aarch64.whl", hash = "sha256:31ae44836c8b9bbd1a3943d29c7c7457709ddf7c6173aa34aefe9d2203e4c405"}, + {file = "torch-2.10.0+cpu-cp310-cp310-linux_s390x.whl", hash = "sha256:beadc2a6a1785b09a46daad378de91ef274b8d3eea7af0bc2d017d97f115afdf"}, + {file = "torch-2.10.0+cpu-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d63ee6a80982fd73fe44bb70d97d2976e010312ff6db81d7bfb9167b06dd45b9"}, + {file = "torch-2.10.0+cpu-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a280ffaea7b9c828e0c1b9b3bd502d9b6a649dc9416997b69b84544bd469f215"}, + {file = "torch-2.10.0+cpu-cp310-cp310-win_amd64.whl", hash = "sha256:6c6f0df770144907092a0d067048d96ed4f278a6c840376d2ff0e27e7579b925"}, + {file = "torch-2.10.0+cpu-cp311-cp311-linux_aarch64.whl", hash = "sha256:ce5c113d1f55f8c1f5af05047a24e50d11d293e0cbbb5bf7a75c6c761edd6eaa"}, + {file = "torch-2.10.0+cpu-cp311-cp311-linux_s390x.whl", hash = "sha256:0e286fcf6ce0cc7b204396c9b4ea0d375f1f0c3e752f68ce3d3aeb265511db8c"}, + {file = "torch-2.10.0+cpu-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1cfcb9b1558c6e52dffd0d4effce83b13c5ae5d97338164c372048c21f9cfccb"}, + {file = "torch-2.10.0+cpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b7cb1ec66cefb90fd7b676eac72cfda3b8d4e4d0cacd7a531963bc2e0a9710ab"}, + {file = "torch-2.10.0+cpu-cp311-cp311-win_amd64.whl", hash = "sha256:17a09465bab2aab8f0f273410297133d8d8fb6dd84dccbd252ca4a4f3a111847"}, + {file = "torch-2.10.0+cpu-cp311-cp311-win_arm64.whl", hash = "sha256:c35c0de592941d4944698dbfa87271ab85d3370eca3b694943a2ab307ac34b3f"}, + {file = "torch-2.10.0+cpu-cp312-cp312-linux_aarch64.whl", hash = "sha256:8de5a36371b775e2d4881ed12cc7f2de400b1ad3d728aa74a281f649f87c9b8c"}, + {file = "torch-2.10.0+cpu-cp312-cp312-linux_s390x.whl", hash = "sha256:9accc30b56cb6756d4a9d04fcb8ebc0bb68c7d55c1ed31a8657397d316d31596"}, + {file = "torch-2.10.0+cpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:179451716487f8cb09b56459667fa1f5c4c0946c1e75fbeae77cfc40a5768d87"}, + {file = "torch-2.10.0+cpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ee40b8a4b4b2cf0670c6fd4f35a7ef23871af956fecb238fbf5da15a72650b1d"}, + {file = "torch-2.10.0+cpu-cp312-cp312-win_amd64.whl", hash = "sha256:21cb5436978ef47c823b7a813ff0f8c2892e266cfe0f1d944879b5fba81bf4e1"}, + {file = "torch-2.10.0+cpu-cp312-cp312-win_arm64.whl", hash = "sha256:3eaa727e6a73affa61564d86b9d03191df45c8650d0666bd3d57c8597ef61e78"}, + {file = "torch-2.10.0+cpu-cp313-cp313-linux_aarch64.whl", hash = "sha256:fd215f3d0f681905c5b56b0630a3d666900a37fcc3ca5b937f95275c66f9fd9c"}, + {file = "torch-2.10.0+cpu-cp313-cp313-linux_s390x.whl", hash = "sha256:170a0623108055be5199370335cf9b41ba6875b3cb6f086db4aee583331a4899"}, + {file = "torch-2.10.0+cpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e51994492cdb76edce29da88de3672a3022f9ef0ffd90345436948d4992be2c7"}, + {file = "torch-2.10.0+cpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8d316e5bf121f1eab1147e49ad0511a9d92e4c45cc357d1ab0bee440da71a095"}, + {file = "torch-2.10.0+cpu-cp313-cp313-win_amd64.whl", hash = "sha256:b719da5af01b59126ac13eefd6ba3dd12d002dc0e8e79b8b365e55267a8189d3"}, + {file = "torch-2.10.0+cpu-cp313-cp313-win_arm64.whl", hash = "sha256:b67d91326e4ed9eccbd6b7d84ed7ffa43f93103aa3f0b24145f3001f3b11b714"}, + {file = "torch-2.10.0+cpu-cp313-cp313t-linux_aarch64.whl", hash = "sha256:5af75e5f49de21b0bdf7672bc27139bd285f9e8dbcabe2d617a2eb656514ac36"}, + {file = "torch-2.10.0+cpu-cp313-cp313t-linux_s390x.whl", hash = "sha256:ba51ef01a510baf8fff576174f702c47e1aa54389a9f1fba323bb1a5003ff0bf"}, + {file = "torch-2.10.0+cpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:0fedcb1a77e8f2aaf7bfd21591bf6d1e0b207473268c9be16b17cb7783253969"}, + {file = "torch-2.10.0+cpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:106dd1930cb30a4a337366ba3f9b25318ebf940f51fd46f789281dd9e736bdc4"}, + {file = "torch-2.10.0+cpu-cp313-cp313t-win_amd64.whl", hash = "sha256:eb1bde1ce198f05c8770017de27e001d404499cf552aaaa014569eff56ca25c0"}, + {file = "torch-2.10.0+cpu-cp314-cp314-linux_aarch64.whl", hash = "sha256:ea2bcc9d1fca66974a71d4bf9a502539283f35d61fcab5a799b4e120846f1e02"}, + {file = "torch-2.10.0+cpu-cp314-cp314-linux_s390x.whl", hash = "sha256:f8294fd2fc6dd8f4435a891a0122307a043b14b21f0dac1bca63c85bfb59e586"}, + {file = "torch-2.10.0+cpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a28fdbcfa2fbacffec81300f24dd1bed2b0ccfdbed107a823cff12bc1db070f6"}, + {file = "torch-2.10.0+cpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:aada8afc068add586464b2a55adb7cc9091eec55caf5320447204741cb6a0604"}, + {file = "torch-2.10.0+cpu-cp314-cp314-win_amd64.whl", hash = "sha256:2adc71fe471e98a608723bfc837f7e1929885ebb912c693597711e139c1cda41"}, + {file = "torch-2.10.0+cpu-cp314-cp314t-linux_aarch64.whl", hash = "sha256:9412bd37b70f5ebd1205242c4ba4cabae35a605947f2b30806d5c9b467936db9"}, + {file = "torch-2.10.0+cpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:e71c476517c33e7db69825a9ff46c7f47a723ec4dac5b2481cff4246d1c632be"}, + {file = "torch-2.10.0+cpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:23882f8d882460aca809882fc42f5e343bf07585274f929ced00177d1be1eb67"}, + {file = "torch-2.10.0+cpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4fcd8b4cc2ae20f2b7749fb275349c55432393868778c2d50a08e81d5ee5591e"}, + {file = "torch-2.10.0+cpu-cp314-cp314t-win_amd64.whl", hash = "sha256:ffc8da9a1341092d6a90cb5b1c1a33cd61abf0fb43f0cd88443c27fa372c26ae"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=0.8.5" +jinja2 = "*" +networkx = ">=2.5.1" +setuptools = {version = "*", markers = "python_version >= \"3.12\""} +sympy = ">=1.13.3" +typing-extensions = ">=4.10.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.13.0)"] +pyyaml = ["pyyaml"] + +[package.source] +type = "legacy" +url = "https://download.pytorch.org/whl/cpu" +reference = "pytorch-cpu" + [[package]] name = "torchvision" version = "0.22.1" @@ -3436,6 +3505,7 @@ description = "image and video datasets and models for torch deep learning" optional = false python-versions = ">=3.9" groups = ["main"] +markers = "sys_platform != \"linux\"" files = [ {file = "torchvision-0.22.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3b47d8369ee568c067795c0da0b4078f39a9dfea6f3bc1f3ac87530dfda1dd56"}, {file = "torchvision-0.22.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:990de4d657a41ed71680cd8be2e98ebcab55371f30993dc9bd2e676441f7180e"}, @@ -3472,6 +3542,52 @@ torch = "2.7.1" gdown = ["gdown (>=4.7.3)"] scipy = ["scipy"] +[[package]] +name = "torchvision" +version = "0.25.0+cpu" +description = "image and video datasets and models for torch deep learning" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "sys_platform == \"linux\"" +files = [ + {file = "torchvision-0.25.0+cpu-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:73ce04dea64914ff1110008311204c366107d651d3b04faa0dbee77efb7133b7"}, + {file = "torchvision-0.25.0+cpu-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7f851245a2687743742157988ed9c42c6e312b95bbe6cfcac9e7d0d0c28ae62f"}, + {file = "torchvision-0.25.0+cpu-cp310-cp310-win_amd64.whl", hash = "sha256:3e2ae9981e32a5b9db685659d5c7af0f04b159ff20394650a90124baf6ada51a"}, + {file = "torchvision-0.25.0+cpu-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:59be99d1c470ef470b134468aa6afa6f968081a503acb4ee883d70332f822e35"}, + {file = "torchvision-0.25.0+cpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:aa016ab73e06a886f72edc8929ed2ed4c85aaaa6e10500ecdef921b03129b19e"}, + {file = "torchvision-0.25.0+cpu-cp311-cp311-win_amd64.whl", hash = "sha256:c7eb5f219fdfaf1f65e68c00eb81172ab4fa08a9874dae9dad2bca360da34d0f"}, + {file = "torchvision-0.25.0+cpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:727334e9a721cfc1ac296ce0bf9e69d9486821bfa5b1e75a8feb6f78041db481"}, + {file = "torchvision-0.25.0+cpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c1be164e93c68b2dbf460fd58975377c892dbcf3358fb72941709c3857351bba"}, + {file = "torchvision-0.25.0+cpu-cp312-cp312-win_amd64.whl", hash = "sha256:2d444009c0956669ada149f61ed78f257c1cc96d259efa6acf3929ca96ceb3f0"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fe54cbd5942cd0b26a90f1748f0d4421caf67be35c281c6c3b8573733a03d630"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:90eec299e1f82cfaf080ccb789df3838cb9a54b57e2ebe33852cd392c692de5c"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313-win_amd64.whl", hash = "sha256:783c8fc580bbfc159bff52f4f72cdd538e42b32956e70dffa42b940db114e151"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e985e12a9a232618e5a43476de5689e4b14989f5da6b93909c57afa57ec27012"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:813f0106eb3e268f3783da67b882458e544c6fb72f946e6ca64b5ed4e62c6a77"}, + {file = "torchvision-0.25.0+cpu-cp313-cp313t-win_amd64.whl", hash = "sha256:9212210f417888e6261c040495180f053084812cf873dedba9fc51ff4b24b2d3"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0c2d0da9bc011a0fde1d125af396a8fbe94d99becf9d313764f24ca7657a3448"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4d72a57a8f0b5146e26dac1fbfa2c905280cd04f5fcb23b9c56253506b683aeb"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314-win_amd64.whl", hash = "sha256:499eae1e535766391b6ee2d1e6e841239c20e2e6d88203a15b8f9f8d60a1f8bd"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7d47d544899fabac52ebe0d4812975608fd7ab79a3d7fb6383275eb667e33f53"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9511339b3b5eb75229e0b5041202e8aed9bef3b1de3a715b9fb319c9e97688fd"}, + {file = "torchvision-0.25.0+cpu-cp314-cp314t-win_amd64.whl", hash = "sha256:fb9f07f6a10f0ac24ac482ae68c6df99110b74a0d80a4c64fddc9753267d8815"}, +] + +[package.dependencies] +numpy = "*" +pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" +torch = "2.10.0" + +[package.extras] +gdown = ["gdown (>=4.7.3)"] +scipy = ["scipy"] + +[package.source] +type = "legacy" +url = "https://download.pytorch.org/whl/cpu" +reference = "pytorch-cpu" + [[package]] name = "triton" version = "3.3.1" @@ -3479,7 +3595,7 @@ description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" groups = ["main"] -markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and sys_platform != \"linux\"" files = [ {file = "triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e"}, {file = "triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b"}, @@ -3949,4 +4065,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "654950c9f4e2b228a4a017066b6e081b9dd89976e8d5683d4e0aebfefb7a2a44" +content-hash = "b5a2aeabb4ad3107e68b24730782c6fe62a6c173ea7db761caad8f9063e65e71" diff --git a/pyproject.toml b/pyproject.toml index b182b99..71a21b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,26 @@ pillow = "^12.1.0" # can leave a corrupted cv2 binary after a parallel install. The Dockerfile # installs the system libraries opencv-python needs instead. opencv-python = "^4.12.0.88" +# CPU-only builds: Lambda has no GPU, and the default CUDA build pulls in +# several GB of nvidia-* libraries that make the cold start far too slow. +# The CPU index has no macOS wheels, so other platforms keep using PyPI +# (whose macOS builds are CPU/MPS only anyway). +torch = [ + { version = ">=2.7", source = "pytorch-cpu", markers = "sys_platform == 'linux'" }, + { version = ">=2.7", source = "pypi", markers = "sys_platform != 'linux'" }, +] # torchvision <= 0.12 declares an unpinned torch dependency, so without a # lower bound the resolver pairs a current torch with an incompatible # torchvision. Newer releases pin their matching torch version. -torchvision = ">=0.22.1" +torchvision = [ + { version = ">=0.22.1", source = "pytorch-cpu", markers = "sys_platform == 'linux'" }, + { version = ">=0.22.1", source = "pypi", markers = "sys_platform != 'linux'" }, +] + +[[tool.poetry.source]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +priority = "explicit" [tool.poetry.group.dev.dependencies] pytest = "^8.2.2" From f77cfbf35ba7e1901df1f0b4e68b0bd02ffeb0ec Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 30 Oct 2026 11:01:24 +0000 Subject: [PATCH 2/4] Defer cv2 and numpy imports to shorten Lambda cold start The worker still took 12.7 s to open its IPC socket on Lambda, past the 10 s init limit. cv2 and numpy were the largest remaining module-level imports and are only needed for drawing, so import them there and compute the timing averages without numpy. Also run compileall after copying the function code so it is precompiled too; Lambda's read-only filesystem can't cache bytecode. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 8 +++++--- evaluation_function/evaluation.py | 17 ++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11980b1..8c1527c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -26,12 +26,14 @@ ENV VIRTUAL_ENV=/app/.venv \ COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} -# Precompile python files for faster startup -RUN python -m compileall -q . - # Copy the evaluation function to the app directory COPY evaluation_function ./evaluation_function +# Precompile python files for faster startup. Runs after the COPY so the +# function's own code is included: Lambda's filesystem is read-only, so +# anything not compiled here is recompiled on every cold start. +RUN python -m compileall -q . + # Command to start the evaluation function with ENV FUNCTION_COMMAND="python" diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py index 00b8e1b..6d59f26 100755 --- a/evaluation_function/evaluation.py +++ b/evaluation_function/evaluation.py @@ -6,8 +6,6 @@ import io import requests import random -import numpy as np -import cv2 import time @@ -97,6 +95,11 @@ def get_class_color(class_name): return tuple(random.choices(range(50, 256), k=3)) def draw_annotations_cv2(img, detections): + # Imported here rather than at module level: cv2 and numpy load large + # shared libraries, which slows the cold start before the IPC server + # can open its socket. Only drawing needs them. + import cv2 + import numpy as np img_cv = np.array(img) @@ -252,10 +255,10 @@ def analyze_images(images, draw_images=True): {"annotated": annotated, "detections": detections, "load_error": None} ) - avg_load_time = np.mean(load_times) if load_times else 0.0 - avg_prediction_time = np.mean(prediction_times) if prediction_times else 0.0 - avg_process_time = np.mean(process_times) if process_times else 0.0 - avg_draw_time = np.mean(draw_times) if draw_times else 0.0 + avg_load_time = sum(load_times) / len(load_times) if load_times else 0.0 + avg_prediction_time = sum(prediction_times) / len(prediction_times) if prediction_times else 0.0 + avg_process_time = sum(process_times) / len(process_times) if process_times else 0.0 + avg_draw_time = sum(draw_times) / len(draw_times) if draw_times else 0.0 return ( per_image, @@ -313,7 +316,7 @@ def analyze_images(images, draw_images=True): append_feedback(f"Separator [{idx}]", " \n\n---\n\n") upload_times.append(0.0) - avg_upload_time = np.mean(upload_times) if upload_times else 0.0 + avg_upload_time = sum(upload_times) / len(upload_times) if upload_times else 0.0 feedback_time = time.time() - feedback_start From 609141ca496e9566400e575ecfcb22e8b434c73d Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 30 Oct 2026 12:32:11 +0000 Subject: [PATCH 3/4] Switch worker transport to stdio to fix Lambda cold start With the ipc transport, shimmy holds Lambda's 10 s init open until the worker has created its socket, which the Python imports still couldn't do on a cold container. Deferring the torch import only moved that cost into the first request, which then hit the 30 s FUNCTION_TIMEOUT. Use stdio like eduVisionAeroRC instead: init returns immediately and the request waits on stdin while Python imports. Restore the module-level ultralytics, cv2 and numpy imports and preload the default model so that work overlaps the cold start rather than the request. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 8 ++++-- evaluation_function/evaluation.py | 44 ++++++++++++++++--------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c1527c..da52649 100755 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,11 @@ ENV FUNCTION_COMMAND="python" # Args to start the evaluation function with ENV FUNCTION_ARGS="-m,evaluation_function.main" -# The transport to use for the RPC server -ENV FUNCTION_RPC_TRANSPORT="ipc" +# Talk to the worker over stdio rather than an IPC socket. With ipc, shimmy +# blocks Lambda's 10 s init until Python has opened the socket, which the +# torch import can't do on a cold start. With stdio, init returns at once and +# the request waits on stdin while Python finishes importing. +ENV FUNCTION_INTERFACE="rpc" +ENV FUNCTION_RPC_TRANSPORT="stdio" ENV LOG_LEVEL="debug" diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py index 6d59f26..de9e505 100755 --- a/evaluation_function/evaluation.py +++ b/evaluation_function/evaluation.py @@ -2,16 +2,37 @@ from typing import Any from lf_toolkit.evaluation import Result, Params from lf_toolkit.evaluation.image_upload import upload_image +from ultralytics import YOLO from PIL import Image, ImageOps import io import requests import random +import numpy as np +import cv2 import time # Cache for loaded models by name _model_cache = {} +DEFAULT_MODEL_NAME = "best.pt" + + +def _load_model(model_name): + if model_name not in _model_cache: + model_dir = os.path.dirname(os.path.abspath(__file__)) + # Strip any directory components so `model_name` can't be used to + # load a model file from outside the evaluation_function directory. + safe_model_name = os.path.basename(model_name) + model_path = os.path.join(model_dir, safe_model_name) + _model_cache[model_name] = YOLO(model_path) + return _model_cache[model_name] + + +# Load the default model at import time so it overlaps the Lambda cold start +# instead of counting against the first request's timeout. +_load_model(DEFAULT_MODEL_NAME) + def evaluation_function( response: Any, @@ -33,7 +54,7 @@ def evaluation_function( ) draw_images = params.get("draw_images", True) - model_name = params.get("model_name", "best.pt") + model_name = params.get("model_name", DEFAULT_MODEL_NAME) conf_threshold = params.get("conf_threshold", 0.5) # Optional array of class names to restrict detection to. Names are # matched against the model's own class names (model.names), @@ -42,24 +63,10 @@ def evaluation_function( model_load_start = time.time() - # Use a dict to cache models by name - if model_name not in _model_cache: - # Imported here rather than at module level: importing ultralytics - # pulls in torch, which takes longer than Lambda's 10 s init window - # on a cold start. Deferring it lets the IPC server start first. - from ultralytics import YOLO - - model_dir = os.path.dirname(os.path.abspath(__file__)) - # Strip any directory components so `model_name` can't be used to - # load a model file from outside the evaluation_function directory. - safe_model_name = os.path.basename(model_name) - model_path = os.path.join(model_dir, safe_model_name) - _model_cache[model_name] = YOLO(model_path) + model = _load_model(model_name) model_load_time = time.time() - model_load_start - model = _model_cache[model_name] - # Resolve `allowed_classes` names to the class indices YOLO expects. # Unmatched names are kept aside so they can be surfaced in debug # feedback instead of silently doing nothing. @@ -95,11 +102,6 @@ def get_class_color(class_name): return tuple(random.choices(range(50, 256), k=3)) def draw_annotations_cv2(img, detections): - # Imported here rather than at module level: cv2 and numpy load large - # shared libraries, which slows the cold start before the IPC server - # can open its socket. Only drawing needs them. - import cv2 - import numpy as np img_cv = np.array(img) From 5c42935846d1e3f5e1ad44bb1144ba48b00eb809 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Fri, 30 Oct 2026 12:47:40 +0000 Subject: [PATCH 4/4] Raise worker send timeout and log import times The first request after a cold start still hit shimmy's 30 s send timeout while Python was importing torch/ultralytics. Raise it to 90 s, and temporarily enable PYTHONPROFILEIMPORTTIME so CloudWatch shows which imports dominate the cold start. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index da52649..85fd4de 100755 --- a/Dockerfile +++ b/Dockerfile @@ -47,4 +47,12 @@ ENV FUNCTION_ARGS="-m,evaluation_function.main" ENV FUNCTION_INTERFACE="rpc" ENV FUNCTION_RPC_TRANSPORT="stdio" +# The first request after a cold start waits for torch/ultralytics to import +# and the model to load, which takes longer than shimmy's 30 s default. +ENV FUNCTION_WORKER_SEND_TIMEOUT="90s" + +# TEMPORARY: log per-module import times to stderr (shimmy forwards it to +# CloudWatch) to find what dominates the cold start. Remove once measured. +ENV PYTHONPROFILEIMPORTTIME=1 + ENV LOG_LEVEL="debug"