From 962f5c1a27208c30da9d71320e6218ff37569903 Mon Sep 17 00:00:00 2001 From: Aarthy Adityan Date: Tue, 15 Sep 2026 19:01:42 -0400 Subject: [PATCH] feat(installer): set container resource limits on a new TestGen install Both services ran with no ceiling at all. That reads as generous and is the opposite: with no limit there is nothing for the container to hit, so the host runs out of memory instead and the kernel picks a victim by size -- which can be the database, or anything else sharing the machine. With a limit, a run that exhausts memory is confined to the container it started in and the rest keeps serving. Sized for the 4 CPU / 16 GB virtual machine the enterprise install guide asks for, matching the example compose files that ship with TestGen. They are ceilings rather than reservations, so the two overlap deliberately and a smaller evaluation machine simply never reaches them. New installs only. The upgrade path patches an existing compose file key by key and is deliberately left alone: imposing a ceiling on a running install could start killing work that fits the machine it was sized for, and that decision belongs to whoever sized it. A test pins that, so a later backfill has to be deliberate. Note this is new installs by compose file, not by command -- `tg install` re-uses a compose file it finds on disk, so an install that follows a `delete --keep-config` keeps the old one, limits included. --- dk-installer.py | 20 ++++++++++++++++++++ tests/test_tg_install.py | 20 ++++++++++++++++++++ tests/test_tg_upgrade.py | 21 +++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/dk-installer.py b/dk-installer.py index 53c5782..e3596b3 100755 --- a/dk-installer.py +++ b/dk-installer.py @@ -71,6 +71,16 @@ # the time the process needs to record what stopped, or the scheduler is killed mid-wait and # a running job is cut instead of stopping at a checkpoint. TESTGEN_STOP_GRACE_PERIOD = 90 +# Container ceilings written into a new Docker install. A limit is protective rather than +# restrictive: the container that exhausts memory is killed on its own and the rest keeps +# running, where an unlimited container lets the host run out and the kernel pick any victim, +# including the database. Sized for the 4 CPU / 16 GB virtual machine the enterprise install +# guide asks for. They are ceilings rather than reservations, so they overlap deliberately and +# a smaller machine simply never reaches them. +TESTGEN_ENGINE_CPU_LIMIT = "3.0" +TESTGEN_ENGINE_MEMORY_LIMIT = "10G" +TESTGEN_POSTGRES_CPU_LIMIT = "3.0" +TESTGEN_POSTGRES_MEMORY_LIMIT = "8G" INSTALL_MARKER_FILE = "dk-{}-install.json" INSTALL_MODE_DOCKER = "docker" INSTALL_MODE_PIP = "pip" @@ -2396,6 +2406,11 @@ def get_compose_file_contents(self, action, args): condition: service_healthy networks: - datakitchen + deploy: + resources: + limits: + cpus: "{TESTGEN_ENGINE_CPU_LIMIT}" + memory: {TESTGEN_ENGINE_MEMORY_LIMIT} postgres: image: postgres:14.1-alpine @@ -2412,6 +2427,11 @@ def get_compose_file_contents(self, action, args): retries: 3 networks: - datakitchen + deploy: + resources: + limits: + cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}" + memory: {TESTGEN_POSTGRES_MEMORY_LIMIT} volumes: postgres_data: diff --git a/tests/test_tg_install.py b/tests/test_tg_install.py index d4d9b87..66045d4 100644 --- a/tests/test_tg_install.py +++ b/tests/test_tg_install.py @@ -11,6 +11,10 @@ AbortAction, TestGenCreateDockerComposeFileStep, ComposeVerifyExistingInstallStep, + TESTGEN_ENGINE_CPU_LIMIT, + TESTGEN_ENGINE_MEMORY_LIMIT, + TESTGEN_POSTGRES_CPU_LIMIT, + TESTGEN_POSTGRES_MEMORY_LIMIT, TESTGEN_STOP_GRACE_PERIOD, ) @@ -136,6 +140,22 @@ def test_tg_compose_sets_engine_stop_grace_period(tg_install_action, start_cmd_m assert lines[grace_idx] == f"{indent}stop_grace_period: {TESTGEN_STOP_GRACE_PERIOD}s" +@pytest.mark.integration +def test_tg_compose_sets_resource_limits(tg_install_action, start_cmd_mock, stdout_mock, compose_path): + """An unlimited container lets the host run out of memory and the kernel pick any victim. + A ceiling per service keeps a run that exhausts memory to the container it started in.""" + tg_install_action.execute() + compose_content = compose_path.read_text() + + assert f'cpus: "{TESTGEN_ENGINE_CPU_LIMIT}"' in compose_content + assert f"memory: {TESTGEN_ENGINE_MEMORY_LIMIT}" in compose_content + assert f'cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}"' in compose_content + assert f"memory: {TESTGEN_POSTGRES_MEMORY_LIMIT}" in compose_content + # Both services, and neither twice. + assert compose_content.count("deploy:") == 2 + assert compose_content.count("limits:") == 2 + + @pytest.mark.integration def test_tg_compose_base_url_ssl(tg_install_action, start_cmd_mock, stdout_mock, args_mock, compose_path): args_mock.ssl_cert_file = "/path/to/cert.crt" diff --git a/tests/test_tg_upgrade.py b/tests/test_tg_upgrade.py index 21d57d6..8dc6b34 100644 --- a/tests/test_tg_upgrade.py +++ b/tests/test_tg_upgrade.py @@ -276,6 +276,27 @@ def test_tg_upgrade_adds_stop_grace_period( @pytest.mark.integration +def test_tg_upgrade_does_not_add_resource_limits( + tg_upgrade_action, + compose_path, + start_cmd_mock, + tg_upgrade_stdout_side_effect, + args_mock, + version_check_mock, +): + """Limits are for new installs. Imposing a ceiling on a running install could start killing + work that fits the machine it was sized for.""" + args_mock.skip_verify = True + set_version_check_mock(version_check_mock, "1.1.0") + compose_path.write_text(get_compose_content("TG_INSTANCE_ID: test-instance-id")) + + tg_upgrade_action.execute(args_mock) + + compose_content = compose_path.read_text() + assert "deploy:" not in compose_content + assert "limits:" not in compose_content + + def test_tg_upgrade_preserves_existing_stop_grace_period( tg_upgrade_action, compose_path,