diff --git a/dk-installer.py b/dk-installer.py index 53c5782..e3596b3 100755 --- a/dk-installer.py +++ b/dk-installer.py @@ -71,6 +71,16 @@ # the time the process needs to record what stopped, or the scheduler is killed mid-wait and # a running job is cut instead of stopping at a checkpoint. TESTGEN_STOP_GRACE_PERIOD = 90 +# Container ceilings written into a new Docker install. A limit is protective rather than +# restrictive: the container that exhausts memory is killed on its own and the rest keeps +# running, where an unlimited container lets the host run out and the kernel pick any victim, +# including the database. Sized for the 4 CPU / 16 GB virtual machine the enterprise install +# guide asks for. They are ceilings rather than reservations, so they overlap deliberately and +# a smaller machine simply never reaches them. +TESTGEN_ENGINE_CPU_LIMIT = "3.0" +TESTGEN_ENGINE_MEMORY_LIMIT = "10G" +TESTGEN_POSTGRES_CPU_LIMIT = "3.0" +TESTGEN_POSTGRES_MEMORY_LIMIT = "8G" INSTALL_MARKER_FILE = "dk-{}-install.json" INSTALL_MODE_DOCKER = "docker" INSTALL_MODE_PIP = "pip" @@ -2396,6 +2406,11 @@ def get_compose_file_contents(self, action, args): condition: service_healthy networks: - datakitchen + deploy: + resources: + limits: + cpus: "{TESTGEN_ENGINE_CPU_LIMIT}" + memory: {TESTGEN_ENGINE_MEMORY_LIMIT} postgres: image: postgres:14.1-alpine @@ -2412,6 +2427,11 @@ def get_compose_file_contents(self, action, args): retries: 3 networks: - datakitchen + deploy: + resources: + limits: + cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}" + memory: {TESTGEN_POSTGRES_MEMORY_LIMIT} volumes: postgres_data: diff --git a/tests/test_tg_install.py b/tests/test_tg_install.py index d4d9b87..66045d4 100644 --- a/tests/test_tg_install.py +++ b/tests/test_tg_install.py @@ -11,6 +11,10 @@ AbortAction, TestGenCreateDockerComposeFileStep, ComposeVerifyExistingInstallStep, + TESTGEN_ENGINE_CPU_LIMIT, + TESTGEN_ENGINE_MEMORY_LIMIT, + TESTGEN_POSTGRES_CPU_LIMIT, + TESTGEN_POSTGRES_MEMORY_LIMIT, TESTGEN_STOP_GRACE_PERIOD, ) @@ -136,6 +140,22 @@ def test_tg_compose_sets_engine_stop_grace_period(tg_install_action, start_cmd_m assert lines[grace_idx] == f"{indent}stop_grace_period: {TESTGEN_STOP_GRACE_PERIOD}s" +@pytest.mark.integration +def test_tg_compose_sets_resource_limits(tg_install_action, start_cmd_mock, stdout_mock, compose_path): + """An unlimited container lets the host run out of memory and the kernel pick any victim. + A ceiling per service keeps a run that exhausts memory to the container it started in.""" + tg_install_action.execute() + compose_content = compose_path.read_text() + + assert f'cpus: "{TESTGEN_ENGINE_CPU_LIMIT}"' in compose_content + assert f"memory: {TESTGEN_ENGINE_MEMORY_LIMIT}" in compose_content + assert f'cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}"' in compose_content + assert f"memory: {TESTGEN_POSTGRES_MEMORY_LIMIT}" in compose_content + # Both services, and neither twice. + assert compose_content.count("deploy:") == 2 + assert compose_content.count("limits:") == 2 + + @pytest.mark.integration def test_tg_compose_base_url_ssl(tg_install_action, start_cmd_mock, stdout_mock, args_mock, compose_path): args_mock.ssl_cert_file = "/path/to/cert.crt" diff --git a/tests/test_tg_upgrade.py b/tests/test_tg_upgrade.py index 21d57d6..8dc6b34 100644 --- a/tests/test_tg_upgrade.py +++ b/tests/test_tg_upgrade.py @@ -276,6 +276,27 @@ def test_tg_upgrade_adds_stop_grace_period( @pytest.mark.integration +def test_tg_upgrade_does_not_add_resource_limits( + tg_upgrade_action, + compose_path, + start_cmd_mock, + tg_upgrade_stdout_side_effect, + args_mock, + version_check_mock, +): + """Limits are for new installs. Imposing a ceiling on a running install could start killing + work that fits the machine it was sized for.""" + args_mock.skip_verify = True + set_version_check_mock(version_check_mock, "1.1.0") + compose_path.write_text(get_compose_content("TG_INSTANCE_ID: test-instance-id")) + + tg_upgrade_action.execute(args_mock) + + compose_content = compose_path.read_text() + assert "deploy:" not in compose_content + assert "limits:" not in compose_content + + def test_tg_upgrade_preserves_existing_stop_grace_period( tg_upgrade_action, compose_path,