feat(installer): set container resource limits on a new TestGen install

dk-installer.py

@@ -71,6 +71,16 @@
 # the time the process needs to record what stopped, or the scheduler is killed mid-wait and
 # a running job is cut instead of stopping at a checkpoint.
 TESTGEN_STOP_GRACE_PERIOD = 90
+# Container ceilings written into a new Docker install. A limit is protective rather than
+# restrictive: the container that exhausts memory is killed on its own and the rest keeps
+# running, where an unlimited container lets the host run out and the kernel pick any victim,
+# including the database. Sized for the 4 CPU / 16 GB virtual machine the enterprise install
+# guide asks for. They are ceilings rather than reservations, so they overlap deliberately and
+# a smaller machine simply never reaches them.
+TESTGEN_ENGINE_CPU_LIMIT = "3.0"
+TESTGEN_ENGINE_MEMORY_LIMIT = "10G"
+TESTGEN_POSTGRES_CPU_LIMIT = "3.0"
+TESTGEN_POSTGRES_MEMORY_LIMIT = "8G"
 INSTALL_MARKER_FILE = "dk-{}-install.json"
 INSTALL_MODE_DOCKER = "docker"
 INSTALL_MODE_PIP = "pip"
@@ -2396,6 +2406,11 @@ def get_compose_file_contents(self, action, args):
                     condition: service_healthy
                 networks:
                   - datakitchen
+                deploy:
+                  resources:
+                    limits:
+                      cpus: "{TESTGEN_ENGINE_CPU_LIMIT}"
+                      memory: {TESTGEN_ENGINE_MEMORY_LIMIT}
 
               postgres:
                 image: postgres:14.1-alpine
@@ -2412,6 +2427,11 @@ def get_compose_file_contents(self, action, args):
                   retries: 3
                 networks:
                   - datakitchen
+                deploy:
+                  resources:
+                    limits:
+                      cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}"
+                      memory: {TESTGEN_POSTGRES_MEMORY_LIMIT}
 
             volumes:
               postgres_data:

tests/test_tg_install.py

@@ -11,6 +11,10 @@
     AbortAction,
     TestGenCreateDockerComposeFileStep,
     ComposeVerifyExistingInstallStep,
+    TESTGEN_ENGINE_CPU_LIMIT,
+    TESTGEN_ENGINE_MEMORY_LIMIT,
+    TESTGEN_POSTGRES_CPU_LIMIT,
+    TESTGEN_POSTGRES_MEMORY_LIMIT,
     TESTGEN_STOP_GRACE_PERIOD,
 )
 
@@ -136,6 +140,22 @@ def test_tg_compose_sets_engine_stop_grace_period(tg_install_action, start_cmd_m
     assert lines[grace_idx] == f"{indent}stop_grace_period: {TESTGEN_STOP_GRACE_PERIOD}s"
 
 
+@pytest.mark.integration
+def test_tg_compose_sets_resource_limits(tg_install_action, start_cmd_mock, stdout_mock, compose_path):
+    """An unlimited container lets the host run out of memory and the kernel pick any victim.
+    A ceiling per service keeps a run that exhausts memory to the container it started in."""
+    tg_install_action.execute()
+    compose_content = compose_path.read_text()
+
+    assert f'cpus: "{TESTGEN_ENGINE_CPU_LIMIT}"' in compose_content
+    assert f"memory: {TESTGEN_ENGINE_MEMORY_LIMIT}" in compose_content
+    assert f'cpus: "{TESTGEN_POSTGRES_CPU_LIMIT}"' in compose_content
+    assert f"memory: {TESTGEN_POSTGRES_MEMORY_LIMIT}" in compose_content
+    # Both services, and neither twice.
+    assert compose_content.count("deploy:") == 2
+    assert compose_content.count("limits:") == 2
+
+
 @pytest.mark.integration
 def test_tg_compose_base_url_ssl(tg_install_action, start_cmd_mock, stdout_mock, args_mock, compose_path):
     args_mock.ssl_cert_file = "/path/to/cert.crt"

tests/test_tg_upgrade.py

@@ -276,6 +276,27 @@ def test_tg_upgrade_adds_stop_grace_period(
 
 
 @pytest.mark.integration
+def test_tg_upgrade_does_not_add_resource_limits(
+    tg_upgrade_action,
+    compose_path,
+    start_cmd_mock,
+    tg_upgrade_stdout_side_effect,
+    args_mock,
+    version_check_mock,
+):
+    """Limits are for new installs. Imposing a ceiling on a running install could start killing
+    work that fits the machine it was sized for."""
+    args_mock.skip_verify = True
+    set_version_check_mock(version_check_mock, "1.1.0")
+    compose_path.write_text(get_compose_content("TG_INSTANCE_ID: test-instance-id"))
+
+    tg_upgrade_action.execute(args_mock)
+
+    compose_content = compose_path.read_text()
+    assert "deploy:" not in compose_content
+    assert "limits:" not in compose_content
+
+
 def test_tg_upgrade_preserves_existing_stop_grace_period(
     tg_upgrade_action,
     compose_path,