feat: add per-weblog metadata to control CI build requirement

.github/workflows/ci.yml

@@ -86,9 +86,7 @@ jobs:
       _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }}
       _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
-      _build_python_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') }}
-      _build_php_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-php-base-images') }}
-      _build_nodejs_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') }}
+      _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }}
       _enable_replay_scenarios: true
       _system_tests_dev_mode: ${{ matrix.version == 'dev' }}
       _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }}

.github/workflows/compute-workflow-parameters.yml

@@ -52,6 +52,11 @@ on:
         default: ''
         required: false
         type: string
+      _build_weblog_base_images:
+        description: "Shall we build weblog base images"
+        default: false
+        required: false
+        type: boolean
 
     # Map the workflow outputs to job outputs
     outputs:
@@ -137,6 +142,7 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
+          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --output $GITHUB_OUTPUT
     - name: log
       run: |
@@ -150,6 +156,7 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
+          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --format json | jq
 
     - name: Extract library target branch

.github/workflows/run-end-to-end.yml

@@ -62,6 +62,11 @@ on:
         default: false
         required: false
         type: boolean
+      _build_weblog_base_image:
+        description: "Shall we build the weblog base image (only valid if weblog build is local)"
+        default: false
+        required: false
+        type: boolean
       push_to_feature_parity_dashbaord:
         description: "Shall we push results to Feature Parity Dashbaord"
         default: false
@@ -145,6 +150,13 @@ jobs:
     - name: Build lambda-proxy image
       if: inputs._build_lambda_proxy_image
       run: ./build.sh -i lambda-proxy
+
+    - name: Build weblog base image
+      if: inputs._build_weblog_base_image
+      run: |
+        source venv/bin/activate
+        ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }}
+
     - name: Pull images
       uses: ./.github/actions/pull_images
       with:

.github/workflows/system-tests.yml

@@ -44,18 +44,8 @@ on:
         default: 'custom'
         required: false
         type: string
-      _build_python_base_images:
-        description: "Shall we build python base images for tests on python tracer"
-        default: false
-        required: false
-        type: boolean
-      _build_php_base_images:
-        description: "Shall we build PHP base images for tests on PHP tracer"
-        default: false
-        required: false
-        type: boolean
-      _build_nodejs_base_images:
-        description: "Shall we build Node.js base images for tests on Node.js tracer"
+      _build_weblog_base_images:
+        description: "Shall we rebuild weblog base images"
         default: false
         required: false
         type: boolean
@@ -169,6 +159,7 @@ jobs:
       binaries_artifact: ${{ inputs.binaries_artifact }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}
       _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }}
+      _build_weblog_base_images: ${{ inputs._build_weblog_base_images }}
 
   parametric:
     needs:
@@ -229,18 +220,11 @@ jobs:
         run: ls -la binaries/
       - name: Export github token to a file
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
-      - name: Build python weblog base images
-        if: inputs.library == 'python' && inputs._build_python_base_images
-        run: |
-          ./utils/build/build_python_base_images.sh
-      - name: Build php weblog base images
-        if: inputs.library == 'php' && inputs._build_php_base_images
-        run: |
-          ./utils/build/build_php_base_images.sh
-      - name: Build nodejs weblog base images
-        if: inputs.library == 'nodejs' && inputs._build_nodejs_base_images
+      - name: Build weblog base images
+        if: matrix.weblog.build_base_image
         run: |
-          ./utils/build/build_nodejs_base_images.sh
+          source venv/bin/activate
+          ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }}
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"
@@ -295,5 +279,6 @@ jobs:
       _build_buddies_images: ${{ inputs._build_buddies_images }}
       _build_proxy_image: ${{ inputs._build_proxy_image }}
       _build_lambda_proxy_image: ${{ inputs._build_lambda_proxy_image }}
+      _build_weblog_base_image: ${{ matrix.job.build_weblog_base_image }}
       _enable_replay_scenarios: ${{ inputs._enable_replay_scenarios }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}

docs/CI/github-actions.md

@@ -100,6 +100,5 @@ Those parameters are used only by system-tests own CI
 | `_build_buddies_images`               | Shall we build buddies images                                        | boolean | false    | false      |
 | `_build_proxy_image`                  | Shall we build proxy image                                           | boolean | false    | false      |
 | `_build_lambda_proxy_image`           | Shall we build the lambda-proxy image                                | boolean | false    | false      |
-| `_build_php_base_images`              | Shall we build php base images for tests on python tracer            | boolean | false    | false      |
-| `_build_python_base_images`           | Shall we build python base images for tests on python tracer         | boolean | false    | false      |
+| `_build_weblog_base_images`           | Shall we build weblog base images                                    | boolean | false    | false      |
 | `_enable_replay_scenarios`            | Enable replay scenarios, should only be used in system-tests CI      | boolean | false    | false      |

docs/understand/weblogs/weblog-metadata.md

@@ -0,0 +1,47 @@
+# Weblog metadata
+
+Each library folder under `utils/build/docker/<library>/` may contain a `weblog_metadata.yml` file
+that declares metadata for weblogs that cannot be inferred from the Dockerfile alone.
+
+## When to add an entry
+
+A weblog discovered via a `<name>.Dockerfile` file defaults to `build_mode: prebuild` and no
+`framework_versions`. Add an entry only when you need to override those defaults.
+
+## Format
+
+```yaml
+<weblog-name>:
+  build_mode: none | local | prebuild   # default: prebuild
+  framework_versions: ["1.0.0", ...]    # optional; omit for non-integration-framework weblogs
+```
+
+The `library` field is intentionally absent — it is inferred from the folder the file lives in.
+
+## `build_mode` values
+
+| Value | Meaning |
+|-------|---------|
+| `none` | No Docker build. The shared `binaries_artifact` is used as-is (integration frameworks, proxies). |
+| `local` | The weblog has a fully baked base image. The build step inside the test job is trivial (only `COPY` instructions). No dedicated CI build job. |
+| `prebuild` | Built ahead of time by a dedicated `build_end_to_end` CI job that uploads a per-weblog artifact; still built locally when the test job runs. |
+
+Both `local` and `prebuild` set `weblog_build_required = true`.
+
+## Integration-framework weblogs
+
+When `framework_versions` is set, a single entry fans out into one weblog per version at load time:
+
+```yaml
+openai-js:
+  build_mode: none
+  framework_versions: ["6.0.0", "7.0.0"]
+```
+
+produces `openai-js@6.0.0` and `openai-js@7.0.0`.
+
+## Loader
+
+`WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges:
+1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata).
+2. Explicit overrides from `weblog_metadata.yml`.

tests/test_the_test/test_ci_orchestrator.py

@@ -1,6 +1,10 @@
 from utils import scenarios
 
-from utils.scripts.ci_orchestrators.workflow_data import _is_supported, get_endtoend_definitions
+from utils.scripts.ci_orchestrators.workflow_data import (
+    _get_endtoend_weblogs,
+    _is_supported,
+    get_endtoend_definitions,
+)
 
 
 @scenarios.test_the_test
@@ -29,3 +33,84 @@ def test_ipv6_is_not_supported_for_uds_weblogs():
     assert not _is_supported("dotnet", "uds", "IPV6", "dev")
     assert not _is_supported("python", "uds-flask", "IPV6", "dev")
     assert _is_supported("python", "flask-poc", "IPV6", "dev")
+
+
+@scenarios.test_the_test
+def test_nodejs_weblogs_dont_require_prebuild():
+    scenario_map = {"endtoend": ["DEFAULT"]}
+    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "")
+    # Node.js weblogs use build_mode="local": no dedicated build_end_to_end job
+    # (parallel_weblogs lists only "prebuild" weblogs, so it is empty), but the
+    # run_end_to_end jobs still build the weblog in-line (weblog_build_required=True).
+    parallel_jobs = defs["endtoend_defs"]["parallel_jobs"]
+    assert defs["endtoend_defs"]["parallel_weblogs"] == []
+    assert len(parallel_jobs) > 0
+    assert all(job["weblog_build_required"] for job in parallel_jobs)
+
+
+@scenarios.test_the_test
+def test_weblog_build_mode_is_resolved_from_metadata():
+    # build_mode is the single source of build requirement for every weblog, declared
+    # in each library's build.yml:
+    #   - weblog not listed in build.yml → "prebuild" (dedicated build job + local build)
+    #   - listed with a build_mode       → as declared
+    build_modes = {w.name: w.build_mode for w in _get_endtoend_weblogs("python", [], "123", "dev", "shared")}
+
+    # Dockerfile weblog absent from build.yml defaults to prebuild
+    assert build_modes["flask-poc"] == "prebuild"
+    # Node.js weblogs opt into local-only builds via build.yml
+    nodejs_modes = {w.name: w.build_mode for w in _get_endtoend_weblogs("nodejs", [], "123", "dev", "shared")}
+    assert nodejs_modes["express4"] == "local"
+    # integration-framework weblogs fan out per version and need no build
+    assert build_modes["openai-py@2.0.0"] == "none"
+
+
+@scenarios.test_the_test
+def test_nodejs_build_base_image():
+    scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
+    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
+
+    assert defs["endtoend_defs"]["parallel_weblogs"] == []
+
+    jobs = {job["weblog"]: job for job in defs["endtoend_defs"]["parallel_jobs"]}
+
+    # express4 is build_mode=local and has a base Dockerfile → should build base image
+    assert jobs["express4"]["build_weblog_base_image"] is True
+
+    # openai-js is build_mode=none and has no base Dockerfile → should not build base image
+    assert jobs["openai-js@6.0.0"]["build_weblog_base_image"] is False
+
+
+@scenarios.test_the_test
+def test_python_build_base_image():
+    scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
+    defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
+
+    # all python weblog has build_mode=prebuild. build_weblog_base_image
+    # only applies to build_mode=local weblogs → should not build base image inline
+    for job in defs["endtoend_defs"]["parallel_jobs"]:
+        assert job["build_weblog_base_image"] is False, job
+
+    # all python weblog with build_mode=prebuild should rebuild base images in the build job
+    for job in defs["endtoend_defs"]["parallel_weblogs"]:
+        assert job["build_base_images"] is True, job
+
+
+@scenarios.test_the_test
+def test_otel_collector():
+    scenario_map = {"endtoend": ["OTEL_COLLECTOR"]}
+    defs = get_endtoend_definitions("otel_collector", scenario_map, [], "prod", 200000, 256, "123", "")
+
+    assert defs["endtoend_defs"]["parallel_jobs"] == [
+        {
+            "binaries_artifact": "",
+            "build_weblog_base_image": False,
+            "expected_job_time": 74.34217318962216,
+            "library": "otel_collector",
+            "runs_on": "ubuntu-latest",
+            "scenarios": ["OTEL_COLLECTOR"],
+            "weblog": "otel_collector",
+            "weblog_build_required": False,
+            "weblog_instance": 1,
+        }
+    ]

utils/_context/weblog_metadata.py

@@ -0,0 +1,107 @@
+from enum import StrEnum
+from dataclasses import dataclass, replace
+from pathlib import Path
+import yaml
+
+
+class BuildMode(StrEnum):
+    none = "none"
+    """ The weblog does not require any build step"""
+
+    local = "local"
+    """ The weblog has a fully baked base image, so the build step is extra light,
+    and does not requires a full job in the CI"""
+
+    prebuild = "prebuild"
+    """ The weblog will be built in a dedicated job in the CI """
+
+
+@dataclass
+class WeblogMetaData:
+    name: str
+    library: str
+    build_mode: BuildMode = BuildMode.prebuild
+    framework_versions: list[str] | None = None
+    artifact_name: str = ""
+    """ not declared in the yml file, but populated later """
+
+    def __post_init__(self):
+        self.build_mode = BuildMode(self.build_mode)
+
+    @property
+    def require_build(self) -> bool:
+        """The run_end_to_end job builds the weblog locally (weblog_build_required)."""
+        return self.build_mode != BuildMode.none
+
+    @property
+    def base_dockerfile(self) -> Path | None:
+        """Returns the path of the base image docker file if exists, else None"""
+        image_name = self.base_image_tag
+
+        if image_name is None:
+            return None
+
+        file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0]
+        assert file_prefix.endswith(".base")
+
+        path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile")
+        return path if path.exists() else None
+
+    @property
+    def base_image_tag(self) -> str | None:
+        """system-tests base image tag read from the first FROM in the weblog Dockerfile."""
+        dockerfile = Path(f"utils/build/docker/{self.library}/{self.name}.Dockerfile")
+        if not dockerfile.exists():
+            return None
+        for line in dockerfile.read_text().splitlines():
+            if line.startswith("FROM "):
+                image_name = line.split()[1]
+                return image_name if image_name.startswith("datadog/system-tests:") else None
+        return None
+
+    @staticmethod
+    def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]:
+        path = Path(f"utils/build/docker/{library}/weblog_metadata.yml")
+        if not path.exists():
+            return {}
+
+        with path.open() as f:
+            data: dict = yaml.safe_load(f) or {}
+
+        return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()}
+
+    @staticmethod
+    def load(library: str) -> list["WeblogMetaData"]:
+        metadata = WeblogMetaData._load_explicit_metadata(library)
+        result: list[WeblogMetaData] = []
+
+        folder = Path(f"utils/build/docker/{library}")
+        if folder.exists():  # some lib does not have any weblog
+            names = [
+                f.name.replace(".Dockerfile", "")
+                for f in folder.iterdir()
+                if f.suffix == ".Dockerfile" and ".base." not in f.name and f.is_file()
+            ]
+        else:
+            names = []
+
+        for name in set(names + [w.name for w in metadata.values() if w.library == library]):
+            item = WeblogMetaData(name=name, library=library) if name not in metadata else metadata[name]
+
+            # integration-framework weblogs fan out into one weblog per version;
+            # all other weblogs map to a single weblog.
+            if item.framework_versions:
+                for version in item.framework_versions:
+                    sub_item = replace(item, name=f"{name}@{version}")
+                    result.append(sub_item)
+            else:
+                result.append(item)
+
+        return result
+
+
+if __name__ == "__main__":
+    x = WeblogMetaData.load("python")
+    from pprint import pprint
+
+    pprint(x)  # noqa: T203

utils/build/docker/envoy/weblog_metadata.yml

@@ -0,0 +1,4 @@
+# docs/understand/weblogs/weblog-metadata.md
+
+envoy:
+  build_mode: none

utils/build/docker/haproxy/weblog_metadata.yml

@@ -0,0 +1,4 @@
+# docs/understand/weblogs/weblog-metadata.md
+
+haproxy-spoa:
+  build_mode: none