diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fc2414648b..5e50b1d1255 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,9 +86,7 @@ jobs: _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }} _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }} _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }} - _build_python_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') }} - _build_php_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-php-base-images') }} - _build_nodejs_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') }} + _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }} _enable_replay_scenarios: true _system_tests_dev_mode: ${{ matrix.version == 'dev' }} _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }} diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index 0405493a000..f5ac934999a 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -52,6 +52,11 @@ on: default: '' required: false type: string + _build_weblog_base_images: + description: "Shall we build weblog base images" + default: false + required: false + type: boolean # Map the workflow outputs to job outputs outputs: @@ -137,6 +142,7 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ + --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --output $GITHUB_OUTPUT - name: log run: | @@ -150,6 +156,7 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ + --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --format json | jq - name: Extract library target branch diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 147ff7a8f60..7b0773cc4e8 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -62,6 +62,11 @@ on: default: false required: false type: boolean + _build_weblog_base_image: + description: "Shall we build the weblog base image (only valid if weblog build is local)" + default: false + required: false + type: boolean push_to_feature_parity_dashbaord: description: "Shall we push results to Feature Parity Dashbaord" default: false @@ -145,6 +150,13 @@ jobs: - name: Build lambda-proxy image if: inputs._build_lambda_proxy_image run: ./build.sh -i lambda-proxy + + - name: Build weblog base image + if: inputs._build_weblog_base_image + run: | + source venv/bin/activate + ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} + - name: Pull images uses: ./.github/actions/pull_images with: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 0ec272f7625..d855fb46731 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -44,18 +44,8 @@ on: default: 'custom' required: false type: string - _build_python_base_images: - description: "Shall we build python base images for tests on python tracer" - default: false - required: false - type: boolean - _build_php_base_images: - description: "Shall we build PHP base images for tests on PHP tracer" - default: false - required: false - type: boolean - _build_nodejs_base_images: - description: "Shall we build Node.js base images for tests on Node.js tracer" + _build_weblog_base_images: + description: "Shall we rebuild weblog base images" default: false required: false type: boolean @@ -169,6 +159,7 @@ jobs: binaries_artifact: ${{ inputs.binaries_artifact }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }} + _build_weblog_base_images: ${{ inputs._build_weblog_base_images }} parametric: needs: @@ -229,18 +220,11 @@ jobs: run: ls -la binaries/ - name: Export github token to a file run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt" - - name: Build python weblog base images - if: inputs.library == 'python' && inputs._build_python_base_images - run: | - ./utils/build/build_python_base_images.sh - - name: Build php weblog base images - if: inputs.library == 'php' && inputs._build_php_base_images - run: | - ./utils/build/build_php_base_images.sh - - name: Build nodejs weblog base images - if: inputs.library == 'nodejs' && inputs._build_nodejs_base_images + - name: Build weblog base images + if: matrix.weblog.build_base_image run: | - ./utils/build/build_nodejs_base_images.sh + source venv/bin/activate + ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} - name: Build weblog id: build run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt" @@ -295,5 +279,6 @@ jobs: _build_buddies_images: ${{ inputs._build_buddies_images }} _build_proxy_image: ${{ inputs._build_proxy_image }} _build_lambda_proxy_image: ${{ inputs._build_lambda_proxy_image }} + _build_weblog_base_image: ${{ matrix.job.build_weblog_base_image }} _enable_replay_scenarios: ${{ inputs._enable_replay_scenarios }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} diff --git a/docs/CI/github-actions.md b/docs/CI/github-actions.md index 4120e3f8c3e..75a2fc61bce 100644 --- a/docs/CI/github-actions.md +++ b/docs/CI/github-actions.md @@ -100,6 +100,5 @@ Those parameters are used only by system-tests own CI | `_build_buddies_images` | Shall we build buddies images | boolean | false | false | | `_build_proxy_image` | Shall we build proxy image | boolean | false | false | | `_build_lambda_proxy_image` | Shall we build the lambda-proxy image | boolean | false | false | -| `_build_php_base_images` | Shall we build php base images for tests on python tracer | boolean | false | false | -| `_build_python_base_images` | Shall we build python base images for tests on python tracer | boolean | false | false | +| `_build_weblog_base_images` | Shall we build weblog base images | boolean | false | false | | `_enable_replay_scenarios` | Enable replay scenarios, should only be used in system-tests CI | boolean | false | false | diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md new file mode 100644 index 00000000000..2730d9e6875 --- /dev/null +++ b/docs/understand/weblogs/weblog-metadata.md @@ -0,0 +1,47 @@ +# Weblog metadata + +Each library folder under `utils/build/docker//` may contain a `weblog_metadata.yml` file +that declares metadata for weblogs that cannot be inferred from the Dockerfile alone. + +## When to add an entry + +A weblog discovered via a `.Dockerfile` file defaults to `build_mode: prebuild` and no +`framework_versions`. Add an entry only when you need to override those defaults. + +## Format + +```yaml +: + build_mode: none | local | prebuild # default: prebuild + framework_versions: ["1.0.0", ...] # optional; omit for non-integration-framework weblogs +``` + +The `library` field is intentionally absent — it is inferred from the folder the file lives in. + +## `build_mode` values + +| Value | Meaning | +|-------|---------| +| `none` | No Docker build. The shared `binaries_artifact` is used as-is (integration frameworks, proxies). | +| `local` | The weblog has a fully baked base image. The build step inside the test job is trivial (only `COPY` instructions). No dedicated CI build job. | +| `prebuild` | Built ahead of time by a dedicated `build_end_to_end` CI job that uploads a per-weblog artifact; still built locally when the test job runs. | + +Both `local` and `prebuild` set `weblog_build_required = true`. + +## Integration-framework weblogs + +When `framework_versions` is set, a single entry fans out into one weblog per version at load time: + +```yaml +openai-js: + build_mode: none + framework_versions: ["6.0.0", "7.0.0"] +``` + +produces `openai-js@6.0.0` and `openai-js@7.0.0`. + +## Loader + +`WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges: +1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata). +2. Explicit overrides from `weblog_metadata.yml`. diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 8cb598c5ca9..710c7cdb94a 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -1,6 +1,10 @@ from utils import scenarios -from utils.scripts.ci_orchestrators.workflow_data import _is_supported, get_endtoend_definitions +from utils.scripts.ci_orchestrators.workflow_data import ( + _get_endtoend_weblogs, + _is_supported, + get_endtoend_definitions, +) @scenarios.test_the_test @@ -29,3 +33,84 @@ def test_ipv6_is_not_supported_for_uds_weblogs(): assert not _is_supported("dotnet", "uds", "IPV6", "dev") assert not _is_supported("python", "uds-flask", "IPV6", "dev") assert _is_supported("python", "flask-poc", "IPV6", "dev") + + +@scenarios.test_the_test +def test_nodejs_weblogs_dont_require_prebuild(): + scenario_map = {"endtoend": ["DEFAULT"]} + defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "") + # Node.js weblogs use build_mode="local": no dedicated build_end_to_end job + # (parallel_weblogs lists only "prebuild" weblogs, so it is empty), but the + # run_end_to_end jobs still build the weblog in-line (weblog_build_required=True). + parallel_jobs = defs["endtoend_defs"]["parallel_jobs"] + assert defs["endtoend_defs"]["parallel_weblogs"] == [] + assert len(parallel_jobs) > 0 + assert all(job["weblog_build_required"] for job in parallel_jobs) + + +@scenarios.test_the_test +def test_weblog_build_mode_is_resolved_from_metadata(): + # build_mode is the single source of build requirement for every weblog, declared + # in each library's build.yml: + # - weblog not listed in build.yml → "prebuild" (dedicated build job + local build) + # - listed with a build_mode → as declared + build_modes = {w.name: w.build_mode for w in _get_endtoend_weblogs("python", [], "123", "dev", "shared")} + + # Dockerfile weblog absent from build.yml defaults to prebuild + assert build_modes["flask-poc"] == "prebuild" + # Node.js weblogs opt into local-only builds via build.yml + nodejs_modes = {w.name: w.build_mode for w in _get_endtoend_weblogs("nodejs", [], "123", "dev", "shared")} + assert nodejs_modes["express4"] == "local" + # integration-framework weblogs fan out per version and need no build + assert build_modes["openai-py@2.0.0"] == "none" + + +@scenarios.test_the_test +def test_nodejs_build_base_image(): + scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]} + defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True) + + assert defs["endtoend_defs"]["parallel_weblogs"] == [] + + jobs = {job["weblog"]: job for job in defs["endtoend_defs"]["parallel_jobs"]} + + # express4 is build_mode=local and has a base Dockerfile → should build base image + assert jobs["express4"]["build_weblog_base_image"] is True + + # openai-js is build_mode=none and has no base Dockerfile → should not build base image + assert jobs["openai-js@6.0.0"]["build_weblog_base_image"] is False + + +@scenarios.test_the_test +def test_python_build_base_image(): + scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]} + defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True) + + # all python weblog has build_mode=prebuild. build_weblog_base_image + # only applies to build_mode=local weblogs → should not build base image inline + for job in defs["endtoend_defs"]["parallel_jobs"]: + assert job["build_weblog_base_image"] is False, job + + # all python weblog with build_mode=prebuild should rebuild base images in the build job + for job in defs["endtoend_defs"]["parallel_weblogs"]: + assert job["build_base_images"] is True, job + + +@scenarios.test_the_test +def test_otel_collector(): + scenario_map = {"endtoend": ["OTEL_COLLECTOR"]} + defs = get_endtoend_definitions("otel_collector", scenario_map, [], "prod", 200000, 256, "123", "") + + assert defs["endtoend_defs"]["parallel_jobs"] == [ + { + "binaries_artifact": "", + "build_weblog_base_image": False, + "expected_job_time": 74.34217318962216, + "library": "otel_collector", + "runs_on": "ubuntu-latest", + "scenarios": ["OTEL_COLLECTOR"], + "weblog": "otel_collector", + "weblog_build_required": False, + "weblog_instance": 1, + } + ] diff --git a/utils/_context/weblog_metadata.py b/utils/_context/weblog_metadata.py new file mode 100644 index 00000000000..7d72d1a2bfb --- /dev/null +++ b/utils/_context/weblog_metadata.py @@ -0,0 +1,107 @@ +from enum import StrEnum +from dataclasses import dataclass, replace +from pathlib import Path +import yaml + + +class BuildMode(StrEnum): + none = "none" + """ The weblog does not require any build step""" + + local = "local" + """ The weblog has a fully baked base image, so the build step is extra light, + and does not requires a full job in the CI""" + + prebuild = "prebuild" + """ The weblog will be built in a dedicated job in the CI """ + + +@dataclass +class WeblogMetaData: + name: str + library: str + build_mode: BuildMode = BuildMode.prebuild + framework_versions: list[str] | None = None + artifact_name: str = "" + """ not declared in the yml file, but populated later """ + + def __post_init__(self): + self.build_mode = BuildMode(self.build_mode) + + @property + def require_build(self) -> bool: + """The run_end_to_end job builds the weblog locally (weblog_build_required).""" + return self.build_mode != BuildMode.none + + @property + def base_dockerfile(self) -> Path | None: + """Returns the path of the base image docker file if exists, else None""" + image_name = self.base_image_tag + + if image_name is None: + return None + + file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0] + assert file_prefix.endswith(".base") + + path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile") + return path if path.exists() else None + + @property + def base_image_tag(self) -> str | None: + """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" + dockerfile = Path(f"utils/build/docker/{self.library}/{self.name}.Dockerfile") + if not dockerfile.exists(): + return None + for line in dockerfile.read_text().splitlines(): + if line.startswith("FROM "): + image_name = line.split()[1] + return image_name if image_name.startswith("datadog/system-tests:") else None + return None + + @staticmethod + def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]: + path = Path(f"utils/build/docker/{library}/weblog_metadata.yml") + if not path.exists(): + return {} + + with path.open() as f: + data: dict = yaml.safe_load(f) or {} + + return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()} + + @staticmethod + def load(library: str) -> list["WeblogMetaData"]: + metadata = WeblogMetaData._load_explicit_metadata(library) + result: list[WeblogMetaData] = [] + + folder = Path(f"utils/build/docker/{library}") + if folder.exists(): # some lib does not have any weblog + names = [ + f.name.replace(".Dockerfile", "") + for f in folder.iterdir() + if f.suffix == ".Dockerfile" and ".base." not in f.name and f.is_file() + ] + else: + names = [] + + for name in set(names + [w.name for w in metadata.values() if w.library == library]): + item = WeblogMetaData(name=name, library=library) if name not in metadata else metadata[name] + + # integration-framework weblogs fan out into one weblog per version; + # all other weblogs map to a single weblog. + if item.framework_versions: + for version in item.framework_versions: + sub_item = replace(item, name=f"{name}@{version}") + result.append(sub_item) + else: + result.append(item) + + return result + + +if __name__ == "__main__": + x = WeblogMetaData.load("python") + from pprint import pprint + + pprint(x) # noqa: T203 diff --git a/utils/build/docker/envoy/weblog_metadata.yml b/utils/build/docker/envoy/weblog_metadata.yml new file mode 100644 index 00000000000..68aa735ef29 --- /dev/null +++ b/utils/build/docker/envoy/weblog_metadata.yml @@ -0,0 +1,4 @@ +# docs/understand/weblogs/weblog-metadata.md + +envoy: + build_mode: none diff --git a/utils/build/docker/haproxy/weblog_metadata.yml b/utils/build/docker/haproxy/weblog_metadata.yml new file mode 100644 index 00000000000..dceb7b65911 --- /dev/null +++ b/utils/build/docker/haproxy/weblog_metadata.yml @@ -0,0 +1,4 @@ +# docs/understand/weblogs/weblog-metadata.md + +haproxy-spoa: + build_mode: none diff --git a/utils/build/docker/java/weblog_metadata.yml b/utils/build/docker/java/weblog_metadata.yml new file mode 100644 index 00000000000..62cbf1d96ed --- /dev/null +++ b/utils/build/docker/java/weblog_metadata.yml @@ -0,0 +1,5 @@ +# docs/understand/weblogs/weblog-metadata.md + +openai-java: + build_mode: none + framework_versions: ["4.29.0"] diff --git a/utils/build/docker/nodejs/fastify/package-lock.json b/utils/build/docker/nodejs/fastify/package-lock.json index a225675180a..d2eb7c55ca3 100644 --- a/utils/build/docker/nodejs/fastify/package-lock.json +++ b/utils/build/docker/nodejs/fastify/package-lock.json @@ -33,6 +33,7 @@ "xml2js": "^0.6.2" }, "devDependencies": { + "@vercel/nft": "^1.10.2", "eslint": "^8.53.0", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.29.0", @@ -906,6 +907,427 @@ "dev": true, "license": "ISC" }, + "node_modules/@vercel/nft": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@vercel/nft/-/nft-1.10.2.tgz", + "integrity": "sha512-w+WyX5Ulmj4dtTZrxaulqrjaLZHSbnPzx75SJsTNYmotKsqn1JlLnDJa+lz5hn90HJofhl/2MAtw0mCrgM3qYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@mapbox/node-pre-gyp": "^2.0.0", + "@rollup/pluginutils": "^5.1.3", + "acorn": "^8.6.0", + "acorn-import-attributes": "^1.9.5", + "async-sema": "^3.1.1", + "bindings": "^1.4.0", + "estree-walker": "2.0.2", + "glob": "^13.0.0", + "graceful-fs": "^4.2.9", + "node-gyp-build": "^4.2.2", + "picomatch": "^4.0.2", + "resolve-from": "^5.0.0" + }, + "bin": { + "nft": "out/cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-2.0.3.tgz", + "integrity": "sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "consola": "^3.2.3", + "detect-libc": "^2.0.0", + "https-proxy-agent": "^7.0.5", + "node-fetch": "^2.6.7", + "nopt": "^8.0.0", + "semver": "^7.5.3", + "tar": "^7.4.0" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch/node_modules/whatwg-url/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch/node_modules/whatwg-url/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/nopt": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz", + "integrity": "sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^3.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/nopt/node_modules/abbrev": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", + "integrity": "sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar": { + "version": "7.5.19", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.19.tgz", + "integrity": "sha512-4LeEWl96twnS2Q7Bz4MGqgazLqO+hJN63GZxXoIqh1T3VweYD997gbU1ItNsQafqqXTXd5WFyFdReLtwvRBNiw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar/node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar/node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@vercel/nft/node_modules/@mapbox/node-pre-gyp/node_modules/tar/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@vercel/nft/node_modules/@rollup/pluginutils": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.4.0.tgz", + "integrity": "sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@vercel/nft/node_modules/@rollup/pluginutils/node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vercel/nft/node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/@vercel/nft/node_modules/async-sema": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz", + "integrity": "sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vercel/nft/node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/@vercel/nft/node_modules/bindings/node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vercel/nft/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vercel/nft/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/minimatch/node_modules/brace-expansion": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", + "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/minimatch/node_modules/brace-expansion/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@vercel/nft/node_modules/glob/node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.5.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", + "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@vercel/nft/node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/@vercel/nft/node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "dev": true, + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/@vercel/nft/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@vercel/nft/node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", diff --git a/utils/build/docker/nodejs/weblog_metadata.yml b/utils/build/docker/nodejs/weblog_metadata.yml new file mode 100644 index 00000000000..802e518616e --- /dev/null +++ b/utils/build/docker/nodejs/weblog_metadata.yml @@ -0,0 +1,23 @@ +# docs/understand/weblogs/weblog-metadata.md + +express4: + build_mode: local +express4-typescript: + build_mode: local +express5: + build_mode: local +fastify: + build_mode: local +nextjs: + build_mode: local +uds-express4: + build_mode: local +openai-js: + build_mode: none + framework_versions: ["6.0.0"] +anthropic-js: + build_mode: none + framework_versions: ["0.71.0"] +google_genai-js: + build_mode: none + framework_versions: ["1.34.0"] diff --git a/utils/build/docker/otel_collector/weblog_metadata.yml b/utils/build/docker/otel_collector/weblog_metadata.yml new file mode 100644 index 00000000000..3873e9082cb --- /dev/null +++ b/utils/build/docker/otel_collector/weblog_metadata.yml @@ -0,0 +1,4 @@ +# docs/understand/weblogs/weblog-metadata.md + +otel_collector: + build_mode: none diff --git a/utils/build/docker/python/weblog_metadata.yml b/utils/build/docker/python/weblog_metadata.yml new file mode 100644 index 00000000000..c0b67e68fa7 --- /dev/null +++ b/utils/build/docker/python/weblog_metadata.yml @@ -0,0 +1,11 @@ +# docs/understand/weblogs/weblog-metadata.md + +openai-py: + build_mode: none + framework_versions: ["2.0.0"] +anthropic-py: + build_mode: none + framework_versions: ["0.75.0"] +google_genai-py: + build_mode: none + framework_versions: ["1.55.0"] diff --git a/utils/scripts/build-base-image.py b/utils/scripts/build-base-image.py new file mode 100755 index 00000000000..f52d9c3ab8b --- /dev/null +++ b/utils/scripts/build-base-image.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import sys + +from utils._context.weblog_metadata import WeblogMetaData + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build a weblog base image") + parser.add_argument("library", help="Library name (e.g. nodejs, python)") + parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") + args = parser.parse_args() + + weblogs = WeblogMetaData.load(args.library) + names = [w.name for w in weblogs] + + if args.weblog not in names: + print(f"Error: weblog '{args.weblog}' not found for library '{args.library}'") + print(f"Available weblogs: {', '.join(sorted(names))}") + sys.exit(1) + + weblog = next(w for w in weblogs if w.name == args.weblog) + dockerfile = weblog.base_dockerfile + if dockerfile is None: + print(f"Error: no base Dockerfile found for {args.weblog}") + sys.exit(1) + + image_tag = weblog.base_image_tag + if image_tag is None: + print(f"Error: could not extract base image tag for {args.weblog}") + sys.exit(1) + + print(f"Building base image: {image_tag}") + subprocess.run(["docker", "build", "--progress=plain", "-f", str(dockerfile), "-t", image_tag, "."], check=True) + + print(f"\nDone. To push:\n docker push {image_tag}") + + +if __name__ == "__main__": + main() diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4c686a28aac..75480901b22 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -1,9 +1,7 @@ from collections import defaultdict -from dataclasses import dataclass import json -import os -from pathlib import Path from utils._context._scenarios import go_proxies +from utils._context.weblog_metadata import WeblogMetaData as Weblog, BuildMode def _load_json(file_path: str) -> dict: @@ -196,21 +194,20 @@ def get_docker_ssi_matrix( # End-to-end corner -@dataclass -class Weblog: - name: str - require_build: bool - artifact_name: str - - def serialize(self) -> dict: - return {"name": self.name, "artifact_name": self.artifact_name} class Job: """a job is a couple weblog/scenarios that will be executed in a single runner""" def __init__( - self, library: str, weblog: Weblog, weblog_instance: int, scenarios_times: dict[str, float], build_time: float + self, + library: str, + weblog: Weblog, + weblog_instance: int, + scenarios_times: dict[str, float], + build_time: float, + *, + build_base_images: bool, ): self.library = library self.weblog = weblog @@ -227,6 +224,9 @@ def __init__( # split mechanism self.build_time = build_time + self.build_base_images = build_base_images + """ Shall the end-to-end scenario rebuild the weblog base image for fully baked weblog """ + def serialize(self) -> dict: return { "runs_on": "ubuntu-latest", @@ -237,6 +237,9 @@ def serialize(self) -> dict: "scenarios": sorted(self.scenarios), "expected_job_time": self.expected_job_time + self.build_time, "binaries_artifact": self.weblog.artifact_name, + "build_weblog_base_image": self.weblog.build_mode == BuildMode.local + and self.build_base_images + and self.weblog.base_dockerfile is not None, } @property @@ -272,6 +275,7 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J weblog_instance=i + 1, scenarios_times={scenario: self._scenarios_times[scenario] for scenario in scenarios}, build_time=self.build_time, + build_base_images=self.build_base_images, ) ) @@ -279,59 +283,26 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J def _get_endtoend_weblogs( - library: str, weblogs_filter: list[str], unique_id: str, ci_environment: str, binaries_artifact: str + library: str, + weblogs_filter: list[str], + unique_id: str, + ci_environment: str, + binaries_artifact: str, ) -> list[Weblog]: - result: list[Weblog] = [] - - integration_frameworks_weblogs = { - # openai - "openai-py": ["2.0.0"], - "openai-js": ["6.0.0"], - "openai-java": ["4.29.0"], - # anthropic - "anthropic-js": ["0.71.0"], - "anthropic-py": ["0.75.0"], - # google_genai - "google_genai-py": ["1.55.0"], - "google_genai-js": ["1.34.0"], - } + weblogs: list[Weblog] = Weblog.load(library) - folder = f"utils/build/docker/{library}" - if Path(folder).exists(): # some lib does not have any weblog - names = [ - f.replace(".Dockerfile", "") - for f in os.listdir(folder) - if f.endswith(".Dockerfile") and ".base." not in f and Path(os.path.join(folder, f)).is_file() - ] - - if len(weblogs_filter) != 0: - # filter weblogs by the weblogs_filter set - names = [weblog for weblog in names if weblog in weblogs_filter] - - for name in names: - if name not in integration_frameworks_weblogs: - result.append( - Weblog( - name=name, - require_build=True, - artifact_name=f"binaries_{ci_environment}_{library}_{name}_{unique_id}", - ) - ) - else: - for version in integration_frameworks_weblogs[name]: - result.append( - Weblog(name=f"{name}@{version}", require_build=False, artifact_name=binaries_artifact) - ) - - # weblog not related to a docker file - for weblog, lib in go_proxies.GO_PROXIES_WEBLOGS.items(): - if lib == library: - result.append(Weblog(name=weblog, require_build=False, artifact_name=binaries_artifact)) + if len(weblogs_filter) != 0: + # filter weblogs by the weblogs_filter set + weblogs = [w for w in weblogs if w.name in weblogs_filter] - if library == "otel_collector": - result.append(Weblog(name="otel_collector", require_build=False, artifact_name=binaries_artifact)) + for weblog in weblogs: + weblog.artifact_name = ( + f"binaries_{ci_environment}_{library}_{weblog.name}_{unique_id}" + if weblog.build_mode == "prebuild" + else binaries_artifact + ) - return sorted(result, key=lambda w: w.name) + return sorted(weblogs, key=lambda w: w.name) def get_endtoend_definitions( @@ -343,6 +314,8 @@ def get_endtoend_definitions( maximum_parallel_jobs: int, unique_id: str, binaries_artifact: str, + *, + build_base_images: bool = False, ) -> dict: scenarios = scenario_map["endtoend"] @@ -352,7 +325,11 @@ def get_endtoend_definitions( # get the list of end-to-end weblogs for the given library weblogs: list[Weblog] = _get_endtoend_weblogs( - library, weblogs_filter, ci_environment=ci_environment, unique_id=unique_id, binaries_artifact=binaries_artifact + library, + weblogs_filter, + ci_environment=ci_environment, + unique_id=unique_id, + binaries_artifact=binaries_artifact, ) # check that jobs can be splitted @@ -376,6 +353,7 @@ def get_endtoend_definitions( weblog_instance=1, scenarios_times=scenarios_times, build_time=_get_build_time(library, weblog, time_stats["build"]), + build_base_images=build_base_images, ) ) @@ -393,12 +371,24 @@ def get_endtoend_definitions( return { "endtoend_defs": { "parallel_enable": len(jobs) > 0, - "parallel_weblogs": [weblog.serialize() for weblog in weblogs if weblog.require_build], + "parallel_weblogs": [ + _get_weblog_build_job(weblog, build_base_images=build_base_images) + for weblog in weblogs + if weblog.build_mode == BuildMode.prebuild + ], "parallel_jobs": [job.serialize() for job in jobs], } } +def _get_weblog_build_job(weblog: Weblog, *, build_base_images: bool) -> dict: + return { + "name": weblog.name, + "artifact_name": weblog.artifact_name, + "build_base_images": build_base_images and weblog.base_dockerfile is not None, + } + + def _split_jobs_for_parallel_execution( jobs: list[Job], desired_execution_time: float, maximum_parallel_jobs: int ) -> list[Job]: @@ -464,7 +454,7 @@ def __init__(self, scenario: str, execution_time: float): def _get_build_time(library: str, weblog: Weblog, build_stats: dict) -> float: - if not weblog.require_build: + if weblog.build_mode != BuildMode.prebuild: return 0.0 if library not in build_stats: diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 2f730e72c4c..6c80897ce44 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,6 +43,7 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, + build_weblog_base_images: bool = False, ): # this data struture is a dict where: # the key is the workflow identifier @@ -87,6 +88,7 @@ def __init__( maximum_parallel_jobs=256, unique_id=self.unique_id, binaries_artifact=self.binaries_artifact, + build_base_images=build_weblog_base_images, ) self.data["parametric"] = { @@ -286,6 +288,12 @@ def _get_workflow_map( "--system-tests-dev-mode", type=str, help="true if running in system-tests CI, with the dev mode", default="" ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) + parser.add_argument( + "--build-weblog-base-images", + type=str, + help="Rebuild weblog base images", + default="", + ) args = parser.parse_args() @@ -305,4 +313,5 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, + build_weblog_base_images=args.build_weblog_base_images == "true", ).export(export_format=args.format, output=args.output)