From cc79cfa2edca5d230762e5f47cb9e34eb0b01abc Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 21 Oct 2026 15:38:27 -0700 Subject: [PATCH 1/2] Reuse benchmark builds within each CI pipeline run Cache application and supporting builds for container, gRPC, HttpClient, SignalR, and WebSocket matrices using run-scoped source and build keys. Isolate Docker and compression variants, document the cache lifecycle, and add configuration regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d203afa3-f8a9-47b3-9353-e38e91a647e9 --- build/README.md | 51 +++++ build/build-reuse.profile.yml | 97 +++++++++ build/containers-scenarios.yml | 2 +- build/grpc-scenarios.yml | 20 +- build/httpclient-scenarios.yml | 4 +- build/job-variables.yml | 2 + build/signalr-scenarios.yml | 3 +- build/tests/test_build_reuse.py | 301 ++++++++++++++++++++++++++++ build/websocket-scenarios.yml | 2 +- scenarios/containers.benchmarks.yml | 6 + 10 files changed, 472 insertions(+), 16 deletions(-) create mode 100644 build/build-reuse.profile.yml create mode 100644 build/tests/test_build_reuse.py diff --git a/build/README.md b/build/README.md index 2d4941624..c06bd3e1d 100644 --- a/build/README.md +++ b/build/README.md @@ -100,6 +100,57 @@ together. The snapshot tests in `scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the generated YAML drifts from the configs. +## Build reuse within a pipeline run + +`build-reuse.profile.yml` supplies opt-in Crank profiles for the high-repeat +container, gRPC, HttpClient, SignalR, and WebSocket matrices. Their scenario +templates load `$(buildReuseProfile)` and pass +`--variable buildCacheId=$(Build.BuildId)`. The first use of each build on an +agent checks out sources and builds; later compatible entries reuse it. A new +pipeline build ID changes both the source key and the build key, so floating +branches, .NET versions, and Docker base images are refreshed on the next run. +`--session` alone does not invalidate Crank caches. Source and build keys have +different prefixes because their metadata lives in the same cache directory. + +| Suite | Builds reused across entries | +|---|---| +| Containers | Application and database images across 9 sizes and 3 request rates | +| gRPC | Separate ASP.NET Core/Go servers and .NET/Go/h2load clients across calls, protocols, and connection/stream counts | +| HttpClient | Client and Kestrel server across GET/POST, HTTP versions, concurrency, and payload sizes | +| SignalR | Server and client across scenarios, transports, and protocols; warmup and load share the client build | +| WebSockets | Client across payload sizes; separate compressed/uncompressed server builds | + +These profiles reuse builds, not running processes or database contents. +Containers and applications are still started and stopped for each benchmark; +warmup and measurement durations are unchanged. Caches are local to each agent, +not shared between machines or architectures, and remain subject to the agent's +normal cache cleanup/lifetime. These profiles do not add a cache TTL; retained +builds need the same agent cleanup/restart policy as other Crank caches. + +Container cache keys include the image name. Different Dockerfiles must have +different image names; the ASP.NET default, Alpine, Composite, and current +variants therefore have separate tags. gRPC scenario entries specify +`serverBuildKey` and `clientBuildKey` to separate implementations, and WebSocket +server keys include the compile-time compression mode. When adding a new +compile-time variant to these matrices, give it a distinct build identity +rather than reusing an existing key. + +Other potential follow-ups are the proxy payload/protocol matrix and the +baseline/trend suites. They are not opted in here: proxies vary configuration +files copied into Docker images, while baseline/trend suites vary projects, +frameworks, and publishing settings that need their own cache identities. +The SslStream/QUIC matrix is currently disabled. Build-performance benchmarks +must continue building because build time is what they measure. + +To check cache isolation without contacting agents or running benchmarks: + +```bash +python -m unittest discover -s build/tests +``` + +The profile tests use the existing `crank --debug` configuration-only mode and +skip when the Crank CLI is not installed. Template wiring checks run without it. + ## Minimal Todo `minimal-todo-scenarios.yml` dispatches `scenarios/minimal-todo/minimal-todo.benchmarks.yml` diff --git a/build/build-reuse.profile.yml b/build/build-reuse.profile.yml new file mode 100644 index 000000000..9e33bc871 --- /dev/null +++ b/build/build-reuse.profile.yml @@ -0,0 +1,97 @@ +# Opt-in CI caches: rebuild on each pipeline run, reuse only compatible variants. +# Source and build keys need separate namespaces because they share cache metadata. +profiles: + containers-build-reuse: + agents: + application: + buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + source: + sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + db: + buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + source: + sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' + options: + reuseBuild: true + load: {} + + grpc-build-reuse: + agents: + application: + buildKey: 'grpc-server-build-{{ serverBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'grpc-server-source-{{ serverBuildKey }}-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + - 'if (!job.Variables.serverBuildKey || !job.Variables.clientBuildKey || !/^[a-z0-9-]+$/.test(String(job.Variables.serverBuildKey)) || !/^[a-z0-9-]+$/.test(String(job.Variables.clientBuildKey))) { throw "serverBuildKey and clientBuildKey must identify the gRPC variants using lowercase letters, digits, or hyphens."; }' + load: + buildKey: 'grpc-client-build-{{ clientBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'grpc-client-source-{{ clientBuildKey }}-{{ buildCacheId }}' + options: + reuseBuild: true + + httpclient-build-reuse: + agents: + load: + aliases: + - secondary + buildKey: 'httpclient-server-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'httpclient-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + application: + aliases: + - main + buildKey: 'httpclient-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'httpclient-source-{{ buildCacheId }}' + options: + reuseBuild: true + + signalr-build-reuse: + agents: + application: + buildKey: 'signalr-server-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'signalr-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + load: + aliases: + - warmup + - secondary + buildKey: 'signalr-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'signalr-source-{{ buildCacheId }}' + options: + reuseBuild: true + + websocket-build-reuse: + agents: + application: + buildKey: 'websocket-server-build-{{ job.Framework }}-{{ compress }}-{{ buildCacheId }}' + source: + sourceKey: 'websocket-source-{{ buildCacheId }}' + options: + reuseBuild: true + onConfigure: + - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' + - 'if (!/^(true|false)$/.test(String(job.Variables.compress))) { throw "compress must be true or false for WebSocket build reuse."; }' + load: + buildKey: 'websocket-client-build-{{ job.Framework }}-{{ buildCacheId }}' + source: + sourceKey: 'websocket-source-{{ buildCacheId }}' + options: + reuseBuild: true diff --git a/build/containers-scenarios.yml b/build/containers-scenarios.yml index cedd393d4..6f1c565f2 100644 --- a/build/containers-scenarios.yml +++ b/build/containers-scenarios.yml @@ -145,6 +145,6 @@ steps: { "name": "crank", "condition": "(${{ parameters.condition }}) && (${{ scenario.condition }})", - "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) $(buildReuseProfile) --profile containers-build-reuse --variable buildCacheId=$(Build.BuildId) --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] } diff --git a/build/grpc-scenarios.yml b/build/grpc-scenarios.yml index b2bcc648c..ed4f0fee9 100644 --- a/build/grpc-scenarios.yml +++ b/build/grpc-scenarios.yml @@ -26,14 +26,14 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet - displayName: "ASP.NET Core - Client: Go" - arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=go - displayName: "Go - Client: Grpc.Net.Client" - arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient + arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient --variable serverBuildKey=go --variable clientBuildKey=dotnet - displayName: "Go - Client: Go" - arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient + arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient --variable serverBuildKey=go --variable clientBuildKey=go # H2Load scenarios - name: h2loadscenarios @@ -41,9 +41,9 @@ parameters: default: - displayName: "ASP.NET Core - Client: h2load" - arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=h2load - displayName: "Go - Client: h2load" - arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient + arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient --variable serverBuildKey=go --variable clientBuildKey=h2load # grpc-net only scenarios - name: grpcnetscenarios @@ -51,7 +51,7 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet - name: callTypes type: object @@ -107,7 +107,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] + "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] } # h2/h3 HttpClient to ASP.NET Core @@ -132,7 +132,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }}" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } # h2c between various clients and servers @@ -154,5 +154,5 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} --variable protocol=h2c --property protocol=h2c" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c" ] } diff --git a/build/httpclient-scenarios.yml b/build/httpclient-scenarios.yml index 2931468b9..c5ce3c8b9 100644 --- a/build/httpclient-scenarios.yml +++ b/build/httpclient-scenarios.yml @@ -94,7 +94,7 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }}" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } # POST scenarios @@ -116,5 +116,5 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }}" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] } diff --git a/build/job-variables.yml b/build/job-variables.yml index 69216f750..798dfda33 100644 --- a/build/job-variables.yml +++ b/build/job-variables.yml @@ -77,6 +77,8 @@ variables: value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml - name: ciProfile value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml +- name: buildReuseProfile + value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/build-reuse.profile.yml - name: grpcJobs value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/grpc.benchmarks.yml - name: efcoreJobs diff --git a/build/signalr-scenarios.yml b/build/signalr-scenarios.yml index 9961df0ce..72111840e 100644 --- a/build/signalr-scenarios.yml +++ b/build/signalr-scenarios.yml @@ -74,6 +74,5 @@ steps: "condition": "${{ parameters.condition }}", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] + "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile signalr-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] } - diff --git a/build/tests/test_build_reuse.py b/build/tests/test_build_reuse.py new file mode 100644 index 000000000..acd0d4877 --- /dev/null +++ b/build/tests/test_build_reuse.py @@ -0,0 +1,301 @@ +"""Validate CI cache profiles with Crank's configuration-only debug mode. + +Run from the repo root: python -m unittest discover -s build/tests +Requires the existing crank CLI; no agents, builds, or network access are used. +""" + +import json +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CRANK = shutil.which("crank") +PROFILES = { + "containers": ("application", "db"), + "grpc": ("application", "load"), + "httpclient": ("server", "client"), + "signalr": ("application", "warmup", "load"), + "websocket": ("application", "load"), +} + + +class TestTemplateWiring(unittest.TestCase): + def test_all_messages_enable_run_scoped_reuse(self): + for suite in PROFILES: + with self.subTest(suite=suite): + text = (ROOT / "build" / f"{suite}-scenarios.yml").read_text( + encoding="utf-8-sig" + ) + messages = [ + line for line in text.splitlines() if '"args": [' in line + ] + self.assertTrue(messages) + for message in messages: + self.assertIn("$(buildReuseProfile)", message) + self.assertIn(f"--profile {suite}-build-reuse", message) + self.assertIn( + "--variable buildCacheId=$(Build.BuildId)", message + ) + + def test_grpc_scenarios_identify_both_build_variants(self): + text = (ROOT / "build" / "grpc-scenarios.yml").read_text() + scenarios = re.findall( + r"arguments: --scenario (grpc\S+)([^\n]+)", text + ) + self.assertEqual(len(scenarios), 7) + for scenario, arguments in scenarios: + with self.subTest(scenario=scenario): + server, client = scenario.split("-") + server_key = "aspnetcore" if "aspnetcore" in server else "go" + client_key = { + "grpcnetclient": "dotnet", + "grpcgoclient": "go", + "h2loadclient": "h2load", + }[client] + self.assertIn(f"serverBuildKey={server_key}", arguments) + self.assertIn(f"clientBuildKey={client_key}", arguments) + + def test_container_dockerfile_variants_have_distinct_tags(self): + text = (ROOT / "scenarios" / "containers.benchmarks.yml").read_text() + variants = re.findall( + r"dockerFile: (docker/container-matrix/\S+)\n" + r"\s+dockerImageName: (\S+)", text + ) + self.assertEqual(len(variants), 7) + tags = {} + for dockerfile, tag in variants: + self.assertEqual(tags.setdefault(tag, dockerfile), dockerfile) + self.assertEqual(len(tags), 4) + + +@unittest.skipUnless(CRANK, "Requires the crank CLI") +class TestBuildReuseProfiles(unittest.TestCase): + def configure(self, suite, run_id="123", variables=None, patches=None, error=None): + roles = PROFILES[suite] + if suite == "containers": + roles = (*roles, "load") + jobs = { + role: { + "source": { + "repository": "https://github.com/aspnet/Benchmarks.git", + "branchOrCommit": "main", + "project": f"src/{role}/App.csproj", + }, + "framework": "net11.0", + } + for role in roles + } + if suite == "containers": + for role in PROFILES[suite]: + jobs[role]["source"] = { + "repository": "https://github.com/TechEmpower/FrameworkBenchmarks.git", + "branchOrCommit": "master", + "dockerFile": f"{role}/Dockerfile", + "dockerImageName": role, + } + if suite == "signalr": + jobs["warmup"]["source"]["project"] = "src/load/App.csproj" + for role, patch in (patches or {}).items(): + jobs[role].update(patch) + + scenario = {role: {"job": role} for role in roles} + if suite == "httpclient": + scenario["client"]["agent"] = "main" + scenario["server"]["agent"] = "secondary" + config = { + "jobs": jobs, + "profiles": { + "machines": { + "agents": { + "application": {"endpoints": [], "aliases": ["main"]}, + "load": { + "endpoints": [], + "aliases": ["warmup", "secondary"], + }, + "db": {"endpoints": []}, + } + } + }, + "scenarios": {"test": scenario}, + } + values = { + "compress": "false", + "serverBuildKey": "aspnetcore", + "clientBuildKey": "dotnet", + } + if run_id is not None: + values["buildCacheId"] = run_id + values.update(variables or {}) + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "scenario.json" + output = Path(temp) / "debug.json" + source.write_text(json.dumps(config), encoding="utf-8") + command = [ + CRANK, + "--config", str(source), + "--config", str(ROOT / "build" / "build-reuse.profile.yml"), + "--scenario", "test", + "--profile", "machines", + "--profile", f"{suite}-build-reuse", + "--debug", str(output), + ] + for key, value in values.items(): + command.extend(("--variable", f"{key}={value}")) + result = subprocess.run( + command, capture_output=True, text=True, timeout=60 + ) + if run_id is None or not str(run_id).isdigit(): + error = "buildCacheId must be set" + if error is not None: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn(error, result.stdout + result.stderr) + return None + self.assertEqual( + result.returncode, 0, result.stdout + result.stderr + ) + return json.loads(output.read_text(encoding="utf-8"))["Jobs"] + + def assert_cached(self, jobs, suite): + for role in PROFILES[suite]: + with self.subTest(suite=suite, role=role): + job = jobs[role] + self.assertTrue(job["Options"]["ReuseBuild"]) + self.assertTrue(job["NoBuild"]) + self.assertTrue(job["BuildKey"].endswith("-123")) + source_key = job["Source"]["SourceKey"] + self.assertTrue(source_key.endswith("-123")) + self.assertNotEqual(job["BuildKey"], source_key) + + def test_each_profile_invalidates_sources_and_builds_on_new_run(self): + for suite in PROFILES: + with self.subTest(suite=suite): + first = self.configure(suite) + self.assert_cached(first, suite) + second = self.configure(suite, "124") + for role in PROFILES[suite]: + self.assertNotEqual( + first[role]["BuildKey"], second[role]["BuildKey"] + ) + self.assertNotEqual( + first[role]["Source"]["SourceKey"], + second[role]["Source"]["SourceKey"], + ) + + def test_missing_or_invalid_run_id_is_rejected(self): + for suite in PROFILES: + for run_id in (None, "not-a-build"): + with self.subTest(suite=suite, run_id=run_id): + self.configure(suite, run_id) + + def test_missing_or_invalid_build_variants_are_rejected(self): + for key, value in ( + ("serverBuildKey", ""), + ("clientBuildKey", "../invalid"), + ): + with self.subTest(key=key): + self.configure( + "grpc", + variables={key: value}, + error="serverBuildKey and clientBuildKey must identify", + ) + self.configure( + "websocket", + variables={"compress": ""}, + error="compress must be true or false", + ) + + def test_container_runtime_limits_and_rate_do_not_change_cache(self): + first = self.configure("containers", variables={"rate": "1000"}) + second = self.configure( + "containers", + variables={"rate": "10000"}, + patches={ + "application": { + "cpuLimitRatio": 0.25, + "memoryLimitInBytes": 256000000, + "cpuSet": "0", + } + }, + ) + for role in PROFILES["containers"]: + self.assertEqual(first[role]["BuildKey"], second[role]["BuildKey"]) + self.assertEqual( + first[role]["Source"]["SourceKey"], + second[role]["Source"]["SourceKey"], + ) + + def test_container_image_variants_have_separate_caches(self): + first = self.configure("containers") + second = self.configure( + "containers", + patches={ + "application": { + "source": { + "repository": "https://github.com/aspnet/Benchmarks.git", + "branchOrCommit": "main", + "dockerFile": "docker/container-matrix/Middleware-alpine.dockerfile", + "dockerImageName": "app-alpine", + } + } + }, + ) + self.assertNotEqual( + first["application"]["BuildKey"], second["application"]["BuildKey"] + ) + self.assertNotEqual( + first["application"]["Source"]["SourceKey"], + second["application"]["Source"]["SourceKey"], + ) + self.assertEqual(first["db"]["BuildKey"], second["db"]["BuildKey"]) + + def test_websocket_compression_builds_are_separate(self): + plain = self.configure("websocket") + compressed = self.configure( + "websocket", + variables={"compress": "true"}, + patches={"application": {"buildArguments": ["/p:UseCompression=true"]}}, + ) + self.assertNotEqual( + plain["application"]["BuildKey"], + compressed["application"]["BuildKey"], + ) + self.assertEqual(plain["load"]["BuildKey"], compressed["load"]["BuildKey"]) + + def test_signalr_warmup_and_load_share_client_build(self): + jobs = self.configure("signalr") + self.assertEqual(jobs["warmup"]["BuildKey"], jobs["load"]["BuildKey"]) + self.assertNotEqual(jobs["application"]["BuildKey"], jobs["load"]["BuildKey"]) + + def test_grpc_server_and_client_variants_are_separate(self): + dotnet = self.configure("grpc") + go = self.configure( + "grpc", variables={"serverBuildKey": "go", "clientBuildKey": "go"} + ) + h2load = self.configure("grpc", variables={"clientBuildKey": "h2load"}) + self.assertNotEqual( + dotnet["application"]["BuildKey"], go["application"]["BuildKey"] + ) + self.assertEqual( + dotnet["application"]["BuildKey"], h2load["application"]["BuildKey"] + ) + self.assertEqual( + len({jobs["load"]["BuildKey"] for jobs in (dotnet, go, h2load)}), 3 + ) + + def test_framework_versions_are_separate(self): + first = self.configure("httpclient") + second = self.configure( + "httpclient", + patches={"client": {"framework": "net10.0"}}, + ) + self.assertNotEqual(first["client"]["BuildKey"], second["client"]["BuildKey"]) + self.assertEqual(first["server"]["BuildKey"], second["server"]["BuildKey"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/build/websocket-scenarios.yml b/build/websocket-scenarios.yml index 73babf846..b48d64408 100644 --- a/build/websocket-scenarios.yml +++ b/build/websocket-scenarios.yml @@ -65,5 +65,5 @@ steps: "condition": "(${{ parameters.condition }})", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] + "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile websocket-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] } diff --git a/scenarios/containers.benchmarks.yml b/scenarios/containers.benchmarks.yml index 21161cba4..0bef6f654 100644 --- a/scenarios/containers.benchmarks.yml +++ b/scenarios/containers.benchmarks.yml @@ -39,6 +39,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-alpine.dockerfile + dockerImageName: app-alpine variables: scenario: json load: @@ -53,6 +54,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-composite.dockerfile + dockerImageName: app-composite variables: scenario: json load: @@ -83,6 +85,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-alpine.dockerfile + dockerImageName: app-alpine variables: scenario: dbfortunesraw load: @@ -99,6 +102,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-composite.dockerfile + dockerImageName: app-composite variables: scenario: dbfortunesraw load: @@ -113,6 +117,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-current.dockerfile + dockerImageName: app-current variables: scenario: json load: @@ -129,6 +134,7 @@ scenarios: job: aspnet source: dockerFile: docker/container-matrix/Middleware-current.dockerfile + dockerImageName: app-current variables: scenario: dbfortunesraw load: From ce6674663c76afe983cd2690754ac5033cafe1af Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 21 Oct 2026 16:41:16 -0700 Subject: [PATCH 2/2] Inline build reuse options and isolate Docker tags per run Remove the shared cache profile and standalone configuration test harness. Keep per-run source and build identities directly in existing templates, and suffix cached container and gRPC Go image names with the pipeline build ID to prevent interleaved runs from sharing mutable tags. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d203afa3-f8a9-47b3-9353-e38e91a647e9 --- build/README.md | 64 ++----- build/build-reuse.profile.yml | 97 ---------- build/containers-scenarios.yml | 24 +-- build/grpc-scenarios.yml | 34 ++-- build/httpclient-scenarios.yml | 4 +- build/job-variables.yml | 8 +- build/signalr-scenarios.yml | 2 +- build/tests/test_build_reuse.py | 301 -------------------------------- build/websocket-scenarios.yml | 2 +- 9 files changed, 62 insertions(+), 474 deletions(-) delete mode 100644 build/build-reuse.profile.yml delete mode 100644 build/tests/test_build_reuse.py diff --git a/build/README.md b/build/README.md index c06bd3e1d..61ee7191a 100644 --- a/build/README.md +++ b/build/README.md @@ -102,54 +102,22 @@ generated YAML drifts from the configs. ## Build reuse within a pipeline run -`build-reuse.profile.yml` supplies opt-in Crank profiles for the high-repeat -container, gRPC, HttpClient, SignalR, and WebSocket matrices. Their scenario -templates load `$(buildReuseProfile)` and pass -`--variable buildCacheId=$(Build.BuildId)`. The first use of each build on an -agent checks out sources and builds; later compatible entries reuse it. A new -pipeline build ID changes both the source key and the build key, so floating -branches, .NET versions, and Docker base images are refreshed on the next run. -`--session` alone does not invalidate Crank caches. Source and build keys have -different prefixes because their metadata lives in the same cache directory. - -| Suite | Builds reused across entries | -|---|---| -| Containers | Application and database images across 9 sizes and 3 request rates | -| gRPC | Separate ASP.NET Core/Go servers and .NET/Go/h2load clients across calls, protocols, and connection/stream counts | -| HttpClient | Client and Kestrel server across GET/POST, HTTP versions, concurrency, and payload sizes | -| SignalR | Server and client across scenarios, transports, and protocols; warmup and load share the client build | -| WebSockets | Client across payload sizes; separate compressed/uncompressed server builds | - -These profiles reuse builds, not running processes or database contents. -Containers and applications are still started and stopped for each benchmark; -warmup and measurement durations are unchanged. Caches are local to each agent, -not shared between machines or architectures, and remain subject to the agent's -normal cache cleanup/lifetime. These profiles do not add a cache TTL; retained -builds need the same agent cleanup/restart policy as other Crank caches. - -Container cache keys include the image name. Different Dockerfiles must have -different image names; the ASP.NET default, Alpine, Composite, and current -variants therefore have separate tags. gRPC scenario entries specify -`serverBuildKey` and `clientBuildKey` to separate implementations, and WebSocket -server keys include the compile-time compression mode. When adding a new -compile-time variant to these matrices, give it a distinct build identity -rather than reusing an existing key. - -Other potential follow-ups are the proxy payload/protocol matrix and the -baseline/trend suites. They are not opted in here: proxies vary configuration -files copied into Docker images, while baseline/trend suites vary projects, -frameworks, and publishing settings that need their own cache identities. -The SslStream/QUIC matrix is currently disabled. Build-performance benchmarks -must continue building because build time is what they measure. - -To check cache isolation without contacting agents or running benchmarks: - -```bash -python -m unittest discover -s build/tests -``` - -The profile tests use the existing `crank --debug` configuration-only mode and -skip when the Crank CLI is not installed. Template wiring checks run without it. +The container, gRPC, HttpClient, SignalR, and WebSocket templates pass Crank's +`reuseBuild`, `buildKey`, and `source.sourceKey` options directly. Keys include +`$(Build.BuildId)`: each agent builds once per compatible variant in a pipeline +run, then refreshes sources and builds on the next run. Source and build keys use +separate namespaces because their metadata shares a cache directory. +`--session` alone does not invalidate caches. + +Keys distinguish Docker images, gRPC implementations and frameworks, and +WebSocket compression builds. Container scenarios with a database also include +`$(containerDatabaseBuildReuse)`. Docker image names get the build ID appended in +`onConfigure` hooks so interleaved runs cannot overwrite each other's image tags; +the gRPC hooks leave non-Docker jobs unchanged. + +Only builds are reused. Applications and database containers still start fresh, +and warmup/measurement durations are unchanged. Caches remain local to each +agent and rely on its existing cleanup/restart policy; no cache TTL is added. ## Minimal Todo diff --git a/build/build-reuse.profile.yml b/build/build-reuse.profile.yml deleted file mode 100644 index 9e33bc871..000000000 --- a/build/build-reuse.profile.yml +++ /dev/null @@ -1,97 +0,0 @@ -# Opt-in CI caches: rebuild on each pipeline run, reuse only compatible variants. -# Source and build keys need separate namespaces because they share cache metadata. -profiles: - containers-build-reuse: - agents: - application: - buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' - source: - sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' - options: - reuseBuild: true - onConfigure: - - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' - db: - buildKey: 'containers-build-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' - source: - sourceKey: 'containers-source-{{ job.Source.DockerImageName }}-{{ buildCacheId }}' - options: - reuseBuild: true - load: {} - - grpc-build-reuse: - agents: - application: - buildKey: 'grpc-server-build-{{ serverBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'grpc-server-source-{{ serverBuildKey }}-{{ buildCacheId }}' - options: - reuseBuild: true - onConfigure: - - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' - - 'if (!job.Variables.serverBuildKey || !job.Variables.clientBuildKey || !/^[a-z0-9-]+$/.test(String(job.Variables.serverBuildKey)) || !/^[a-z0-9-]+$/.test(String(job.Variables.clientBuildKey))) { throw "serverBuildKey and clientBuildKey must identify the gRPC variants using lowercase letters, digits, or hyphens."; }' - load: - buildKey: 'grpc-client-build-{{ clientBuildKey }}-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'grpc-client-source-{{ clientBuildKey }}-{{ buildCacheId }}' - options: - reuseBuild: true - - httpclient-build-reuse: - agents: - load: - aliases: - - secondary - buildKey: 'httpclient-server-build-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'httpclient-source-{{ buildCacheId }}' - options: - reuseBuild: true - onConfigure: - - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' - application: - aliases: - - main - buildKey: 'httpclient-client-build-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'httpclient-source-{{ buildCacheId }}' - options: - reuseBuild: true - - signalr-build-reuse: - agents: - application: - buildKey: 'signalr-server-build-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'signalr-source-{{ buildCacheId }}' - options: - reuseBuild: true - onConfigure: - - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' - load: - aliases: - - warmup - - secondary - buildKey: 'signalr-client-build-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'signalr-source-{{ buildCacheId }}' - options: - reuseBuild: true - - websocket-build-reuse: - agents: - application: - buildKey: 'websocket-server-build-{{ job.Framework }}-{{ compress }}-{{ buildCacheId }}' - source: - sourceKey: 'websocket-source-{{ buildCacheId }}' - options: - reuseBuild: true - onConfigure: - - 'if (!/^[0-9]+$/.test(String(job.Variables.buildCacheId))) { throw "buildCacheId must be set to the pipeline build ID."; }' - - 'if (!/^(true|false)$/.test(String(job.Variables.compress))) { throw "compress must be true or false for WebSocket build reuse."; }' - load: - buildKey: 'websocket-client-build-{{ job.Framework }}-{{ buildCacheId }}' - source: - sourceKey: 'websocket-source-{{ buildCacheId }}' - options: - reuseBuild: true diff --git a/build/containers-scenarios.yml b/build/containers-scenarios.yml index 6f1c565f2..9c84e0ed8 100644 --- a/build/containers-scenarios.yml +++ b/build/containers-scenarios.yml @@ -62,52 +62,52 @@ parameters: arguments: --scenario json_aspnet --property scenario=JsonAspNet condition: Math.round(Date.now() / 43200000) % 8 == 0 # once every 8 half-days - displayName: Fortunes ASP.NET - arguments: --scenario fortunes_aspnet --property scenario=FortunesAspNet --application.environmentVariables DOTNET_gcServer=1 --application.environmentVariables DOTNET_GCDynamicAdaptationMode=0 + arguments: --scenario fortunes_aspnet --property scenario=FortunesAspNet --application.environmentVariables DOTNET_gcServer=1 --application.environmentVariables DOTNET_GCDynamicAdaptationMode=0 $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 8 == 0 # once every 8 half-days - displayName: Fortunes ASP.NET - DATAS - arguments: --scenario fortunes_aspnet --property scenario=FortunesAspNetDATAS --application.environmentVariables DOTNET_GCDynamicAdaptationMode=1 + arguments: --scenario fortunes_aspnet --property scenario=FortunesAspNetDATAS --application.environmentVariables DOTNET_GCDynamicAdaptationMode=1 $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 8 == 0 # once every 8 half-days - displayName: Json ASP.NET Alpine arguments: --scenario json_aspnet_alpine --property scenario=JsonAspNetAlpine condition: Math.round(Date.now() / 43200000) % 8 == 2 # once every 8 half-days - displayName: Fortunes ASP.NET Alpine - arguments: --scenario fortunes_aspnet_alpine --property scenario=FortunesAspNetAlpine + arguments: --scenario fortunes_aspnet_alpine --property scenario=FortunesAspNetAlpine $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 8 == 2 # once every 8 half-days - displayName: Json ASP.NET Composite arguments: --scenario json_aspnet_composite --property scenario=JsonAspNetComposite condition: Math.round(Date.now() / 43200000) % 8 == 4 # once every 8 half-days - displayName: Fortunes ASP.NET Composite - arguments: --scenario fortunes_aspnet_composite --property scenario=FortunesAspNetComposite + arguments: --scenario fortunes_aspnet_composite --property scenario=FortunesAspNetComposite $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 8 == 4 # once every 8 half-days - displayName: Json ASP.NET 8.0 arguments: --scenario json_aspnet_current --property scenario=JsonAspNet80 condition: Math.round(Date.now() / 43200000) % 8 == 6 # once every 8 half-days - displayName: Fortunes ASP.NET 8.0 - arguments: --scenario fortunes_aspnet_current --property scenario=FortunesAspNet80 + arguments: --scenario fortunes_aspnet_current --property scenario=FortunesAspNet80 $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 8 == 6 # once every 8 half-days - displayName: Json NodeJS - arguments: --scenario json_nodejs --property scenario=JsonNodeJs + arguments: --scenario json_nodejs --property scenario=JsonNodeJs $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 2 # once every 10 half-days - displayName: Fortunes NodeJS - arguments: --scenario fortunes_nodejs --property scenario=FortunesNodeJs + arguments: --scenario fortunes_nodejs --property scenario=FortunesNodeJs $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 3 # once every 10 half-days - displayName: Json FastHttp - arguments: --scenario json_fasthttp --property scenario=JsonFastHttp + arguments: --scenario json_fasthttp --property scenario=JsonFastHttp $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 4 # once every 10 half-days - displayName: Fortunes FastHttp - arguments: --scenario fortunes_fasthttp --property scenario=FortunesFastHttp + arguments: --scenario fortunes_fasthttp --property scenario=FortunesFastHttp $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 5 # once every 10 half-days - displayName: Json Vertx - arguments: --scenario json_vertx --property scenario=JsonVertx + arguments: --scenario json_vertx --property scenario=JsonVertx $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 8 # once every 10 half-days - displayName: Fortunes Vertx - arguments: --scenario fortunes_vertx --property scenario=FortunesVertx + arguments: --scenario fortunes_vertx --property scenario=FortunesVertx $(containerDatabaseBuildReuse) condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days # Json NTex and Fortunes NTex are disabled: their docker builds @@ -145,6 +145,6 @@ steps: { "name": "crank", "condition": "(${{ parameters.condition }}) && (${{ scenario.condition }})", - "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) $(buildReuseProfile) --profile containers-build-reuse --variable buildCacheId=$(Build.BuildId) --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + "args": [ "--session $(session) $(containerMatrixJobs) ${{ scenario.arguments }} ${{ size.arguments }} ${{ rate.arguments }} ${{ parameters.arguments }} $(azureProfile) --application.options.reuseBuild true --application.buildKey containers-build-{{job.Source.DockerImageName}}-$(Build.BuildId) --application.source.sourceKey containers-source-{{job.Source.DockerImageName}}-$(Build.BuildId) --application.onConfigure \"job.DockerImageName += '-$(Build.BuildId)';\" --no-metadata --no-measurements --load.options.reuseBuild true --command-line-property --table ContainerBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] } diff --git a/build/grpc-scenarios.yml b/build/grpc-scenarios.yml index ed4f0fee9..1b52dbf6c 100644 --- a/build/grpc-scenarios.yml +++ b/build/grpc-scenarios.yml @@ -26,14 +26,22 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + serverBuildKey: aspnetcore + clientBuildKey: dotnet - displayName: "ASP.NET Core - Client: Go" - arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=go + arguments: --scenario grpcaspnetcoreserver-grpcgoclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcgoclient --application.options.collectCounters true + serverBuildKey: aspnetcore + clientBuildKey: go - displayName: "Go - Client: Grpc.Net.Client" - arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient --variable serverBuildKey=go --variable clientBuildKey=dotnet + arguments: --scenario grpcgoserver-grpcnetclient $(grpcJobs) --property server=grpcgoserver --property client=grpcnetclient + serverBuildKey: go + clientBuildKey: dotnet - displayName: "Go - Client: Go" - arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient --variable serverBuildKey=go --variable clientBuildKey=go + arguments: --scenario grpcgoserver-grpcgoclient $(grpcJobs) --property server=grpcgoserver --property client=grpcgoclient + serverBuildKey: go + clientBuildKey: go # H2Load scenarios - name: h2loadscenarios @@ -41,9 +49,13 @@ parameters: default: - displayName: "ASP.NET Core - Client: h2load" - arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=h2load + arguments: --scenario grpcaspnetcoreserver-h2loadclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=h2loadclient --application.options.collectCounters true + serverBuildKey: aspnetcore + clientBuildKey: h2load - displayName: "Go - Client: h2load" - arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient --variable serverBuildKey=go --variable clientBuildKey=h2load + arguments: --scenario grpcgoserver-h2loadclient $(grpcJobs) --property server=grpcgoserver --property client=h2loadclient + serverBuildKey: go + clientBuildKey: h2load # grpc-net only scenarios - name: grpcnetscenarios @@ -51,7 +63,9 @@ parameters: default: - displayName: "ASP.NET Core - Client: Grpc.Net.Client" - arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true --variable serverBuildKey=aspnetcore --variable clientBuildKey=dotnet + arguments: --scenario grpcaspnetcoreserver-grpcnetclient $(grpcJobs) --property server=grpcaspnetcoreserver --property client=grpcnetclient --application.options.collectCounters true + serverBuildKey: aspnetcore + clientBuildKey: dotnet - name: callTypes type: object @@ -107,7 +121,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] + "args": [ "--application.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ stream.arguments }} --application.options.reuseBuild true --application.buildKey grpc-server-build-${{ s.serverBuildKey }}-{{job.Framework}}-$(Build.BuildId) --application.source.sourceKey grpc-server-source-${{ s.serverBuildKey }}-$(Build.BuildId) --application.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\" --load.options.reuseBuild true --load.buildKey grpc-client-build-${{ s.clientBuildKey }}-{{job.Framework}}-$(Build.BuildId) --load.source.sourceKey grpc-client-source-${{ s.clientBuildKey }}-$(Build.BuildId) --load.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\" --variable protocol=h2c --property protocol=h2c --property scenario=unary --property payload=0 --variable body=AAAAAAcKBVdvcmxk --variable path=/grpc.testing.BenchmarkService/UnaryCall" ] } # h2/h3 HttpClient to ASP.NET Core @@ -132,7 +146,7 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId)" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --no-metadata --no-measurements ${{ parameters.arguments }} ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} ${{ protocol.arguments }} --application.options.reuseBuild true --application.buildKey grpc-server-build-${{ s.serverBuildKey }}-{{job.Framework}}-$(Build.BuildId) --application.source.sourceKey grpc-server-source-${{ s.serverBuildKey }}-$(Build.BuildId) --application.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\" --load.options.reuseBuild true --load.buildKey grpc-client-build-${{ s.clientBuildKey }}-{{job.Framework}}-$(Build.BuildId) --load.source.sourceKey grpc-client-source-${{ s.clientBuildKey }}-$(Build.BuildId) --load.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\"" ] } # h2c between various clients and servers @@ -154,5 +168,5 @@ steps: { "condition": "(${{ parameters.condition }}) && ((new Date().getUTCHours() - 7 + 24) % 24 >= 13)", "name": "crank", - "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} $(buildReuseProfile) --profile grpc-build-reuse --variable buildCacheId=$(Build.BuildId) --variable protocol=h2c --property protocol=h2c" ] + "args": [ "--application.framework net11.0 --load.framework net11.0 --command-line-property --table GrpcBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} --no-metadata --no-measurements ${{ s.arguments }} ${{ callType.arguments }} ${{ payload.arguments }} ${{ stream.arguments }} --application.options.reuseBuild true --application.buildKey grpc-server-build-${{ s.serverBuildKey }}-{{job.Framework}}-$(Build.BuildId) --application.source.sourceKey grpc-server-source-${{ s.serverBuildKey }}-$(Build.BuildId) --application.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\" --load.options.reuseBuild true --load.buildKey grpc-client-build-${{ s.clientBuildKey }}-{{job.Framework}}-$(Build.BuildId) --load.source.sourceKey grpc-client-source-${{ s.clientBuildKey }}-$(Build.BuildId) --load.onConfigure \"if (job.DockerImageName) job.DockerImageName += '-$(Build.BuildId)';\" --variable protocol=h2c --property protocol=h2c" ] } diff --git a/build/httpclient-scenarios.yml b/build/httpclient-scenarios.yml index c5ce3c8b9..d142ffab4 100644 --- a/build/httpclient-scenarios.yml +++ b/build/httpclient-scenarios.yml @@ -94,7 +94,7 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ response.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ response.arguments }} --server.options.reuseBuild true --server.buildKey httpclient-server-build-{{job.Framework}}-$(Build.BuildId) --server.source.sourceKey httpclient-source-$(Build.BuildId) --client.options.reuseBuild true --client.buildKey httpclient-client-build-{{job.Framework}}-$(Build.BuildId) --client.source.sourceKey httpclient-source-$(Build.BuildId)" ] } # POST scenarios @@ -116,5 +116,5 @@ steps: { "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0)", "name": "crank", - "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }} $(buildReuseProfile) --profile httpclient-build-reuse --variable buildCacheId=$(Build.BuildId)" ] + "args": [ "--description \"${{ version.displayName }} ${{ s.displayName }} ${{ content.displayName }} ${{ ct.displayName }} $(System.JobDisplayName)\" ${{ parameters.tfm }} --command-line-property --table HttpClientBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) ${{ parameters.arguments }} ${{ parameters.httpClientJobs }} --no-metadata --no-measurements ${{ s.arguments }} ${{ version.arguments }} ${{ ct.arguments }} ${{ content.arguments }} --server.options.reuseBuild true --server.buildKey httpclient-server-build-{{job.Framework}}-$(Build.BuildId) --server.source.sourceKey httpclient-source-$(Build.BuildId) --client.options.reuseBuild true --client.buildKey httpclient-client-build-{{job.Framework}}-$(Build.BuildId) --client.source.sourceKey httpclient-source-$(Build.BuildId)" ] } diff --git a/build/job-variables.yml b/build/job-variables.yml index 798dfda33..4ee0a9530 100644 --- a/build/job-variables.yml +++ b/build/job-variables.yml @@ -77,8 +77,12 @@ variables: value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml - name: ciProfile value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml -- name: buildReuseProfile - value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/build-reuse.profile.yml +- name: containerDatabaseBuildReuse + value: >- + --db.options.reuseBuild true + --db.buildKey containers-build-{{job.Source.DockerImageName}}-$(Build.BuildId) + --db.source.sourceKey containers-source-{{job.Source.DockerImageName}}-$(Build.BuildId) + --db.onConfigure \"job.DockerImageName += '-$(Build.BuildId)';\" - name: grpcJobs value: --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/grpc.benchmarks.yml - name: efcoreJobs diff --git a/build/signalr-scenarios.yml b/build/signalr-scenarios.yml index 72111840e..5d7707ef2 100644 --- a/build/signalr-scenarios.yml +++ b/build/signalr-scenarios.yml @@ -74,5 +74,5 @@ steps: "condition": "${{ parameters.condition }}", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile signalr-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] + "args": [ "${{ s.arguments }} ${{ transport.arguments }} ${{ protocol.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --warmup.framework net11.0 --application.framework net11.0 --application.options.reuseBuild true --application.buildKey signalr-server-build-{{job.Framework}}-$(Build.BuildId) --application.source.sourceKey signalr-source-$(Build.BuildId) --warmup.options.reuseBuild true --warmup.buildKey signalr-client-build-{{job.Framework}}-$(Build.BuildId) --warmup.source.sourceKey signalr-source-$(Build.BuildId) --load.buildKey signalr-client-build-{{job.Framework}}-$(Build.BuildId) --load.source.sourceKey signalr-source-$(Build.BuildId) --session $(session) --command-line-property --table SignalRBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata --warmup.options.discardResults true" ] } diff --git a/build/tests/test_build_reuse.py b/build/tests/test_build_reuse.py deleted file mode 100644 index acd0d4877..000000000 --- a/build/tests/test_build_reuse.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Validate CI cache profiles with Crank's configuration-only debug mode. - -Run from the repo root: python -m unittest discover -s build/tests -Requires the existing crank CLI; no agents, builds, or network access are used. -""" - -import json -from pathlib import Path -import re -import shutil -import subprocess -import tempfile -import unittest - - -ROOT = Path(__file__).resolve().parents[2] -CRANK = shutil.which("crank") -PROFILES = { - "containers": ("application", "db"), - "grpc": ("application", "load"), - "httpclient": ("server", "client"), - "signalr": ("application", "warmup", "load"), - "websocket": ("application", "load"), -} - - -class TestTemplateWiring(unittest.TestCase): - def test_all_messages_enable_run_scoped_reuse(self): - for suite in PROFILES: - with self.subTest(suite=suite): - text = (ROOT / "build" / f"{suite}-scenarios.yml").read_text( - encoding="utf-8-sig" - ) - messages = [ - line for line in text.splitlines() if '"args": [' in line - ] - self.assertTrue(messages) - for message in messages: - self.assertIn("$(buildReuseProfile)", message) - self.assertIn(f"--profile {suite}-build-reuse", message) - self.assertIn( - "--variable buildCacheId=$(Build.BuildId)", message - ) - - def test_grpc_scenarios_identify_both_build_variants(self): - text = (ROOT / "build" / "grpc-scenarios.yml").read_text() - scenarios = re.findall( - r"arguments: --scenario (grpc\S+)([^\n]+)", text - ) - self.assertEqual(len(scenarios), 7) - for scenario, arguments in scenarios: - with self.subTest(scenario=scenario): - server, client = scenario.split("-") - server_key = "aspnetcore" if "aspnetcore" in server else "go" - client_key = { - "grpcnetclient": "dotnet", - "grpcgoclient": "go", - "h2loadclient": "h2load", - }[client] - self.assertIn(f"serverBuildKey={server_key}", arguments) - self.assertIn(f"clientBuildKey={client_key}", arguments) - - def test_container_dockerfile_variants_have_distinct_tags(self): - text = (ROOT / "scenarios" / "containers.benchmarks.yml").read_text() - variants = re.findall( - r"dockerFile: (docker/container-matrix/\S+)\n" - r"\s+dockerImageName: (\S+)", text - ) - self.assertEqual(len(variants), 7) - tags = {} - for dockerfile, tag in variants: - self.assertEqual(tags.setdefault(tag, dockerfile), dockerfile) - self.assertEqual(len(tags), 4) - - -@unittest.skipUnless(CRANK, "Requires the crank CLI") -class TestBuildReuseProfiles(unittest.TestCase): - def configure(self, suite, run_id="123", variables=None, patches=None, error=None): - roles = PROFILES[suite] - if suite == "containers": - roles = (*roles, "load") - jobs = { - role: { - "source": { - "repository": "https://github.com/aspnet/Benchmarks.git", - "branchOrCommit": "main", - "project": f"src/{role}/App.csproj", - }, - "framework": "net11.0", - } - for role in roles - } - if suite == "containers": - for role in PROFILES[suite]: - jobs[role]["source"] = { - "repository": "https://github.com/TechEmpower/FrameworkBenchmarks.git", - "branchOrCommit": "master", - "dockerFile": f"{role}/Dockerfile", - "dockerImageName": role, - } - if suite == "signalr": - jobs["warmup"]["source"]["project"] = "src/load/App.csproj" - for role, patch in (patches or {}).items(): - jobs[role].update(patch) - - scenario = {role: {"job": role} for role in roles} - if suite == "httpclient": - scenario["client"]["agent"] = "main" - scenario["server"]["agent"] = "secondary" - config = { - "jobs": jobs, - "profiles": { - "machines": { - "agents": { - "application": {"endpoints": [], "aliases": ["main"]}, - "load": { - "endpoints": [], - "aliases": ["warmup", "secondary"], - }, - "db": {"endpoints": []}, - } - } - }, - "scenarios": {"test": scenario}, - } - values = { - "compress": "false", - "serverBuildKey": "aspnetcore", - "clientBuildKey": "dotnet", - } - if run_id is not None: - values["buildCacheId"] = run_id - values.update(variables or {}) - with tempfile.TemporaryDirectory() as temp: - source = Path(temp) / "scenario.json" - output = Path(temp) / "debug.json" - source.write_text(json.dumps(config), encoding="utf-8") - command = [ - CRANK, - "--config", str(source), - "--config", str(ROOT / "build" / "build-reuse.profile.yml"), - "--scenario", "test", - "--profile", "machines", - "--profile", f"{suite}-build-reuse", - "--debug", str(output), - ] - for key, value in values.items(): - command.extend(("--variable", f"{key}={value}")) - result = subprocess.run( - command, capture_output=True, text=True, timeout=60 - ) - if run_id is None or not str(run_id).isdigit(): - error = "buildCacheId must be set" - if error is not None: - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertIn(error, result.stdout + result.stderr) - return None - self.assertEqual( - result.returncode, 0, result.stdout + result.stderr - ) - return json.loads(output.read_text(encoding="utf-8"))["Jobs"] - - def assert_cached(self, jobs, suite): - for role in PROFILES[suite]: - with self.subTest(suite=suite, role=role): - job = jobs[role] - self.assertTrue(job["Options"]["ReuseBuild"]) - self.assertTrue(job["NoBuild"]) - self.assertTrue(job["BuildKey"].endswith("-123")) - source_key = job["Source"]["SourceKey"] - self.assertTrue(source_key.endswith("-123")) - self.assertNotEqual(job["BuildKey"], source_key) - - def test_each_profile_invalidates_sources_and_builds_on_new_run(self): - for suite in PROFILES: - with self.subTest(suite=suite): - first = self.configure(suite) - self.assert_cached(first, suite) - second = self.configure(suite, "124") - for role in PROFILES[suite]: - self.assertNotEqual( - first[role]["BuildKey"], second[role]["BuildKey"] - ) - self.assertNotEqual( - first[role]["Source"]["SourceKey"], - second[role]["Source"]["SourceKey"], - ) - - def test_missing_or_invalid_run_id_is_rejected(self): - for suite in PROFILES: - for run_id in (None, "not-a-build"): - with self.subTest(suite=suite, run_id=run_id): - self.configure(suite, run_id) - - def test_missing_or_invalid_build_variants_are_rejected(self): - for key, value in ( - ("serverBuildKey", ""), - ("clientBuildKey", "../invalid"), - ): - with self.subTest(key=key): - self.configure( - "grpc", - variables={key: value}, - error="serverBuildKey and clientBuildKey must identify", - ) - self.configure( - "websocket", - variables={"compress": ""}, - error="compress must be true or false", - ) - - def test_container_runtime_limits_and_rate_do_not_change_cache(self): - first = self.configure("containers", variables={"rate": "1000"}) - second = self.configure( - "containers", - variables={"rate": "10000"}, - patches={ - "application": { - "cpuLimitRatio": 0.25, - "memoryLimitInBytes": 256000000, - "cpuSet": "0", - } - }, - ) - for role in PROFILES["containers"]: - self.assertEqual(first[role]["BuildKey"], second[role]["BuildKey"]) - self.assertEqual( - first[role]["Source"]["SourceKey"], - second[role]["Source"]["SourceKey"], - ) - - def test_container_image_variants_have_separate_caches(self): - first = self.configure("containers") - second = self.configure( - "containers", - patches={ - "application": { - "source": { - "repository": "https://github.com/aspnet/Benchmarks.git", - "branchOrCommit": "main", - "dockerFile": "docker/container-matrix/Middleware-alpine.dockerfile", - "dockerImageName": "app-alpine", - } - } - }, - ) - self.assertNotEqual( - first["application"]["BuildKey"], second["application"]["BuildKey"] - ) - self.assertNotEqual( - first["application"]["Source"]["SourceKey"], - second["application"]["Source"]["SourceKey"], - ) - self.assertEqual(first["db"]["BuildKey"], second["db"]["BuildKey"]) - - def test_websocket_compression_builds_are_separate(self): - plain = self.configure("websocket") - compressed = self.configure( - "websocket", - variables={"compress": "true"}, - patches={"application": {"buildArguments": ["/p:UseCompression=true"]}}, - ) - self.assertNotEqual( - plain["application"]["BuildKey"], - compressed["application"]["BuildKey"], - ) - self.assertEqual(plain["load"]["BuildKey"], compressed["load"]["BuildKey"]) - - def test_signalr_warmup_and_load_share_client_build(self): - jobs = self.configure("signalr") - self.assertEqual(jobs["warmup"]["BuildKey"], jobs["load"]["BuildKey"]) - self.assertNotEqual(jobs["application"]["BuildKey"], jobs["load"]["BuildKey"]) - - def test_grpc_server_and_client_variants_are_separate(self): - dotnet = self.configure("grpc") - go = self.configure( - "grpc", variables={"serverBuildKey": "go", "clientBuildKey": "go"} - ) - h2load = self.configure("grpc", variables={"clientBuildKey": "h2load"}) - self.assertNotEqual( - dotnet["application"]["BuildKey"], go["application"]["BuildKey"] - ) - self.assertEqual( - dotnet["application"]["BuildKey"], h2load["application"]["BuildKey"] - ) - self.assertEqual( - len({jobs["load"]["BuildKey"] for jobs in (dotnet, go, h2load)}), 3 - ) - - def test_framework_versions_are_separate(self): - first = self.configure("httpclient") - second = self.configure( - "httpclient", - patches={"client": {"framework": "net10.0"}}, - ) - self.assertNotEqual(first["client"]["BuildKey"], second["client"]["BuildKey"]) - self.assertEqual(first["server"]["BuildKey"], second["server"]["BuildKey"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/build/websocket-scenarios.yml b/build/websocket-scenarios.yml index b48d64408..8261bd2a5 100644 --- a/build/websocket-scenarios.yml +++ b/build/websocket-scenarios.yml @@ -65,5 +65,5 @@ steps: "condition": "(${{ parameters.condition }})", "retries": 1, "name": "crank", - "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 $(buildReuseProfile) --profile websocket-build-reuse --variable buildCacheId=$(Build.BuildId) --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] + "args": [ "${{ s.arguments }} ${{ size.arguments }} ${{ compress.arguments }} --application.options.collectCounters true --load.options.collectCounters true ${{ parameters.arguments }} --load.framework net11.0 --application.framework net11.0 --application.options.reuseBuild true --application.buildKey websocket-server-build-{{job.Framework}}-{{compress}}-$(Build.BuildId) --application.source.sourceKey websocket-source-$(Build.BuildId) --load.buildKey websocket-client-build-{{job.Framework}}-$(Build.BuildId) --load.source.sourceKey websocket-source-$(Build.BuildId) --session $(session) --command-line-property --table WebsocketBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --no-measurements --load.options.reuseBuild true --no-metadata " ] }