From 3887aead06e366e61b6c4c2e88e70dca41991d88 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 27 Sep 2026 13:37:52 +0000 Subject: [PATCH 1/2] roundtrip: shard the upgrade property test with MXCLI_UPGRADE_SHARD Round-robin over the scripts that reach execution; every shard still upgrades and checks every script. TestShardsPartition proves the shards together execute each script exactly once. Locally 75+74+75 = 224, the unsharded count. Part of #757. Co-Authored-By: Claude Opus 5.5 --- mdl/roundtrip/shard_test.go | 65 ++++++++++++++++++++++++++ mdl/roundtrip/upgrade_property_test.go | 33 +++++++++++-- mdl/roundtrip/upgrade_shard_test.go | 35 ++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 mdl/roundtrip/shard_test.go create mode 100644 mdl/roundtrip/upgrade_shard_test.go diff --git a/mdl/roundtrip/shard_test.go b/mdl/roundtrip/shard_test.go new file mode 100644 index 000000000..4d18c3487 --- /dev/null +++ b/mdl/roundtrip/shard_test.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +package roundtrip + +import ( + "fmt" + "testing" +) + +// TestParseShard pins the MXCLI_UPGRADE_SHARD spelling CI passes. +func TestParseShard(t *testing.T) { + for _, tc := range []struct { + in string + i, n int + bad bool + }{ + {in: "", i: 1, n: 1}, + {in: "1/1", i: 1, n: 1}, + {in: "2/3", i: 2, n: 3}, + {in: " 3/3 ", i: 3, n: 3}, + {in: "0/3", bad: true}, + {in: "4/3", bad: true}, + {in: "1/0", bad: true}, + {in: "3", bad: true}, + {in: "a/b", bad: true}, + {in: "1/2/3", bad: true}, + } { + s, err := parseShard(tc.in) + if tc.bad { + if err == nil { + t.Errorf("parseShard(%q) = %+v, want an error", tc.in, s) + } + continue + } + if err != nil || s.index != tc.i || s.count != tc.n { + t.Errorf("parseShard(%q) = %+v, %v; want %d/%d", tc.in, s, err, tc.i, tc.n) + } + } +} + +// TestShardsPartition: CI runs the shards as separate jobs and never the +// unsharded test, so the shards together must execute every script exactly +// once. A script in no shard would silently leave per-PR CI. +func TestShardsPartition(t *testing.T) { + for n := 1; n <= 7; n++ { + const items = 101 + seen := make([]int, items) + for i := 1; i <= n; i++ { + s, err := parseShard(fmt.Sprintf("%d/%d", i, n)) + if err != nil { + t.Fatal(err) + } + for k := 0; k < items; k++ { + if s.has(k) { + seen[k]++ + } + } + } + for k, c := range seen { + if c != 1 { + t.Fatalf("n=%d: item %d is in %d shards, want exactly 1", n, k, c) + } + } + } +} diff --git a/mdl/roundtrip/upgrade_property_test.go b/mdl/roundtrip/upgrade_property_test.go index 0ef29a964..fe508c0fc 100644 --- a/mdl/roundtrip/upgrade_property_test.go +++ b/mdl/roundtrip/upgrade_property_test.go @@ -56,7 +56,12 @@ import ( // Executing the whole corpus takes about 15 minutes, which on its own exceeds // what the CI integration step has left (ako/mxcli#742 timed out there). // MXCLI_UPGRADE_ALL=1 executes every script, header-only ones included; run it -// when a langver.Change or a gated rewrite lands. +// when a langver.Change or a gated rewrite lands. The nightly workflow runs it. +// +// MXCLI_UPGRADE_SHARD=i/n executes only every n-th executable script, starting +// at the i-th, so per-PR CI can split the executions across parallel jobs +// (ako/mxcli#757); the shards together execute each script exactly once +// (TestShardsPartition). Unset is the whole set. func TestUpgradeExecutesToTheSameModel(t *testing.T) { a, b := newHarness(t), newHarness(t) defer a.close() @@ -64,7 +69,12 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) { scripts := upgradeExampleScripts(t) all := os.Getenv("MXCLI_UPGRADE_ALL") != "" - var same, outOfScope, unparsed, headerOnly, keptVersion []string + sh, err := parseShard(os.Getenv("MXCLI_UPGRADE_SHARD")) + if err != nil { + t.Fatal(err) + } + executable := 0 // scripts that reached execution, in every shard + var same, outOfScope, unparsed, headerOnly, keptVersion, otherShard []string for _, path := range scripts { src, err := os.ReadFile(path) if err != nil { @@ -100,6 +110,14 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) { headerOnly = append(headerOnly, rel) return } + // Every shard upgrades and checks every script above, which is + // cheap; only the execution below, which is not, is split. + k := executable + executable++ + if !sh.has(k) { + otherShard = append(otherShard, rel) + return + } errA, errB, diff := executeBoth(t, a, b, string(src), res.Source) if len(diff) > 0 { @@ -120,6 +138,10 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) { } t.Logf("%d scripts execute on PedApp and upgrade to the same model", len(same)) + if sh.count > 1 { + t.Logf("shard %s: %d of %d executable scripts belong to other shards and were not executed here", + sh, len(otherShard), executable) + } t.Logf("%d scripts are out of scope: their original does not execute cleanly on PedApp:\n %s", len(outOfScope), strings.Join(outOfScope, "\n ")) if !all { @@ -130,10 +152,11 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) { len(keptVersion), strings.Join(keptVersion, "\n ")) t.Logf("%d scripts do not parse (negative tests) and cannot be upgraded:\n %s", len(unparsed), strings.Join(unparsed, "\n ")) - if os.Getenv("MXCLI_UPGRADE_EXAMPLES") == "" && len(same) < 50 { + if floor := 50 / sh.count; os.Getenv("MXCLI_UPGRADE_EXAMPLES") == "" && len(same) < floor { // Hundreds execute today; a handful means the harness broke, and a - // property checked on nothing passes. - t.Errorf("only %d scripts executed cleanly — the harness is not exercising the property", len(same)) + // property checked on nothing passes. A shard sees 1/n of them. + t.Errorf("only %d scripts executed cleanly (shard %s) — the harness is not exercising the property", + len(same), sh) } } diff --git a/mdl/roundtrip/upgrade_shard_test.go b/mdl/roundtrip/upgrade_shard_test.go new file mode 100644 index 000000000..69778d2d0 --- /dev/null +++ b/mdl/roundtrip/upgrade_shard_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package roundtrip + +import ( + "fmt" + "strconv" + "strings" +) + +// shard selects every count-th item starting at index-1: MXCLI_UPGRADE_SHARD +// ("i/n", 1-based) splits the upgrade property test's executions across CI +// jobs (ako/mxcli#757). Round-robin rather than contiguous ranges, because +// the corpus is sorted by path and neighbouring scripts cost alike. +type shard struct{ index, count int } + +// parseShard reads "i/n"; empty is the whole set, 1/1. +func parseShard(s string) (shard, error) { + s = strings.TrimSpace(s) + if s == "" { + return shard{1, 1}, nil + } + a, b, ok := strings.Cut(s, "/") + i, errI := strconv.Atoi(a) + n, errN := strconv.Atoi(b) + if !ok || errI != nil || errN != nil || n < 1 || i < 1 || i > n { + return shard{}, fmt.Errorf("MXCLI_UPGRADE_SHARD=%q: want i/n with 1 <= i <= n", s) + } + return shard{i, n}, nil +} + +// has reports whether the k-th item (0-based) belongs to this shard. +func (s shard) has(k int) bool { return k%s.count == s.index-1 } + +func (s shard) String() string { return fmt.Sprintf("%d/%d", s.index, s.count) } From 4cca3ad14859f5ef92299c009fda133ec89b8c9c Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 27 Sep 2026 13:37:52 +0000 Subject: [PATCH 2/2] ci: split integration tests into parallel suites; full upgrade corpus nightly The single make test-integration step took ~22 of its 30 minutes, its wall time being mdl/roundtrip alone. Per-PR CI now runs executor, roundtrip, upgrade (3 shards) and other as parallel jobs, with an integration-passed aggregate. Nightly raises its step to 60 min and runs MXCLI_UPGRADE_ALL in 3 shards once. Closes #757. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 44 ++++++++++++++- .github/workflows/push-test.yml | 97 ++++++++++++++++++++++++++++++--- Makefile | 44 ++++++++++++++- 3 files changed, 174 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 14a26cb78..d30612c0c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -54,9 +54,51 @@ jobs: - name: Setup mxbuild ${{ matrix.mendix-version }} run: ./bin/mxcli setup mxbuild --version ${{ matrix.mendix-version }} + # One step, every package: go test runs the packages in parallel, so the + # wall time is the slowest one's (mdl/roundtrip, ~21 min on this runner in + # ako/mxcli#757's measurement) and grows as the round trip gains fixtures. + # Per-PR CI splits this into parallel jobs instead (push-test.yml). - name: "Integration tests (Mendix ${{ matrix.mendix-version }})" run: make test-integration - timeout-minutes: 30 + timeout-minutes: 60 + + # The upgrade property test over the WHOLE mdl-examples corpus, header-only + # scripts included (MXCLI_UPGRADE_ALL). Per-PR CI executes only the scripts + # the upgrade rewrites beyond the header and terminators; this is the run that + # proves langver's gating on the rest (ako/mxcli#757). Independent of the + # Mendix version — it runs on the committed PedApp fixture and never calls + # mx — so it runs once, not per matrix entry. It does not gate the release. + upgrade-full-corpus: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: ['1/3', '2/3', '3/3'] + name: upgrade full corpus (${{ matrix.shard }}) + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.26.6' + - name: Cache ANTLR4 JAR + uses: actions/cache@v6 + with: + path: ~/.m2/repository/org/antlr/antlr4 + key: antlr4-4.13.2 + - name: Install ANTLR4 + run: pip install 'antlr4-tools==0.2.2' + - name: Generate parser + run: make grammar + env: + ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2' + - name: Build + run: make build + - name: Upgrade property test, full corpus (${{ matrix.shard }}) + run: make test-integration-upgrade + env: + MXCLI_UPGRADE_ALL: '1' + MXCLI_UPGRADE_SHARD: ${{ matrix.shard }} + timeout-minutes: 45 nightly: needs: test diff --git a/.github/workflows/push-test.yml b/.github/workflows/push-test.yml index 262c5d23f..1d5f323fe 100644 --- a/.github/workflows/push-test.yml +++ b/.github/workflows/push-test.yml @@ -167,17 +167,98 @@ jobs: run: ./scripts/check-skill-mdl.sh ./bin/mxcli .claude/skills/mendix - name: Check docs-site MDL blocks run: ./scripts/check-skill-mdl.sh ./bin/mxcli docs-site/src - - name: Setup mxbuild - run: ./bin/mxcli setup mxbuild --version 11.12.2 - # One engine since the legacy sdk/mpr backend was deleted - # (docs/plans/2026-09-14-retire-legacy-engine.md), so there is no matrix to - # narrow here any more and MXCLI_TEST_ENGINES is left unset. - - name: Integration tests - run: make test-integration - timeout-minutes: 30 + # The integration tests are the `integration` job below. - name: Lint Go run: make lint-go - name: Vulnerability scan run: | go install golang.org/x/vuln/cmd/govulncheck@latest govulncheck ./... + + # The integration tests, split into suites that run as parallel jobs + # (ako/mxcli#757). As one `make test-integration` step they took ~22 of its 30 + # minutes once the upgrade property test landed, and the step's wall time was + # mdl/roundtrip's alone (~21 min) — go test already ran the packages in + # parallel, so no timeout below 30 min was going to hold as the round trip + # grows (#743 adds TestApp to it). + # + # Measured per package on the ubuntu runner (run 36319810164) and what each + # suite is expected to take, including ~3 min of checkout/build/mxbuild: + # executor mdl/executor ~15.5 min -> ~19 min + # roundtrip mdl/roundtrip, round-trip laws ~1.7 min -> ~4 min + # upgrade upgrade property test, 3 shards ~19 min -> ~10 min each + # other cmd/mxcli{,/docker,/marketplace} ~5.7 min -> ~8 min + # The full-corpus upgrade run (MXCLI_UPGRADE_ALL) is nightly, not here. + # + # Nothing here is dropped from per-PR CI: the round-trip laws and the + # execute-both upgrade test run at their default scope, the shards together + # executing every script exactly once (TestShardsPartition). Only the unit + # tests of packages WITHOUT integration tests are no longer re-run under the + # tag — `make test` in build-and-test runs those. + integration: + name: integration (${{ matrix.suite }}${{ matrix.shard && format(' {0}', matrix.shard) || '' }}) + runs-on: ubuntu-latest + # Each suite's go test has its own -timeout (Makefile); this is the job's + # backstop, with room for the setup steps. + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - suite: executor + mxbuild: true + - suite: roundtrip + - suite: upgrade + shard: 1/3 + - suite: upgrade + shard: 2/3 + - suite: upgrade + shard: 3/3 + - suite: other + mxbuild: true + steps: + # #743 adds TestApp as a git submodule to the round trip; its checkout + # will need `submodules: true` here for the roundtrip suite. + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.26.6' + - name: Cache ANTLR4 JAR + uses: actions/cache@v6 + with: + path: ~/.m2/repository/org/antlr/antlr4 + key: antlr4-4.13.2 + - name: Install ANTLR4 + run: pip install 'antlr4-tools==0.2.2' + - name: Generate parser + run: make grammar + env: + ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2' + - name: Build + run: make build + # mdl/roundtrip runs on the committed PedApp fixture and never calls mx. + # One engine since the legacy sdk/mpr backend was deleted + # (docs/plans/2026-09-14-retire-legacy-engine.md), so MXCLI_TEST_ENGINES + # is left unset. + - name: Setup mxbuild + if: matrix.mxbuild + run: ./bin/mxcli setup mxbuild --version 11.12.2 + - name: Integration tests (${{ matrix.suite }}) + run: make test-integration-${{ matrix.suite }} + env: + MXCLI_UPGRADE_SHARD: ${{ matrix.shard }} + timeout-minutes: 30 + + # One stable check name for "every integration suite passed", whatever the + # matrix above becomes — a branch rule can require this instead of each leg. + integration-passed: + if: always() + needs: integration + runs-on: ubuntu-latest + steps: + - name: All integration suites passed + run: | + if [ "${{ needs.integration.result }}" != "success" ]; then + echo "integration: ${{ needs.integration.result }}" + exit 1 + fi diff --git a/Makefile b/Makefile index ba84eece8..8a9792fd1 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ GO_BUILD_FLAGS = -trimpath # Clean version for VS Code extension (must be valid semver: major.minor.patch) VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0") -.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet +.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions test-integration test-integration-executor test-integration-roundtrip test-integration-upgrade test-integration-other grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache) # Usage: $(call copy-if-changed,src,dst) @@ -313,7 +313,47 @@ check-tunnel-deps: # therefore left unset everywhere; it survives only so that a stale # `MXCLI_TEST_ENGINES=legacy` is fatal rather than silently selecting nothing. test-integration: - CGO_ENABLED=0 go test -tags integration -count=1 -timeout 30m ./... + CGO_ENABLED=0 go test -tags integration -count=1 -timeout 60m ./... + +# The same integration tests split into suites that CI runs as parallel jobs +# (ako/mxcli#757): one step running them all spent ~22 of its 30 minutes once +# the upgrade property test landed, with mdl/roundtrip alone the critical path. +# +# Unlike test-integration, the suites run only the packages that HAVE +# integration-tagged tests. Every other package's tests are identical with and +# without the tag, and `make test` already runs them. +# +# test-integration-executor mdl/executor: doctype scripts through exec + mx check +# test-integration-roundtrip mdl/roundtrip: the round-trip laws, the upgrade controls +# test-integration-upgrade mdl/roundtrip: the upgrade property test alone; +# MXCLI_UPGRADE_SHARD=i/n runs one of n shards, +# MXCLI_UPGRADE_ALL=1 the full corpus (nightly) +# test-integration-other every other package with integration tests, +# found by build tag so a new one cannot be missed +# (hidden dirs are skipped: nested worktrees live there) +# +# `-skip` / `-run` split mdl/roundtrip by test name, so a new round-trip test +# lands in the roundtrip suite without editing this file. +INTEGRATION_PKGS = $(shell grep -rl --include='*_test.go' --exclude-dir='.?*' --exclude-dir=reference --exclude-dir=mx-test-projects \ + '^//go:build.*integration' . | xargs -n1 dirname | sed 's|^\./||; s|^|./|' | sort -u) +INTEGRATION_SPLIT = ./mdl/executor ./mdl/roundtrip +UPGRADE_PROPERTY = ^TestUpgradeExecutesToTheSameModel$$ +INTEGRATION_GO_TEST = CGO_ENABLED=0 go test -tags integration -count=1 + +test-integration-executor: + $(INTEGRATION_GO_TEST) -timeout 40m ./mdl/executor/ + +test-integration-roundtrip: + $(INTEGRATION_GO_TEST) -timeout 40m -skip '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/ + +test-integration-upgrade: + $(INTEGRATION_GO_TEST) -timeout 40m -run '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/ + +test-integration-other: + @pkgs="$(filter-out $(INTEGRATION_SPLIT),$(INTEGRATION_PKGS))"; \ + if [ -z "$$pkgs" ]; then echo "no integration packages outside the split suites"; exit 1; fi; \ + echo "integration packages: $$pkgs"; \ + $(INTEGRATION_GO_TEST) -timeout 40m $$pkgs # Run MDL integration tests (requires Docker and a Mendix project) # Usage: make test-mdl MPR=path/to/app.mpr