CI: split integration tests into parallel suites; shard the upgrade property test (#757)

.github/workflows/nightly.yml

@@ -54,9 +54,51 @@ jobs:
       - name: Setup mxbuild ${{ matrix.mendix-version }}
         run: ./bin/mxcli setup mxbuild --version ${{ matrix.mendix-version }}
 
+      # One step, every package: go test runs the packages in parallel, so the
+      # wall time is the slowest one's (mdl/roundtrip, ~21 min on this runner in
+      # ako/mxcli#757's measurement) and grows as the round trip gains fixtures.
+      # Per-PR CI splits this into parallel jobs instead (push-test.yml).
       - name: "Integration tests (Mendix ${{ matrix.mendix-version }})"
         run: make test-integration
-        timeout-minutes: 30
+        timeout-minutes: 60
+
+  # The upgrade property test over the WHOLE mdl-examples corpus, header-only
+  # scripts included (MXCLI_UPGRADE_ALL). Per-PR CI executes only the scripts
+  # the upgrade rewrites beyond the header and terminators; this is the run that
+  # proves langver's gating on the rest (ako/mxcli#757). Independent of the
+  # Mendix version — it runs on the committed PedApp fixture and never calls
+  # mx — so it runs once, not per matrix entry. It does not gate the release.
+  upgrade-full-corpus:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ['1/3', '2/3', '3/3']
+    name: upgrade full corpus (${{ matrix.shard }})
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.26.6'
+      - name: Cache ANTLR4 JAR
+        uses: actions/cache@v6
+        with:
+          path: ~/.m2/repository/org/antlr/antlr4
+          key: antlr4-4.13.2
+      - name: Install ANTLR4
+        run: pip install 'antlr4-tools==0.2.2'
+      - name: Generate parser
+        run: make grammar
+        env:
+          ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2'
+      - name: Build
+        run: make build
+      - name: Upgrade property test, full corpus (${{ matrix.shard }})
+        run: make test-integration-upgrade
+        env:
+          MXCLI_UPGRADE_ALL: '1'
+          MXCLI_UPGRADE_SHARD: ${{ matrix.shard }}
+        timeout-minutes: 45
 
   nightly:
     needs: test

.github/workflows/push-test.yml

@@ -167,17 +167,98 @@ jobs:
         run: ./scripts/check-skill-mdl.sh ./bin/mxcli .claude/skills/mendix
       - name: Check docs-site MDL blocks
         run: ./scripts/check-skill-mdl.sh ./bin/mxcli docs-site/src
-      - name: Setup mxbuild
-        run: ./bin/mxcli setup mxbuild --version 11.12.2
-      # One engine since the legacy sdk/mpr backend was deleted
-      # (docs/plans/2026-09-14-retire-legacy-engine.md), so there is no matrix to
-      # narrow here any more and MXCLI_TEST_ENGINES is left unset.
-      - name: Integration tests
-        run: make test-integration
-        timeout-minutes: 30
+      # The integration tests are the `integration` job below.
       - name: Lint Go
         run: make lint-go
       - name: Vulnerability scan
         run: |
           go install golang.org/x/vuln/cmd/govulncheck@latest
           govulncheck ./...
+
+  # The integration tests, split into suites that run as parallel jobs
+  # (ako/mxcli#757). As one `make test-integration` step they took ~22 of its 30
+  # minutes once the upgrade property test landed, and the step's wall time was
+  # mdl/roundtrip's alone (~21 min) — go test already ran the packages in
+  # parallel, so no timeout below 30 min was going to hold as the round trip
+  # grows (#743 adds TestApp to it).
+  #
+  # Measured per package on the ubuntu runner (run 36319810164) and what each
+  # suite is expected to take, including ~3 min of checkout/build/mxbuild:
+  #   executor   mdl/executor                          ~15.5 min -> ~19 min
+  #   roundtrip  mdl/roundtrip, round-trip laws         ~1.7 min -> ~4 min
+  #   upgrade    upgrade property test, 3 shards        ~19 min  -> ~10 min each
+  #   other      cmd/mxcli{,/docker,/marketplace}       ~5.7 min -> ~8 min
+  # The full-corpus upgrade run (MXCLI_UPGRADE_ALL) is nightly, not here.
+  #
+  # Nothing here is dropped from per-PR CI: the round-trip laws and the
+  # execute-both upgrade test run at their default scope, the shards together
+  # executing every script exactly once (TestShardsPartition). Only the unit
+  # tests of packages WITHOUT integration tests are no longer re-run under the
+  # tag — `make test` in build-and-test runs those.
+  integration:
+    name: integration (${{ matrix.suite }}${{ matrix.shard && format(' {0}', matrix.shard) || '' }})
+    runs-on: ubuntu-latest
+    # Each suite's go test has its own -timeout (Makefile); this is the job's
+    # backstop, with room for the setup steps.
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: executor
+            mxbuild: true
+          - suite: roundtrip
+          - suite: upgrade
+            shard: 1/3
+          - suite: upgrade
+            shard: 2/3
+          - suite: upgrade
+            shard: 3/3
+          - suite: other
+            mxbuild: true
+    steps:
+      # #743 adds TestApp as a git submodule to the round trip; its checkout
+      # will need `submodules: true` here for the roundtrip suite.
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.26.6'
+      - name: Cache ANTLR4 JAR
+        uses: actions/cache@v6
+        with:
+          path: ~/.m2/repository/org/antlr/antlr4
+          key: antlr4-4.13.2
+      - name: Install ANTLR4
+        run: pip install 'antlr4-tools==0.2.2'
+      - name: Generate parser
+        run: make grammar
+        env:
+          ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2'
+      - name: Build
+        run: make build
+      # mdl/roundtrip runs on the committed PedApp fixture and never calls mx.
+      # One engine since the legacy sdk/mpr backend was deleted
+      # (docs/plans/2026-09-14-retire-legacy-engine.md), so MXCLI_TEST_ENGINES
+      # is left unset.
+      - name: Setup mxbuild
+        if: matrix.mxbuild
+        run: ./bin/mxcli setup mxbuild --version 11.12.2
+      - name: Integration tests (${{ matrix.suite }})
+        run: make test-integration-${{ matrix.suite }}
+        env:
+          MXCLI_UPGRADE_SHARD: ${{ matrix.shard }}
+        timeout-minutes: 30
+
+  # One stable check name for "every integration suite passed", whatever the
+  # matrix above becomes — a branch rule can require this instead of each leg.
+  integration-passed:
+    if: always()
+    needs: integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: All integration suites passed
+        run: |
+          if [ "${{ needs.integration.result }}" != "success" ]; then
+            echo "integration: ${{ needs.integration.result }}"
+            exit 1
+          fi

Makefile

@@ -35,7 +35,7 @@ GO_BUILD_FLAGS = -trimpath
 # Clean version for VS Code extension (must be valid semver: major.minor.patch)
 VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0")
 
-.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet
+.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions test-integration test-integration-executor test-integration-roundtrip test-integration-upgrade test-integration-other grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet
 
 # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache)
 # Usage: $(call copy-if-changed,src,dst)
@@ -313,7 +313,47 @@ check-tunnel-deps:
 # therefore left unset everywhere; it survives only so that a stale
 # `MXCLI_TEST_ENGINES=legacy` is fatal rather than silently selecting nothing.
 test-integration:
-	CGO_ENABLED=0 go test -tags integration -count=1 -timeout 30m ./...
+	CGO_ENABLED=0 go test -tags integration -count=1 -timeout 60m ./...
+
+# The same integration tests split into suites that CI runs as parallel jobs
+# (ako/mxcli#757): one step running them all spent ~22 of its 30 minutes once
+# the upgrade property test landed, with mdl/roundtrip alone the critical path.
+#
+# Unlike test-integration, the suites run only the packages that HAVE
+# integration-tagged tests. Every other package's tests are identical with and
+# without the tag, and `make test` already runs them.
+#
+#   test-integration-executor   mdl/executor: doctype scripts through exec + mx check
+#   test-integration-roundtrip  mdl/roundtrip: the round-trip laws, the upgrade controls
+#   test-integration-upgrade    mdl/roundtrip: the upgrade property test alone;
+#                               MXCLI_UPGRADE_SHARD=i/n runs one of n shards,
+#                               MXCLI_UPGRADE_ALL=1 the full corpus (nightly)
+#   test-integration-other      every other package with integration tests,
+#                               found by build tag so a new one cannot be missed
+#                               (hidden dirs are skipped: nested worktrees live there)
+#
+# `-skip` / `-run` split mdl/roundtrip by test name, so a new round-trip test
+# lands in the roundtrip suite without editing this file.
+INTEGRATION_PKGS = $(shell grep -rl --include='*_test.go' --exclude-dir='.?*' --exclude-dir=reference --exclude-dir=mx-test-projects \
+	'^//go:build.*integration' . | xargs -n1 dirname | sed 's|^\./||; s|^|./|' | sort -u)
+INTEGRATION_SPLIT = ./mdl/executor ./mdl/roundtrip
+UPGRADE_PROPERTY = ^TestUpgradeExecutesToTheSameModel$$
+INTEGRATION_GO_TEST = CGO_ENABLED=0 go test -tags integration -count=1
+
+test-integration-executor:
+	$(INTEGRATION_GO_TEST) -timeout 40m ./mdl/executor/
+
+test-integration-roundtrip:
+	$(INTEGRATION_GO_TEST) -timeout 40m -skip '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/
+
+test-integration-upgrade:
+	$(INTEGRATION_GO_TEST) -timeout 40m -run '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/
+
+test-integration-other:
+	@pkgs="$(filter-out $(INTEGRATION_SPLIT),$(INTEGRATION_PKGS))"; \
+	if [ -z "$$pkgs" ]; then echo "no integration packages outside the split suites"; exit 1; fi; \
+	echo "integration packages: $$pkgs"; \
+	$(INTEGRATION_GO_TEST) -timeout 40m $$pkgs
 
 # Run MDL integration tests (requires Docker and a Mendix project)
 # Usage: make test-mdl MPR=path/to/app.mpr

mdl/roundtrip/shard_test.go

@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package roundtrip
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestParseShard pins the MXCLI_UPGRADE_SHARD spelling CI passes.
+func TestParseShard(t *testing.T) {
+	for _, tc := range []struct {
+		in   string
+		i, n int
+		bad  bool
+	}{
+		{in: "", i: 1, n: 1},
+		{in: "1/1", i: 1, n: 1},
+		{in: "2/3", i: 2, n: 3},
+		{in: " 3/3 ", i: 3, n: 3},
+		{in: "0/3", bad: true},
+		{in: "4/3", bad: true},
+		{in: "1/0", bad: true},
+		{in: "3", bad: true},
+		{in: "a/b", bad: true},
+		{in: "1/2/3", bad: true},
+	} {
+		s, err := parseShard(tc.in)
+		if tc.bad {
+			if err == nil {
+				t.Errorf("parseShard(%q) = %+v, want an error", tc.in, s)
+			}
+			continue
+		}
+		if err != nil || s.index != tc.i || s.count != tc.n {
+			t.Errorf("parseShard(%q) = %+v, %v; want %d/%d", tc.in, s, err, tc.i, tc.n)
+		}
+	}
+}
+
+// TestShardsPartition: CI runs the shards as separate jobs and never the
+// unsharded test, so the shards together must execute every script exactly
+// once. A script in no shard would silently leave per-PR CI.
+func TestShardsPartition(t *testing.T) {
+	for n := 1; n <= 7; n++ {
+		const items = 101
+		seen := make([]int, items)
+		for i := 1; i <= n; i++ {
+			s, err := parseShard(fmt.Sprintf("%d/%d", i, n))
+			if err != nil {
+				t.Fatal(err)
+			}
+			for k := 0; k < items; k++ {
+				if s.has(k) {
+					seen[k]++
+				}
+			}
+		}
+		for k, c := range seen {
+			if c != 1 {
+				t.Fatalf("n=%d: item %d is in %d shards, want exactly 1", n, k, c)
+			}
+		}
+	}
+}

mdl/roundtrip/upgrade_property_test.go

@@ -56,15 +56,25 @@ import (
 // Executing the whole corpus takes about 15 minutes, which on its own exceeds
 // what the CI integration step has left (ako/mxcli#742 timed out there).
 // MXCLI_UPGRADE_ALL=1 executes every script, header-only ones included; run it
-// when a langver.Change or a gated rewrite lands.
+// when a langver.Change or a gated rewrite lands. The nightly workflow runs it.
+//
+// MXCLI_UPGRADE_SHARD=i/n executes only every n-th executable script, starting
+// at the i-th, so per-PR CI can split the executions across parallel jobs
+// (ako/mxcli#757); the shards together execute each script exactly once
+// (TestShardsPartition). Unset is the whole set.
 func TestUpgradeExecutesToTheSameModel(t *testing.T) {
 	a, b := newHarness(t), newHarness(t)
 	defer a.close()
 	defer b.close()
 
 	scripts := upgradeExampleScripts(t)
 	all := os.Getenv("MXCLI_UPGRADE_ALL") != ""
-	var same, outOfScope, unparsed, headerOnly, keptVersion []string
+	sh, err := parseShard(os.Getenv("MXCLI_UPGRADE_SHARD"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	executable := 0 // scripts that reached execution, in every shard
+	var same, outOfScope, unparsed, headerOnly, keptVersion, otherShard []string
 	for _, path := range scripts {
 		src, err := os.ReadFile(path)
 		if err != nil {
@@ -100,6 +110,14 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) {
 				headerOnly = append(headerOnly, rel)
 				return
 			}
+			// Every shard upgrades and checks every script above, which is
+			// cheap; only the execution below, which is not, is split.
+			k := executable
+			executable++
+			if !sh.has(k) {
+				otherShard = append(otherShard, rel)
+				return
+			}
 
 			errA, errB, diff := executeBoth(t, a, b, string(src), res.Source)
 			if len(diff) > 0 {
@@ -120,6 +138,10 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) {
 	}
 
 	t.Logf("%d scripts execute on PedApp and upgrade to the same model", len(same))
+	if sh.count > 1 {
+		t.Logf("shard %s: %d of %d executable scripts belong to other shards and were not executed here",
+			sh, len(otherShard), executable)
+	}
 	t.Logf("%d scripts are out of scope: their original does not execute cleanly on PedApp:\n  %s",
 		len(outOfScope), strings.Join(outOfScope, "\n  "))
 	if !all {
@@ -130,10 +152,11 @@ func TestUpgradeExecutesToTheSameModel(t *testing.T) {
 		len(keptVersion), strings.Join(keptVersion, "\n  "))
 	t.Logf("%d scripts do not parse (negative tests) and cannot be upgraded:\n  %s",
 		len(unparsed), strings.Join(unparsed, "\n  "))
-	if os.Getenv("MXCLI_UPGRADE_EXAMPLES") == "" && len(same) < 50 {
+	if floor := 50 / sh.count; os.Getenv("MXCLI_UPGRADE_EXAMPLES") == "" && len(same) < floor {
 		// Hundreds execute today; a handful means the harness broke, and a
-		// property checked on nothing passes.
-		t.Errorf("only %d scripts executed cleanly — the harness is not exercising the property", len(same))
+		// property checked on nothing passes. A shard sees 1/n of them.
+		t.Errorf("only %d scripts executed cleanly (shard %s) — the harness is not exercising the property",
+			len(same), sh)
 	}
 }
 

mdl/roundtrip/upgrade_shard_test.go

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package roundtrip
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// shard selects every count-th item starting at index-1: MXCLI_UPGRADE_SHARD
+// ("i/n", 1-based) splits the upgrade property test's executions across CI
+// jobs (ako/mxcli#757). Round-robin rather than contiguous ranges, because
+// the corpus is sorted by path and neighbouring scripts cost alike.
+type shard struct{ index, count int }
+
+// parseShard reads "i/n"; empty is the whole set, 1/1.
+func parseShard(s string) (shard, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return shard{1, 1}, nil
+	}
+	a, b, ok := strings.Cut(s, "/")
+	i, errI := strconv.Atoi(a)
+	n, errN := strconv.Atoi(b)
+	if !ok || errI != nil || errN != nil || n < 1 || i < 1 || i > n {
+		return shard{}, fmt.Errorf("MXCLI_UPGRADE_SHARD=%q: want i/n with 1 <= i <= n", s)
+	}
+	return shard{i, n}, nil
+}
+
+// has reports whether the k-th item (0-based) belongs to this shard.
+func (s shard) has(k int) bool { return k%s.count == s.index-1 }
+
+func (s shard) String() string { return fmt.Sprintf("%d/%d", s.index, s.count) }