Microflow/nanoflow create or modify as diff-then-patch: an unchanged definition writes nothing (#747)

.claude/skills/mendix/write-microflows/SKILL.md

@@ -42,12 +42,12 @@ Choose the mode by who owns the microflow ([choose-edit-mode](../choose-edit-mod
 
 - **Created by your MDL scripts, and not edited in Studio Pro since:** edit the script
   (or fresh `describe` output) and re-run `create or modify`.
-- **Authored in Studio Pro:** there is **no `alter microflow` yet**, and re-emitting it
-  with `create or modify` renumbers element IDs, removes merges and resets connector
-  curves even for a one-line change. Keep the change minimal: put new logic in a new
-  sub-microflow and change the existing flow only to call it. Commit first, then
-  `describe` it again after `exec` and diff it against the original output. Anything
-  that differs and that you did not change is a loss.
+- **Authored in Studio Pro:** prefer `alter microflow X { insert/replace/drop … }`
+  (targets from `describe microflow X with handles`). `create or modify` of `describe`
+  output patches too: unchanged writes nothing; a top-level or `if`-branch statement
+  change is spliced in. Other changes (header, loop body, error handler, moved node)
+  rebuild the flow under mdl 0 (`MDL-V1-REBUILD`: IDs renumbered, merges and curves
+  lost) and are refused under `mdl 1;`.
 
 ## When to Use a Microflow vs a Nanoflow
 

.claude/skills/mendix/write-nanoflows/SKILL.md

@@ -23,12 +23,16 @@ Choose the mode by who owns the nanoflow ([choose-edit-mode](../choose-edit-mode
 
 - **Created by your MDL scripts, and not edited in Studio Pro since:** edit the script
   (or fresh `describe` output) and re-run `create or modify`.
-- **Authored in Studio Pro:** there is **no `alter nanoflow` yet**, and re-emitting it
-  with `create or modify` has dropped annotation links and changed the export level on
-  Studio Pro nanoflows, even with no edit at all. Keep the change minimal: put new logic
-  in a new nanoflow and change the existing one only to call it. Commit first, then
-  `describe` it again after `exec` and diff it against the original output. Anything
-  that differs and that you did not change is a loss.
+- **Authored in Studio Pro:** prefer `alter nanoflow X { insert/replace/drop … }`
+  (targets by output variable, caption or statement pattern). `create or modify` of `describe`
+  output also works as a patch: an unchanged definition writes nothing, and an inserted,
+  replaced or dropped statement (at the top level or in an `if` branch) is spliced in,
+  leaving every other node, merge and curve as stored. A change it cannot splice — the
+  header, anything inside a loop body or error handler, a moved node — rebuilds the
+  whole nanoflow under mdl 0
+  (warning `MDL-V1-REBUILD`: element IDs renumbered, merges removed, curves reset) and
+  is refused under `mdl 1;` (header and loop-body changes have no splice yet; move
+  nodes in Studio Pro).
 
 ## When to Use a Nanoflow vs a Microflow
 

.github/workflows/nightly.yml

@@ -54,9 +54,51 @@ jobs:
       - name: Setup mxbuild ${{ matrix.mendix-version }}
         run: ./bin/mxcli setup mxbuild --version ${{ matrix.mendix-version }}
 
+      # One step, every package: go test runs the packages in parallel, so the
+      # wall time is the slowest one's (mdl/roundtrip, ~21 min on this runner in
+      # ako/mxcli#757's measurement) and grows as the round trip gains fixtures.
+      # Per-PR CI splits this into parallel jobs instead (push-test.yml).
       - name: "Integration tests (Mendix ${{ matrix.mendix-version }})"
         run: make test-integration
-        timeout-minutes: 30
+        timeout-minutes: 60
+
+  # The upgrade property test over the WHOLE mdl-examples corpus, header-only
+  # scripts included (MXCLI_UPGRADE_ALL). Per-PR CI executes only the scripts
+  # the upgrade rewrites beyond the header and terminators; this is the run that
+  # proves langver's gating on the rest (ako/mxcli#757). Independent of the
+  # Mendix version — it runs on the committed PedApp fixture and never calls
+  # mx — so it runs once, not per matrix entry. It does not gate the release.
+  upgrade-full-corpus:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ['1/3', '2/3', '3/3']
+    name: upgrade full corpus (${{ matrix.shard }})
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.26.6'
+      - name: Cache ANTLR4 JAR
+        uses: actions/cache@v6
+        with:
+          path: ~/.m2/repository/org/antlr/antlr4
+          key: antlr4-4.13.2
+      - name: Install ANTLR4
+        run: pip install 'antlr4-tools==0.2.2'
+      - name: Generate parser
+        run: make grammar
+        env:
+          ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2'
+      - name: Build
+        run: make build
+      - name: Upgrade property test, full corpus (${{ matrix.shard }})
+        run: make test-integration-upgrade
+        env:
+          MXCLI_UPGRADE_ALL: '1'
+          MXCLI_UPGRADE_SHARD: ${{ matrix.shard }}
+        timeout-minutes: 45
 
   nightly:
     needs: test

.github/workflows/push-test.yml

@@ -167,17 +167,98 @@ jobs:
         run: ./scripts/check-skill-mdl.sh ./bin/mxcli .claude/skills/mendix
       - name: Check docs-site MDL blocks
         run: ./scripts/check-skill-mdl.sh ./bin/mxcli docs-site/src
-      - name: Setup mxbuild
-        run: ./bin/mxcli setup mxbuild --version 11.12.2
-      # One engine since the legacy sdk/mpr backend was deleted
-      # (docs/plans/2026-09-14-retire-legacy-engine.md), so there is no matrix to
-      # narrow here any more and MXCLI_TEST_ENGINES is left unset.
-      - name: Integration tests
-        run: make test-integration
-        timeout-minutes: 30
+      # The integration tests are the `integration` job below.
       - name: Lint Go
         run: make lint-go
       - name: Vulnerability scan
         run: |
           go install golang.org/x/vuln/cmd/govulncheck@latest
           govulncheck ./...
+
+  # The integration tests, split into suites that run as parallel jobs
+  # (ako/mxcli#757). As one `make test-integration` step they took ~22 of its 30
+  # minutes once the upgrade property test landed, and the step's wall time was
+  # mdl/roundtrip's alone (~21 min) — go test already ran the packages in
+  # parallel, so no timeout below 30 min was going to hold as the round trip
+  # grows (#743 adds TestApp to it).
+  #
+  # Measured per package on the ubuntu runner (run 36319810164) and what each
+  # suite is expected to take, including ~3 min of checkout/build/mxbuild:
+  #   executor   mdl/executor                          ~15.5 min -> ~19 min
+  #   roundtrip  mdl/roundtrip, round-trip laws         ~1.7 min -> ~4 min
+  #   upgrade    upgrade property test, 3 shards        ~19 min  -> ~10 min each
+  #   other      cmd/mxcli{,/docker,/marketplace}       ~5.7 min -> ~8 min
+  # The full-corpus upgrade run (MXCLI_UPGRADE_ALL) is nightly, not here.
+  #
+  # Nothing here is dropped from per-PR CI: the round-trip laws and the
+  # execute-both upgrade test run at their default scope, the shards together
+  # executing every script exactly once (TestShardsPartition). Only the unit
+  # tests of packages WITHOUT integration tests are no longer re-run under the
+  # tag — `make test` in build-and-test runs those.
+  integration:
+    name: integration (${{ matrix.suite }}${{ matrix.shard && format(' {0}', matrix.shard) || '' }})
+    runs-on: ubuntu-latest
+    # Each suite's go test has its own -timeout (Makefile); this is the job's
+    # backstop, with room for the setup steps.
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: executor
+            mxbuild: true
+          - suite: roundtrip
+          - suite: upgrade
+            shard: 1/3
+          - suite: upgrade
+            shard: 2/3
+          - suite: upgrade
+            shard: 3/3
+          - suite: other
+            mxbuild: true
+    steps:
+      # #743 adds TestApp as a git submodule to the round trip; its checkout
+      # will need `submodules: true` here for the roundtrip suite.
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.26.6'
+      - name: Cache ANTLR4 JAR
+        uses: actions/cache@v6
+        with:
+          path: ~/.m2/repository/org/antlr/antlr4
+          key: antlr4-4.13.2
+      - name: Install ANTLR4
+        run: pip install 'antlr4-tools==0.2.2'
+      - name: Generate parser
+        run: make grammar
+        env:
+          ANTLR4_TOOLS_ANTLR_VERSION: '4.13.2'
+      - name: Build
+        run: make build
+      # mdl/roundtrip runs on the committed PedApp fixture and never calls mx.
+      # One engine since the legacy sdk/mpr backend was deleted
+      # (docs/plans/2026-09-14-retire-legacy-engine.md), so MXCLI_TEST_ENGINES
+      # is left unset.
+      - name: Setup mxbuild
+        if: matrix.mxbuild
+        run: ./bin/mxcli setup mxbuild --version 11.12.2
+      - name: Integration tests (${{ matrix.suite }})
+        run: make test-integration-${{ matrix.suite }}
+        env:
+          MXCLI_UPGRADE_SHARD: ${{ matrix.shard }}
+        timeout-minutes: 30
+
+  # One stable check name for "every integration suite passed", whatever the
+  # matrix above becomes — a branch rule can require this instead of each leg.
+  integration-passed:
+    if: always()
+    needs: integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: All integration suites passed
+        run: |
+          if [ "${{ needs.integration.result }}" != "success" ]; then
+            echo "integration: ${{ needs.integration.result }}"
+            exit 1
+          fi

Makefile

@@ -35,7 +35,7 @@ GO_BUILD_FLAGS = -trimpath
 # Clean version for VS Code extension (must be valid semver: major.minor.patch)
 VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0")
 
-.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet
+.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions test-integration test-integration-executor test-integration-roundtrip test-integration-upgrade test-integration-other grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet
 
 # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache)
 # Usage: $(call copy-if-changed,src,dst)
@@ -313,7 +313,47 @@ check-tunnel-deps:
 # therefore left unset everywhere; it survives only so that a stale
 # `MXCLI_TEST_ENGINES=legacy` is fatal rather than silently selecting nothing.
 test-integration:
-	CGO_ENABLED=0 go test -tags integration -count=1 -timeout 30m ./...
+	CGO_ENABLED=0 go test -tags integration -count=1 -timeout 60m ./...
+
+# The same integration tests split into suites that CI runs as parallel jobs
+# (ako/mxcli#757): one step running them all spent ~22 of its 30 minutes once
+# the upgrade property test landed, with mdl/roundtrip alone the critical path.
+#
+# Unlike test-integration, the suites run only the packages that HAVE
+# integration-tagged tests. Every other package's tests are identical with and
+# without the tag, and `make test` already runs them.
+#
+#   test-integration-executor   mdl/executor: doctype scripts through exec + mx check
+#   test-integration-roundtrip  mdl/roundtrip: the round-trip laws, the upgrade controls
+#   test-integration-upgrade    mdl/roundtrip: the upgrade property test alone;
+#                               MXCLI_UPGRADE_SHARD=i/n runs one of n shards,
+#                               MXCLI_UPGRADE_ALL=1 the full corpus (nightly)
+#   test-integration-other      every other package with integration tests,
+#                               found by build tag so a new one cannot be missed
+#                               (hidden dirs are skipped: nested worktrees live there)
+#
+# `-skip` / `-run` split mdl/roundtrip by test name, so a new round-trip test
+# lands in the roundtrip suite without editing this file.
+INTEGRATION_PKGS = $(shell grep -rl --include='*_test.go' --exclude-dir='.?*' --exclude-dir=reference --exclude-dir=mx-test-projects \
+	'^//go:build.*integration' . | xargs -n1 dirname | sed 's|^\./||; s|^|./|' | sort -u)
+INTEGRATION_SPLIT = ./mdl/executor ./mdl/roundtrip
+UPGRADE_PROPERTY = ^TestUpgradeExecutesToTheSameModel$$
+INTEGRATION_GO_TEST = CGO_ENABLED=0 go test -tags integration -count=1
+
+test-integration-executor:
+	$(INTEGRATION_GO_TEST) -timeout 40m ./mdl/executor/
+
+test-integration-roundtrip:
+	$(INTEGRATION_GO_TEST) -timeout 40m -skip '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/
+
+test-integration-upgrade:
+	$(INTEGRATION_GO_TEST) -timeout 40m -run '$(UPGRADE_PROPERTY)' ./mdl/roundtrip/
+
+test-integration-other:
+	@pkgs="$(filter-out $(INTEGRATION_SPLIT),$(INTEGRATION_PKGS))"; \
+	if [ -z "$$pkgs" ]; then echo "no integration packages outside the split suites"; exit 1; fi; \
+	echo "integration packages: $$pkgs"; \
+	$(INTEGRATION_GO_TEST) -timeout 40m $$pkgs
 
 # Run MDL integration tests (requires Docker and a Mendix project)
 # Usage: make test-mdl MPR=path/to/app.mpr

mdl/executor/cmd_alter_flow.go

@@ -45,31 +45,48 @@ func execAlterFlow(ctx *ExecContext, s *ast.AlterFlowStmt) error {
 		targets[i] = c
 	}
 
+	mut, err := a.apply(ctx, s.Operations, targets)
+	if err != nil {
+		return err
+	}
+	if err := mut.Save(); err != nil {
+		return mdlerrors.NewBackend("save altered "+s.Kind(), err)
+	}
+	fmt.Fprintf(ctx.Output, "Altered %s %s\n", s.Kind(), s.Name)
+	return nil
+}
+
+// apply opens the stored flow for splicing and applies ops, each aimed at the
+// candidate at the same index of targets (resolved against the flow as stored).
+// It returns the mutator unsaved: the caller saves, or discards it on error so
+// nothing is written.
+func (a *alterFlowContext) apply(ctx *ExecContext, ops []*ast.AlterFlowOperation, targets []mfmutator.Candidate) (backend.MicroflowMutator, error) {
+	s := a.stmt
 	mut, err := ctx.Backend.OpenMicroflowForMutation(a.mf.ID)
 	if err != nil {
-		return mdlerrors.NewBackend("open "+s.Kind()+" for alter", err)
+		return nil, mdlerrors.NewBackend("open "+s.Kind()+" for alter", err)
 	}
-	for i, op := range s.Operations {
+	for i, op := range ops {
 		target := targets[i]
 		fail := func(err error) error {
 			return mdlerrors.NewValidation(fmt.Sprintf("alter %s %s: %s %s: %v", s.Kind(), s.Name, op.Op, op.Target, err))
 		}
 		if op.Op == ast.AlterFlowDrop {
 			if err := a.checkOutputUnused(target, nil); err != nil {
-				return fail(err)
+				return nil, fail(err)
 			}
 			if err := mut.Drop(target.ID); err != nil {
-				return fail(err)
+				return nil, fail(err)
 			}
 			a.noteRemoved(target, nil)
 			continue
 		}
 		frag, err := a.buildFragment(ctx, op.Body)
 		if err != nil {
-			return fail(err)
+			return nil, fail(err)
 		}
 		if err := a.checkFragmentScope(ctx, op, target, frag); err != nil {
-			return fail(err)
+			return nil, fail(err)
 		}
 		switch op.Op {
 		case ast.AlterFlowInsertAfter:
@@ -84,18 +101,14 @@ func execAlterFlow(ctx *ExecContext, s *ast.AlterFlowStmt) error {
 			err = fmt.Errorf("unknown operation")
 		}
 		if err != nil {
-			return fail(err)
+			return nil, fail(err)
 		}
 		if op.Op == ast.AlterFlowReplace {
 			a.noteRemoved(target, frag)
 		}
 		a.noteFragment(ctx, frag)
 	}
-	if err := mut.Save(); err != nil {
-		return mdlerrors.NewBackend("save altered "+s.Kind(), err)
-	}
-	fmt.Fprintf(ctx.Output, "Altered %s %s\n", s.Kind(), s.Name)
-	return nil
+	return mut, nil
 }
 
 // alterFlowContext is what the operations of one statement share: the stored
@@ -115,11 +128,15 @@ type alterFlowContext struct {
 	declaredByOps map[string]bool
 	readByOps     map[string][]string
 	removedByOps  map[string]bool
+	// removedIDs are the stored activities earlier operations took out; what
+	// they read no longer counts as a use.
+	removedIDs map[model.ID]bool
 }
 
 // noteRemoved records that target's output is gone, unless the fragment that
 // replaces it declares it again.
 func (a *alterFlowContext) noteRemoved(target mfmutator.Candidate, replacement *backend.MicroflowFragment) {
+	a.removedIDs[target.ID] = true
 	v := target.OutputVariable
 	if v == "" || fragmentDeclares(replacement, v) {
 		return
@@ -161,7 +178,8 @@ func loadAlterFlow(ctx *ExecContext, s *ast.AlterFlowStmt) (*alterFlowContext, e
 		return nil, mdlerrors.NewBackend("build hierarchy", err)
 	}
 	a := &alterFlowContext{stmt: s, entityNames: getEntityNames(ctx, h),
-		declaredByOps: map[string]bool{}, readByOps: map[string][]string{}, removedByOps: map[string]bool{}}
+		declaredByOps: map[string]bool{}, readByOps: map[string][]string{}, removedByOps: map[string]bool{},
+		removedIDs: map[model.ID]bool{}}
 	// A copy: nanoflow names are added below, and the cached map is shared.
 	a.microflowNames = map[model.ID]string{}
 	for id, n := range getMicroflowNames(ctx, h) {
@@ -487,7 +505,7 @@ func (a *alterFlowContext) checkOutputUnused(target mfmutator.Candidate, replace
 	ref := regexp.MustCompile(`\$` + regexp.QuoteMeta(v) + `\b`)
 	var users []string
 	for _, c := range a.cands {
-		if c.ID == target.ID {
+		if c.ID == target.ID || a.removedIDs[c.ID] {
 			continue
 		}
 		for _, text := range append([]string{c.Statement}, c.Alternates...) {