Introduce Menudet 1.0 draft for portable array and table computation

.github/menudet-native-ref

@@ -0,0 +1 @@
+e77b91aca382a6123ca04f4e2dbc2430f147e2d6

.github/workflows/build.yml

@@ -15,6 +15,7 @@ jobs:
     env:
       CMAKE_GENERATOR: Ninja
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.12"]
@@ -33,6 +34,31 @@ jobs:
     steps:
     - uses: actions/checkout@v7
 
+    # Experimental tests require the explicit native pair, not the unchanged
+    # release dependency. Keep this source override local to numpy-compat CI.
+    - name: Select experimental native revision
+      if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+      id: native_ref
+      shell: bash
+      run: echo "sha=$(cat .github/menudet-native-ref)" >> "$GITHUB_OUTPUT"
+
+    - uses: actions/checkout@v7
+      if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+      with:
+        repository: Blosc/miniexpr
+        ref: ${{ steps.native_ref.outputs.sha }}
+        path: ci-native
+
+    - name: Override experimental native source
+      if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+      shell: bash
+      run: |
+        source_path="$PWD/ci-native"
+        if [ "$RUNNER_OS" = Windows ]; then
+          source_path=$(cygpath -m "$source_path")
+        fi
+        echo "CMAKE_ARGS=-DFETCHCONTENT_SOURCE_DIR_MINIEXPR=$source_path" >> "$GITHUB_ENV"
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v7
       with:
@@ -67,17 +93,18 @@ jobs:
       if: runner.os == 'Windows'
       run: echo "C:\\Program Files\\LLVM\\bin" >> $env:GITHUB_PATH
 
-    - name: Install specific numpy version
-      if: matrix.numpy-version
-      run: pip install "numpy==${{ matrix.numpy-version }}.*"
+    - name: Resolve compiler cache launcher (Windows)
+      if: runner.os == 'Windows'
+      run: |
+        $launcher = (Get-Command sccache -CommandType Application -ErrorAction Stop).Source.Replace('\', '/')
+        "CMAKE_C_COMPILER_LAUNCHER=$launcher" >> $env:GITHUB_ENV
+        "CMAKE_CXX_COMPILER_LAUNCHER=$launcher" >> $env:GITHUB_ENV
 
     - name: Build (Windows)
       if: runner.os == 'Windows'
       id: build_windows
       run: pip install -e . --group test
       env:
-        CMAKE_C_COMPILER_LAUNCHER: sccache
-        CMAKE_CXX_COMPILER_LAUNCHER: sccache
         SCCACHE_DIR: C:\Users\runneradmin\AppData\Local\sccache
         CC: clang-cl
         CXX: clang-cl
@@ -89,9 +116,31 @@ jobs:
       id: build_non_windows
       run: pip install -e . --group test
 
+    - name: Install Arrow dictionary test dependency
+      # Pandas is already a dev dependency; Arrow's dictionary conversions now
+      # import it even when the test does not construct a pandas object.
+      run: python -m pip install pandas
+
+    # Build isolation uses NumPy 2 headers for the dual-ABI extension. Optional
+    # test dependencies may upgrade the runtime too, so pin it AFTER installing.
+    - name: Install NumPy 1.26-compatible test runtime
+      if: matrix.numpy-version == '1.26'
+      # PyArrow 26 dropped the NumPy 1 ABI without encoding it in Requires-Dist.
+      # Keep this host-compatibility job on a known dual-ABI Arrow release.
+      run: pip install "numpy==1.26.*" "zarr>=3.1.5,<3.2" "pyarrow==21.0.0" pandas
+
+    - name: Verify requested NumPy runtime
+      if: matrix.numpy-version
+      run: python -c "import numpy as np; assert np.__version__.startswith('${{ matrix.numpy-version }}.'), np.__version__"
+
+    - name: Require a NumExpr-free runtime
+      run: |
+        python -m pip uninstall -y numexpr
+        python -c "import importlib.util, importlib.metadata; assert importlib.util.find_spec('numexpr') is None; assert not any(r.lower().startswith('numexpr') for r in importlib.metadata.requires('blosc2') or [])"
+
     - name: Test (Windows)
       if: runner.os == 'Windows'
-      run: python -m pytest -m "not heavy and not network and not tui" --durations=25
+      run: python scripts/qualify_without_numexpr.py --require-absent -- -m "not heavy and not network and not tui" --durations=25
       env:
         # A test over this many seconds is a deadlock, not a slow test (the
         # slowest is ~7 s): dump stacks and kill the worker so xdist names it
@@ -101,16 +150,44 @@ jobs:
         PYTEST_DEADMAN_SECONDS: "300"
         PYTEST_DEADMAN_SESSION_SECONDS: "1500"
 #        BLOSC_NTHREADS: "1"
-#        NUMEXPR_NUM_THREADS: "1"
 #        OMP_NUM_THREADS: "1"
 
     - name: Test (non-Windows)
       if: runner.os != 'Windows'
-      run: python -m pytest -m "not heavy and not network and not tui" --durations=25
+      run: python scripts/qualify_without_numexpr.py --require-absent -- -m "not heavy and not network and not tui" --durations=25
       env:
         PYTEST_DEADMAN_SECONDS: "300"
         PYTEST_DEADMAN_SESSION_SECONDS: "1500"
 
+    - name: Verify independence with optional NumExpr installed
+      if: runner.os == 'Linux' && matrix.python-version == '3.12' && !matrix.numpy-version
+      run: |
+        python -m pip install numexpr
+        python -m pytest tests/test_no_numexpr.py -q -n 0
+
+    - name: Verify experimental safe-graph corpus
+      if: always() && (steps.build_windows.outcome == 'success' || steps.build_non_windows.outcome == 'success')
+      run: python scripts/verify_safe_lazyexpr.py --report safe-lazyexpr.json
+
+    - name: Upload safe-graph platform evidence
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: safe-lazyexpr-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version || 'default' }}
+        path: safe-lazyexpr.json
+        if-no-files-found: ignore
+
+    - name: Certify bounded Menudet math against independent references
+      run: python scripts/certify_menudet_math.py --report menudet-math.json
+
+    - name: Upload Menudet math certification
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: menudet-math-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version || 'default' }}
+        path: menudet-math.json
+        if-no-files-found: ignore
+
     # TUIs are timing-sensitive and only boot a headless terminal: one OS is
     # enough, and a flake here is feedback, not a release blocker.
     - name: Test TUI (non-blocking)

.github/workflows/cibuildwheels.yml

@@ -1,6 +1,8 @@
 name: Python wheels
 
 on:
+  # Certify development revisions without creating a release tag or publishing.
+  workflow_dispatch:
   # Trigger the workflow only for tags and PRs to the main branch
   push:
     tags:
@@ -11,6 +13,22 @@ on:
 
 env:
   CIBW_BUILD_VERBOSITY: 1
+  # Every tested wheel checks portable descriptors/persistence and independent
+  # numeric references, not only whether the extension can be imported.
+  # Free-threaded CPython safely enables the GIL for this extension. Allow only
+  # that expected import warning; never force PYTHON_GIL=0 or claim no-GIL safety.
+  CIBW_TEST_COMMAND: >-
+    python -m pytest
+    {project}/tests/test_portable_artifact.py
+    {project}/tests/test_portable_descriptor.py
+    {project}/tests/test_portable_descriptor_host.py
+    {project}/tests/test_portable_lazy.py
+    {project}/tests/test_portable_jit_integration.py
+    {project}/tests/test_portable_policy.py
+    {project}/tests/test_portable_table_export.py
+    {project}/tests/test_dsl_portable.py -q -n 0 &&
+    python {project}/scripts/certify_menudet_math.py &&
+    python {project}/scripts/verify_safe_lazyexpr.py
   # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long
   # Now that github provides native arm64 runners, we can enable tests again
   # CIBW_TEST_SKIP: "*linux*aarch64*"
@@ -156,7 +174,10 @@ jobs:
       CIBW_BUILD: ${{ matrix.cibw_build }}
       # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths
       CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF"
-      CIBW_TEST_COMMAND: "pytest {project}/tests"
+      CIBW_TEST_COMMAND: >-
+        pytest {project}/tests &&
+        python {project}/scripts/certify_menudet_math.py &&
+        python {project}/scripts/verify_safe_lazyexpr.py
       # Pin the Pyodide version explicitly per target for reproducible builds.
       # Note: in-memory SChunk get_slice is broken on the Pyodide 0.29.x
       # Emscripten toolchain (works on 314); see gh-664. The affected test is
@@ -282,6 +303,9 @@ jobs:
           python -m pytest tests/test_open.py tests/test_vlmeta.py \
                            tests/ndarray/test_evaluate.py -q -p no:cacheprovider
 
+      - name: Certify Menudet on the installed abi3 wheel
+        run: python scripts/certify_menudet_math.py
+
 
   upload_pypi:
     needs: [ build_wheels, build_wheels_wasm, test_abi3_matrix ]

.github/workflows/menudet-paired.yml

@@ -0,0 +1,116 @@
+name: Menudet paired qualification
+
+# Keep release dependency pins unchanged. Qualification explicitly selects the
+# experimental native revision and fails rather than hiding missing capabilities.
+on:
+  push:
+    branches: [numpy-compat]
+  workflow_dispatch:
+    inputs:
+      native_ref:
+        description: Fully qualified exact Blosc/miniexpr SHA to integrate with this Python revision
+        required: true
+        type: string
+
+concurrency:
+  group: menudet-pair-${{ github.ref }}-${{ inputs.native_ref }}
+  cancel-in-progress: true
+
+jobs:
+  paired:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    env:
+      MENUDET_REQUIRE_ARRAY_RUNTIME: '1'
+      MENUDET_REQUIRE_GRAPH_RUNTIME: '1'
+      CMAKE_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v7
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: blosc2
+          python-version: '3.12'
+          auto-activate-base: false
+      - name: Require exact full native qualification
+        id: qualified_native
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REQUESTED_NATIVE: ${{ inputs.native_ref }}
+        run: |
+          native_sha="$REQUESTED_NATIVE"
+          if [ -z "$native_sha" ]; then native_sha=$(cat .github/menudet-native-ref); fi
+          python scripts/verify_native_qualification.py "$native_sha"
+          echo "sha=$native_sha" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v7
+        with:
+          repository: Blosc/miniexpr
+          ref: ${{ steps.qualified_native.outputs.sha }}
+          path: native
+      - uses: ilammy/msvc-dev-cmd@v1
+        if: runner.os == 'Windows'
+      - name: Select Windows compiler
+        if: runner.os == 'Windows'
+        run: |
+          echo 'CC=clang-cl' >> "$GITHUB_ENV"
+          echo 'CXX=clang-cl' >> "$GITHUB_ENV"
+      - name: Build explicit native and Python pair
+        run: |
+          python -m pip install ninja cmake
+          # Match Python's static linkage; Windows Ninja cannot emit both the
+          # shared import library and static library as miniexpr.lib.
+          cmake -S native -B native-build -DMINIEXPR_BUILD_ARTIFACT=ON -DMINIEXPR_BUILD_SHARED=OFF -DCMAKE_BUILD_TYPE=Release
+          # Python's corpus adapter needs this runner, not the native test matrix.
+          cmake --build native-build --target numpy_compat_runner --parallel 4
+          native_source="$PWD/native"
+          if [ "$RUNNER_OS" = Windows ]; then
+            native_source=$(cygpath -m "$native_source")
+          fi
+          python -m pip install -e . --group test --config-settings="cmake.define.FETCHCONTENT_SOURCE_DIR_MINIEXPR=$native_source"
+          python -m pip install 'numpy==2.5.3'
+      - name: Require corpus and native array capabilities
+        run: |
+          export MENUDET_NUMPY_FUNCTION_CORPUS="$PWD/native/tests/numpy-compat/functions-v1.0.json"
+          export MENUDET_NUMPY_ARITHMETIC_CORPUS="$PWD/native/tests/numpy-compat/arithmetic-v1.0.json"
+          export MENUDET_NUMPY_CORPUS="$PWD/native/tests/numpy-compat/vectors.json"
+          export MENUDET_NUMPY_CORPUS_V2="$PWD/native/tests/numpy-compat/vectors-v2.json"
+          export MENUDET_NUMPY_RUNNER="$PWD/native-build/tests/numpy_compat_runner"
+          if [ "$RUNNER_OS" = Windows ]; then
+            export MENUDET_NUMPY_RUNNER="${MENUDET_NUMPY_RUNNER}.exe"
+            export PATH="$PWD/native-build:$PATH"
+          fi
+          python -c 'import blosc2; assert blosc2.NativeGraph.from_expression("x - sum(x)", {"x": "float32"}).info()["stages"] == 2'
+          python -m pytest tests/test_menudet_arrays.py tests/test_menudet_native_graph.py tests/test_native_graph_preparation.py tests/test_native_graph_stages.py tests/test_menudet_functions.py tests/test_menudet_arithmetic.py tests/test_menudet_conformance.py tests/test_dsl_portable.py tests/test_portable_artifact.py tests/test_portable_lazy.py tests/test_portable_table_export.py tests/test_portable_policy.py tests/test_portable_descriptor.py tests/test_native_graph_weak_captures.py tests/test_dsl_scalar_dispatch.py -q -n 0
+      - name: Build exact-pair distribution wheel
+        env:
+          ME_DSL_JIT_COMPILER: tcc
+          MENUDET_REQUIRE_JIT: ${{ runner.os != 'Windows' && '1' || '0' }}
+        run: |
+          native_source="$PWD/native"
+          if [ "$RUNNER_OS" = Windows ]; then
+            native_source=$(cygpath -m "$native_source")
+          fi
+          python -m pip wheel . --no-deps --wheel-dir graph-wheels --config-settings="cmake.define.FETCHCONTENT_SOURCE_DIR_MINIEXPR=$native_source"
+          python -m pip install --force-reinstall --no-deps graph-wheels/*.whl
+          python -m pytest tests/test_portable_jit_integration.py tests/test_menudet_jit.py -q -n 0
+      - name: Record exact paired revisions
+        if: always()
+        run: |
+          git rev-parse HEAD > paired-revisions.txt
+          git -C native rev-parse HEAD >> paired-revisions.txt
+          python --version >> paired-revisions.txt
+          python -m pip freeze >> paired-revisions.txt
+          cmake --version >> paired-revisions.txt
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: menudet-pair-${{ matrix.os }}
+          path: |
+            paired-revisions.txt
+            native-qualification.json
+            graph-wheels/*.whl

.github/workflows/wasm-tests.yml

@@ -19,9 +19,27 @@ jobs:
       # WASM/Pyodide has no SIMD/runtime CPU detection; disable optimised paths
       CMAKE_ARGS: "-DWITH_ZLIB_OPTIM=OFF -DWITH_OPTIM=OFF -DWITH_RUNTIME_CPU_DETECTION=OFF"
       # cibuildwheel runs the suite automatically in the node/pyodide runtime
-      CIBW_TEST_COMMAND: "pytest {project}/tests"
+      CIBW_TEST_COMMAND: >-
+        pytest {project}/tests &&
+        python {project}/scripts/certify_menudet_math.py &&
+        python {project}/scripts/verify_safe_lazyexpr.py
     steps:
       - uses: actions/checkout@v7
+      # Qualify the experimental pair without changing the shipping native pin.
+      - name: Select experimental native revision
+        if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+        id: native_ref
+        run: echo "sha=$(cat .github/menudet-native-ref)" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v7
+        if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+        with:
+          repository: Blosc/miniexpr
+          ref: ${{ steps.native_ref.outputs.sha }}
+          path: ci-native
+      - name: Override experimental native source
+        if: github.ref == 'refs/heads/numpy-compat' || github.ref == 'refs/heads/portable-jit-improvements'
+        run: |
+          echo "CMAKE_ARGS=$CMAKE_ARGS -DFETCHCONTENT_SOURCE_DIR_MINIEXPR=$PWD/ci-native" >> "$GITHUB_ENV"
       - uses: actions/setup-python@v7
         with:
           python-version: '3.x'
@@ -30,4 +48,31 @@ jobs:
       - name: Install cibuildwheel
         run: pip install "cibuildwheel==4.1.*"
       - name: Build and test wheel
-        run: cibuildwheel --platform pyodide   # platform=pyodide is CLI-only
+        run: |
+          set -o pipefail
+          cibuildwheel --platform pyodide 2>&1 | tee wasm-ci.log
+      - name: Extract safe-graph runtime evidence
+        if: always()
+        run: |
+          python - <<'PY'
+          import json
+          from pathlib import Path
+
+          log = Path("wasm-ci.log")
+          if log.exists():
+              text = log.read_text()
+              marker = '{\n  "scope": "bounded experimental safe-graph corpus;'
+              start = text.rfind(marker)
+              if start >= 0:
+                  report, _ = json.JSONDecoder().raw_decode(text[start:])
+                  Path("safe-lazyexpr-wasm.json").write_text(json.dumps(report, indent=2) + "\n")
+          PY
+      - name: Upload WASM platform evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: safe-lazyexpr-wasm
+          path: |
+            safe-lazyexpr-wasm.json
+            wasm-ci.log
+          if-no-files-found: ignore