From eb989e77ecd35b4c57321e3c7cb736cdf34fb133 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 13:22:32 +0200 Subject: [PATCH 01/49] ci: seed on-device Python runtime from python_build_run_id (android) When python_build_run_id is set, the recipe-tester currently only used that run's CROSS-COMPILE support tree; the ON-DEVICE libpython was still downloaded by serious_python from a hardcoded flet-dev/python-build RELEASE -- so a runtime fix (e.g. the 3.13+ mimalloc x86_64 seccomp open() crash) couldn't be validated on-device without publishing a release. Now: setup.sh keeps the run's python-*-dart-* tarballs in downloads/, and the android recipe-tester step seeds them into serious_python's cache ($FLET_CACHE_DIR/python-build/v/) with a fresh mtime. serious_python's download task is onlyIfModified(true), so it 304-skips and bundles THIS run's runtime. No release needed. Safe: a re-download just yields the released runtime (never a false pass). Scoped to android (where the mimalloc crash manifests, on CI's x86_64 emulator). iOS seeding (darwin cache is date-keyed + marker-based) is a follow-up for the _pyrepl leg. --- .github/workflows/build-wheels-version.yml | 22 ++++++++++++++++++++++ setup.sh | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index 20c466f2..d8508efb 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -478,6 +478,28 @@ jobs: done ./tests/recipe-tester/stage_recipe.sh "$PKG_NAME" "$PKG_VERSION" + + # Seed the on-device Python runtime from the python-build run under test. + # When python_build_run_id is set, setup.sh kept that run's dart tarballs + # in downloads/. serious_python otherwise downloads the runtime from a + # HARDCODED flet-dev/python-build release; pre-seeding its cache (keyed by + # version at $FLET_CACHE_DIR/python-build/v/) with a fresh mtime makes + # the download's onlyIfModified check 304-skip, so `flet build` bundles THIS + # run's runtime -- validating an unreleased runtime fix with no release. + # Safe: if the CDN ignores the conditional and re-downloads, you just get + # the released runtime back (same result as before, never a false pass). + if ls "$GITHUB_WORKSPACE"/downloads/python-android-dart-*.tar.gz >/dev/null 2>&1; then + export FLET_CACHE_DIR="$GITHUB_WORKSPACE/.flet-cache" + for t in "$GITHUB_WORKSPACE"/downloads/python-android-dart-*.tar.gz; do + ver=$(basename "$t" | sed -E 's/^python-android-dart-([0-9.]+)-.*/\1/') + d="$FLET_CACHE_DIR/python-build/v$ver" + mkdir -p "$d" + cp "$t" "$d/" + touch "$d/$(basename "$t")" + echo "Seeded on-device runtime: $(basename "$t") -> $d" + done + fi + cd tests/recipe-tester PIP_FIND_LINKS="$GITHUB_WORKSPACE/dist-test" \ uvx --prerelease allow --with 'flet-cli>=0.86.0.dev0' --with 'flet>=0.86.0.dev0' flet build apk -vv --yes --python-version "$(echo "$INPUT_PYTHON_VERSION" | cut -d. -f1-2)" diff --git a/setup.sh b/setup.sh index 261897af..6efc9aba 100755 --- a/setup.sh +++ b/setup.sh @@ -178,6 +178,12 @@ download_support() { return 1 fi mv "$stage/${tarball}" "downloads/${tarball}" + # Keep the on-device Python runtime tarballs (python-{android,ios}-dart-*) + # from the same artifact. serious_python normally downloads these from a + # hardcoded flet-dev/python-build RELEASE; keeping them here lets the + # recipe-tester step seed serious_python's cache with THIS run's runtime, + # so `flet build` validates an unreleased runtime fix (no release needed). + cp "$stage"/python-*-dart-*.tar.gz "downloads/" 2>/dev/null || true rm -rf "$stage" else local url="https://github.com/flet-dev/python-build/releases/download/${PYTHON_BUILD_RELEASE}/${tarball}" From c4d60f44b624527380cc7639a71ed54d6f2731cd Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 13:54:32 +0200 Subject: [PATCH 02/49] ci: seed on-device iOS runtime too (python_build_run_id) Mirror the android runtime seed for iOS. The darwin cache is date-keyed ($FLET_CACHE_DIR/python-build/v-/) and its download is a plain [ ! -f tarball ] check, so pin SERIOUS_PYTHON_BUILD_DATE to a sentinel we control (per-step, iOS job only) and drop the fixed python-ios-dart- tarball at that exact path; prepare_ios.sh finds it, skips the download, and extracts THIS run's runtime -- carrying the _pyrepl fix to the iOS 3.14 sim. Only fires when python_build_run_id supplied the tarball (setup.sh kept it); the sentinel date is never fetched because the file already exists. --- .github/workflows/build-wheels-version.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index d8508efb..da529007 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -541,6 +541,27 @@ jobs: done ./tests/recipe-tester/stage_recipe.sh "$PKG_NAME" "$PKG_VERSION" + + # Seed the on-device iOS runtime from the python-build run under test + # (see the android step for the rationale). The darwin cache is + # date-keyed (v-) and its download is a plain `[ ! -f ]` + # existence check, so we pin SERIOUS_PYTHON_BUILD_DATE to a sentinel we + # control and drop the fixed tarball at that exact cache path; prepare_ios.sh + # then finds it, skips the download, and extracts THIS run's runtime. + # Only fires when the tarball is present (python_build_run_id set); the + # sentinel is never fetched from a URL because the file already exists. + if ls "$GITHUB_WORKSPACE"/downloads/python-ios-dart-*.tar.gz >/dev/null 2>&1; then + export FLET_CACHE_DIR="$GITHUB_WORKSPACE/.flet-cache" + export SERIOUS_PYTHON_BUILD_DATE="seeded" + for t in "$GITHUB_WORKSPACE"/downloads/python-ios-dart-*.tar.gz; do + ver=$(basename "$t" | sed -E 's/^python-ios-dart-([0-9.]+)\.tar\.gz$/\1/') + d="$FLET_CACHE_DIR/python-build/v$ver-$SERIOUS_PYTHON_BUILD_DATE" + mkdir -p "$d" + cp "$t" "$d/" + echo "Seeded on-device iOS runtime: $(basename "$t") -> $d" + done + fi + cd tests/recipe-tester PIP_FIND_LINKS="$GITHUB_WORKSPACE/dist-test" \ uvx --prerelease allow --with 'flet-cli>=0.86.0.dev0' --with 'flet>=0.86.0.dev0' flet build ios-simulator -vv --yes --python-version "$(echo "$INPUT_PYTHON_VERSION" | cut -d. -f1-2)" From a53cfea137dd92bcf45cd22908c87e54fe7ae1b3 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 15:38:57 +0200 Subject: [PATCH 03/49] recipe-tester: ship path-hungry packages extracted to disk (Flet 0.86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flet 0.86 bundles site-packages as a compressed sitepackages.zip imported via zipimport. Packages that read a bundled DATA file through a real __file__ path (rather than importlib.resources) then crash on-device with NotADirectoryError, because __file__ resolves inside the zip. This regressed a batch of recipes that were green on-device before the 0.86 migration (#104). serious_python exposes an Android escape hatch — [tool.flet.android].extract_packages (flet build --android-extract-packages) — that ships listed packages extracted to disk, placed on sys.path before the zip, so __file__-relative data reads resolve. Wire it per-recipe: - pyproject.toml.tpl: new [tool.flet.android] extract_packages = [__EXTRACT_PACKAGES__] - stage_recipe.sh: expand the token from recipes//extract_packages.txt (one relative path per line; absent file => [] no-op) - recipes/{matplotlib,astropy,scikit-learn,thinc,spacy}/extract_packages.txt Verified on-device (android arm64, flet 0.86.0.dev2): - matplotlib: NotADirectoryError(matplotlib/mpl-data/matplotlibrc) -> 1 passed - astropy: NotADirectoryError(astropy/CITATION) -> 5 passed scikit-learn (sklearn/utils/_repr_html/estimator.css), thinc (thinc/backends/_custom_kernels.cu) and spacy (imports thinc) use the identical data-file-inside-package pattern and are covered by the same mechanism. Not covered by extract_packages (separate 0.86 issues, tracked separately): - python-magic: magic.mgc lives in flet-libmagic's opt/ tree; serious_python's copyOpt copies only **/*.so, so the data file is dropped entirely. - opencv-python: cv2's loader requires the literal config.py SOURCE, which 0.86 strips when compiling bundled packages to .pyc. --- recipes/astropy/extract_packages.txt | 2 ++ recipes/matplotlib/extract_packages.txt | 4 +++ recipes/scikit-learn/extract_packages.txt | 2 ++ recipes/spacy/extract_packages.txt | 3 ++ recipes/thinc/extract_packages.txt | 2 ++ tests/recipe-tester/pyproject.toml.tpl | 10 +++++++ tests/recipe-tester/stage_recipe.sh | 35 +++++++++++++++++++++++ 7 files changed, 58 insertions(+) create mode 100644 recipes/astropy/extract_packages.txt create mode 100644 recipes/matplotlib/extract_packages.txt create mode 100644 recipes/scikit-learn/extract_packages.txt create mode 100644 recipes/spacy/extract_packages.txt create mode 100644 recipes/thinc/extract_packages.txt diff --git a/recipes/astropy/extract_packages.txt b/recipes/astropy/extract_packages.txt new file mode 100644 index 00000000..90332354 --- /dev/null +++ b/recipes/astropy/extract_packages.txt @@ -0,0 +1,2 @@ +# astropy reads astropy/CITATION via __file__ at import (Flet 0.86 sitepackages.zip). +astropy diff --git a/recipes/matplotlib/extract_packages.txt b/recipes/matplotlib/extract_packages.txt new file mode 100644 index 00000000..4735f871 --- /dev/null +++ b/recipes/matplotlib/extract_packages.txt @@ -0,0 +1,4 @@ +# Packages shipped extracted to disk (see tests/recipe-tester/stage_recipe.sh). +# matplotlib reads mpl-data/matplotlibrc (and fonts/styles) via a real __file__ +# path at import, which fails inside Flet 0.86's sitepackages.zip. +matplotlib diff --git a/recipes/scikit-learn/extract_packages.txt b/recipes/scikit-learn/extract_packages.txt new file mode 100644 index 00000000..400387ae --- /dev/null +++ b/recipes/scikit-learn/extract_packages.txt @@ -0,0 +1,2 @@ +# scikit-learn reads sklearn/utils/_repr_html/estimator.css via __file__ at import. +sklearn diff --git a/recipes/spacy/extract_packages.txt b/recipes/spacy/extract_packages.txt new file mode 100644 index 00000000..3e32adfb --- /dev/null +++ b/recipes/spacy/extract_packages.txt @@ -0,0 +1,3 @@ +# spacy imports thinc (-> _custom_kernels.cu) and reads its own lang data via __file__. +spacy +thinc diff --git a/recipes/thinc/extract_packages.txt b/recipes/thinc/extract_packages.txt new file mode 100644 index 00000000..d99ae784 --- /dev/null +++ b/recipes/thinc/extract_packages.txt @@ -0,0 +1,2 @@ +# thinc reads thinc/backends/_custom_kernels.cu via __file__ at import. +thinc diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 21581404..4ba94d1d 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -31,3 +31,13 @@ app = false [tool.flet.app] path = "." + +# Flet 0.86 ships site-packages inside a compressed `sitepackages.zip` (imported +# via zipimport). Packages that read a bundled DATA file through a real `__file__` +# path (rather than `importlib.resources`) then fail on-device with +# `NotADirectoryError` because the parent is a zip, not a directory. List such +# "path-hungry" packages here to ship them extracted to disk instead. Populated +# per-recipe by `stage_recipe.sh` from `recipes//extract_packages.txt` +# (empty `[]` — the default — is a no-op). +[tool.flet.android] +extract_packages = [__EXTRACT_PACKAGES__] diff --git a/tests/recipe-tester/stage_recipe.sh b/tests/recipe-tester/stage_recipe.sh index 65d1b23d..c668ccf1 100755 --- a/tests/recipe-tester/stage_recipe.sh +++ b/tests/recipe-tester/stage_recipe.sh @@ -77,6 +77,36 @@ if [ -f "$REQS_FILE" ]; then done < "$REQS_FILE" fi +# Path-hungry packages to ship EXTRACTED to disk instead of inside Flet 0.86's +# compressed sitepackages.zip — those that read bundled data via a real __file__ +# path (rather than importlib.resources) and otherwise crash on-device with +# NotADirectoryError. One relative path per line in recipes//extract_packages.txt +# (blanks and full-line comments skipped); emitted into +# [tool.flet.android].extract_packages as a TOML array. Absent file => [] (no-op). +EXTRACT_FILE="$RECIPE_DIR/extract_packages.txt" +EXTRACT_ENTRIES=() +if [ -f "$EXTRACT_FILE" ]; then + while IFS= read -r line || [ -n "$line" ]; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [ -z "$line" ] && continue + [ "${line:0:1}" = "#" ] && continue + # Entries become double-quoted TOML strings; a literal double quote in + # a path would end the string. Package names / relative paths never have one. + if [[ "$line" == *'"'* ]]; then + echo "::error::double quotes are not supported in $EXTRACT_FILE: $line" >&2 + exit 1 + fi + EXTRACT_ENTRIES+=("$line") + done < "$EXTRACT_FILE" +fi +# Comma-joined TOML array body, e.g. "matplotlib", "thinc" (empty when no file). +EXTRACT_LIST="" +for e in ${EXTRACT_ENTRIES[@]+"${EXTRACT_ENTRIES[@]}"}; do + [ -n "$EXTRACT_LIST" ] && EXTRACT_LIST="$EXTRACT_LIST, " + EXTRACT_LIST="$EXTRACT_LIST\"$e\"" +done + # Expand the template line-by-line with printf '%s' rather than sed: the # replacement text (PEP 508 specs) may contain characters that are unsafe in # a sed RHS, and BSD/GNU sed disagree on escaping rules. @@ -96,6 +126,8 @@ while IFS= read -r tpl_line || [ -n "$tpl_line" ]; do for dep in ${TEST_DEPS[@]+"${TEST_DEPS[@]}"}; do printf " '%s',\n" "$dep" >> "$OUT" done + elif [[ "$tpl_line" == *"__EXTRACT_PACKAGES__"* ]]; then + printf '%s\n' "${tpl_line/__EXTRACT_PACKAGES__/$EXTRACT_LIST}" >> "$OUT" else printf '%s\n' "$tpl_line" >> "$OUT" fi @@ -108,6 +140,9 @@ echo " pyproject.toml: generated (gitignored)" if [ ${#TEST_DEPS[@]} -gt 0 ]; then echo " test-only deps (tests/requirements.txt): ${TEST_DEPS[*]}" fi +if [ ${#EXTRACT_ENTRIES[@]} -gt 0 ]; then + echo " extract-to-disk (extract_packages.txt): ${EXTRACT_ENTRIES[*]}" +fi echo "" echo "Next:" echo " cd $(realpath --relative-to="$PWD" "$SCRIPT_DIR" 2>/dev/null || echo "$SCRIPT_DIR")" From 9c6354f7a0699e36ddc2ffc1266441616e53cf75 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 15:39:04 +0200 Subject: [PATCH 04/49] refactor: centralize version variables in meta.yaml for multiple recipes Replaced hardcoded version strings with `{% set version %}` variables across several recipes to streamline maintenance and update consistency. --- recipes/flet-libcrc32c/meta.yaml | 6 ++++-- recipes/flet-libfreetype/meta.yaml | 6 ++++-- recipes/flet-libgeos/meta.yaml | 6 ++++-- recipes/flet-libjpeg/meta.yaml | 6 ++++-- recipes/flet-libpng/meta.yaml | 6 ++++-- recipes/flet-libpyjni/meta.yaml | 6 ++++-- recipes/flet-libyaml/meta.yaml | 6 ++++-- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/recipes/flet-libcrc32c/meta.yaml b/recipes/flet-libcrc32c/meta.yaml index 6d5e9a78..dd5bb7b5 100644 --- a/recipes/flet-libcrc32c/meta.yaml +++ b/recipes/flet-libcrc32c/meta.yaml @@ -1,12 +1,14 @@ +{% set version = "1.1.2" %} + package: name: flet-libcrc32c - version: "1.1.2" + version: '{{ version }}' build: number: 10 source: - url: https://github.com/google/crc32c/archive/refs/tags/1.1.2.tar.gz + url: https://github.com/google/crc32c/archive/refs/tags/{{ version }}.tar.gz requirements: build: diff --git a/recipes/flet-libfreetype/meta.yaml b/recipes/flet-libfreetype/meta.yaml index 2ffbb78d..c78a9018 100644 --- a/recipes/flet-libfreetype/meta.yaml +++ b/recipes/flet-libfreetype/meta.yaml @@ -1,12 +1,14 @@ +{% set version = "2.13.3" %} + package: name: flet-libfreetype - version: "2.13.3" + version: '{{ version }}' build: number: 10 source: - url: https://downloads.sourceforge.net/project/freetype/freetype2/2.13.3/freetype-2.13.3.tar.gz + url: https://downloads.sourceforge.net/project/freetype/freetype2/{{ version }}/freetype-{{ version }}.tar.gz patches: - config.patch diff --git a/recipes/flet-libgeos/meta.yaml b/recipes/flet-libgeos/meta.yaml index a19b9459..3b59c5f1 100644 --- a/recipes/flet-libgeos/meta.yaml +++ b/recipes/flet-libgeos/meta.yaml @@ -1,12 +1,14 @@ +{% set version = "3.13.1" %} + package: name: flet-libgeos - version: "3.13.1" + version: '{{ version }}' build: number: 1 source: - url: http://download.osgeo.org/geos/geos-3.13.1.tar.bz2 + url: http://download.osgeo.org/geos/geos-{{ version }}.tar.bz2 requirements: build: diff --git a/recipes/flet-libjpeg/meta.yaml b/recipes/flet-libjpeg/meta.yaml index d78c02f5..be44b418 100644 --- a/recipes/flet-libjpeg/meta.yaml +++ b/recipes/flet-libjpeg/meta.yaml @@ -1,9 +1,11 @@ +{% set version = "3.0.90" %} + package: name: flet-libjpeg - version: "3.0.90" + version: '{{ version }}' source: - url: https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.0.90/libjpeg-turbo-3.0.90.tar.gz + url: https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/{{ version }}/libjpeg-turbo-{{ version }}.tar.gz build: number: 10 diff --git a/recipes/flet-libpng/meta.yaml b/recipes/flet-libpng/meta.yaml index ed6941c8..c78dd14a 100644 --- a/recipes/flet-libpng/meta.yaml +++ b/recipes/flet-libpng/meta.yaml @@ -1,12 +1,14 @@ +{% set version = "1.6.43" %} + package: name: flet-libpng - version: "1.6.43" + version: '{{ version }}' build: number: 10 source: - url: https://github.com/pnggroup/libpng/archive/refs/tags/v1.6.43.tar.gz + url: https://github.com/pnggroup/libpng/archive/refs/tags/v{{ version }}.tar.gz patches: - config.patch \ No newline at end of file diff --git a/recipes/flet-libpyjni/meta.yaml b/recipes/flet-libpyjni/meta.yaml index 6ae7c229..f415b53e 100644 --- a/recipes/flet-libpyjni/meta.yaml +++ b/recipes/flet-libpyjni/meta.yaml @@ -1,6 +1,8 @@ +{% set version = "1.0.1" %} + package: name: flet-libpyjni - version: "1.0.1" + version: '{{ version }}' # JNI bridge to the Android JVM — no equivalent on iOS, build can't # succeed there. (Mirrors flet-libcpp-shared's platforms gate.) platforms: [android] @@ -9,7 +11,7 @@ build: number: 10 source: - url: https://github.com/flet-dev/libpyjni/releases/download/v1.0.1/pyjni-1.0.1.tar.gz + url: https://github.com/flet-dev/libpyjni/releases/download/v{{ version }}/pyjni-{{ version }}.tar.gz requirements: build: diff --git a/recipes/flet-libyaml/meta.yaml b/recipes/flet-libyaml/meta.yaml index 569bbda3..3a1909ab 100644 --- a/recipes/flet-libyaml/meta.yaml +++ b/recipes/flet-libyaml/meta.yaml @@ -1,9 +1,11 @@ +{% set version = "0.2.5" %} + package: name: flet-libyaml - version: "0.2.5" + version: '{{ version }}' build: number: 10 source: - url: https://github.com/yaml/libyaml/releases/download/0.2.5/yaml-0.2.5.tar.gz + url: https://github.com/yaml/libyaml/releases/download/{{ version }}/yaml-{{ version }}.tar.gz From b659f190b24ab8a97a0cfa30ff850cfc615a8f62 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 15:59:38 +0200 Subject: [PATCH 05/49] recipe-tester: declare extract_packages in meta.yaml, not a sidecar file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the path-hungry extract list from the recipes//extract_packages.txt sidecar (previous commit) into a first-class `extract_packages:` field in the recipe's meta.yaml — its natural home next to patches/requirements, and one fewer per-recipe file convention. - schema/meta-schema.yaml: document `extract_packages` (array of strings, default []); forge's build ignores it, the recipe-tester consumes it. - read_extract_packages.py: render meta.yaml the way forge does (Jinja -> YAML) and print the list; run hermetically via `uv run --with jinja2 --with pyyaml` so the deps are present regardless of the caller's environment. - stage_recipe.sh: read the field via that helper instead of the sidecar file. - move matplotlib/astropy/scikit-learn/thinc/spacy lists into their meta.yaml; delete the 5 sidecars. Output is unchanged: the generated pyproject.toml is byte-identical to the sidecar version (matplotlib -> ["matplotlib"], spacy -> ["spacy","thinc"], numpy -> []), the schema still validates metas with and without the field, and matplotlib re-verified green on-device (arm64, flet 0.86.0.dev2). --- recipes/astropy/extract_packages.txt | 2 - recipes/astropy/meta.yaml | 4 ++ recipes/matplotlib/extract_packages.txt | 4 -- recipes/matplotlib/meta.yaml | 5 +++ recipes/scikit-learn/extract_packages.txt | 2 - recipes/scikit-learn/meta.yaml | 4 ++ recipes/spacy/extract_packages.txt | 3 -- recipes/spacy/meta.yaml | 6 +++ recipes/thinc/extract_packages.txt | 2 - recipes/thinc/meta.yaml | 4 ++ src/forge/schema/meta-schema.yaml | 15 ++++++++ tests/recipe-tester/pyproject.toml.tpl | 2 +- tests/recipe-tester/read_extract_packages.py | 31 +++++++++++++++ tests/recipe-tester/stage_recipe.sh | 40 ++++++++++---------- 14 files changed, 90 insertions(+), 34 deletions(-) delete mode 100644 recipes/astropy/extract_packages.txt delete mode 100644 recipes/matplotlib/extract_packages.txt delete mode 100644 recipes/scikit-learn/extract_packages.txt delete mode 100644 recipes/spacy/extract_packages.txt delete mode 100644 recipes/thinc/extract_packages.txt create mode 100644 tests/recipe-tester/read_extract_packages.py diff --git a/recipes/astropy/extract_packages.txt b/recipes/astropy/extract_packages.txt deleted file mode 100644 index 90332354..00000000 --- a/recipes/astropy/extract_packages.txt +++ /dev/null @@ -1,2 +0,0 @@ -# astropy reads astropy/CITATION via __file__ at import (Flet 0.86 sitepackages.zip). -astropy diff --git a/recipes/astropy/meta.yaml b/recipes/astropy/meta.yaml index fe495550..fe514dbb 100644 --- a/recipes/astropy/meta.yaml +++ b/recipes/astropy/meta.yaml @@ -2,6 +2,10 @@ package: name: astropy version: "8.0.0" +# astropy reads astropy/CITATION via a real __file__ path at import. +extract_packages: + - astropy + build: number: 1 diff --git a/recipes/matplotlib/extract_packages.txt b/recipes/matplotlib/extract_packages.txt deleted file mode 100644 index 4735f871..00000000 --- a/recipes/matplotlib/extract_packages.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Packages shipped extracted to disk (see tests/recipe-tester/stage_recipe.sh). -# matplotlib reads mpl-data/matplotlibrc (and fonts/styles) via a real __file__ -# path at import, which fails inside Flet 0.86's sitepackages.zip. -matplotlib diff --git a/recipes/matplotlib/meta.yaml b/recipes/matplotlib/meta.yaml index 55014450..49e9f1bb 100644 --- a/recipes/matplotlib/meta.yaml +++ b/recipes/matplotlib/meta.yaml @@ -2,6 +2,11 @@ package: name: matplotlib version: "3.10.9" +# matplotlib reads mpl-data/matplotlibrc (+ fonts/styles) via a real __file__ +# path at import; extract it so those reads resolve outside Flet 0.86's zip. +extract_packages: + - matplotlib + requirements: build: - ninja diff --git a/recipes/scikit-learn/extract_packages.txt b/recipes/scikit-learn/extract_packages.txt deleted file mode 100644 index 400387ae..00000000 --- a/recipes/scikit-learn/extract_packages.txt +++ /dev/null @@ -1,2 +0,0 @@ -# scikit-learn reads sklearn/utils/_repr_html/estimator.css via __file__ at import. -sklearn diff --git a/recipes/scikit-learn/meta.yaml b/recipes/scikit-learn/meta.yaml index 0d220cf0..96cba40b 100644 --- a/recipes/scikit-learn/meta.yaml +++ b/recipes/scikit-learn/meta.yaml @@ -2,6 +2,10 @@ package: name: scikit-learn version: "1.9.0" +# scikit-learn reads sklearn/utils/_repr_html/estimator.css via __file__ at import. +extract_packages: + - sklearn + build: number: 1 backend-args: diff --git a/recipes/spacy/extract_packages.txt b/recipes/spacy/extract_packages.txt deleted file mode 100644 index 3e32adfb..00000000 --- a/recipes/spacy/extract_packages.txt +++ /dev/null @@ -1,3 +0,0 @@ -# spacy imports thinc (-> _custom_kernels.cu) and reads its own lang data via __file__. -spacy -thinc diff --git a/recipes/spacy/meta.yaml b/recipes/spacy/meta.yaml index ba2a07e5..619c693b 100644 --- a/recipes/spacy/meta.yaml +++ b/recipes/spacy/meta.yaml @@ -2,6 +2,12 @@ package: name: spacy version: "3.8.13" +# spacy imports thinc (-> thinc/backends/_custom_kernels.cu) and reads its own +# lang data via __file__; extract both. +extract_packages: + - spacy + - thinc + build: number: 1 diff --git a/recipes/thinc/extract_packages.txt b/recipes/thinc/extract_packages.txt deleted file mode 100644 index d99ae784..00000000 --- a/recipes/thinc/extract_packages.txt +++ /dev/null @@ -1,2 +0,0 @@ -# thinc reads thinc/backends/_custom_kernels.cu via __file__ at import. -thinc diff --git a/recipes/thinc/meta.yaml b/recipes/thinc/meta.yaml index 342b8e70..a320a784 100644 --- a/recipes/thinc/meta.yaml +++ b/recipes/thinc/meta.yaml @@ -2,6 +2,10 @@ package: name: thinc version: "8.3.13" +# thinc reads thinc/backends/_custom_kernels.cu via __file__ at import. +extract_packages: + - thinc + build: number: 1 diff --git a/src/forge/schema/meta-schema.yaml b/src/forge/schema/meta-schema.yaml index 4679eb41..36f8ec60 100644 --- a/src/forge/schema/meta-schema.yaml +++ b/src/forge/schema/meta-schema.yaml @@ -93,6 +93,21 @@ properties: Patches to apply to the code. Each entry is a filename in the `patches` folder of the recipe. + extract_packages: + type: array + default: [] + items: + type: string + description: >- + Android only. Import-relative paths (usually the package's import name) + to ship EXTRACTED to disk instead of inside Flet 0.86's compressed + `sitepackages.zip`. Needed for "path-hungry" packages that read bundled + DATA files through a real `__file__` path (rather than + `importlib.resources`) — inside the zip such reads fail with + `NotADirectoryError`. Consumed by the recipe-tester + (`tests/recipe-tester/stage_recipe.sh` -> flet build + `[tool.flet.android].extract_packages`); forge's build itself ignores it. + build: type: object default: {} diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 4ba94d1d..08f78713 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -37,7 +37,7 @@ path = "." # path (rather than `importlib.resources`) then fail on-device with # `NotADirectoryError` because the parent is a zip, not a directory. List such # "path-hungry" packages here to ship them extracted to disk instead. Populated -# per-recipe by `stage_recipe.sh` from `recipes//extract_packages.txt` +# per-recipe by `stage_recipe.sh` from each recipe's meta.yaml `extract_packages:` # (empty `[]` — the default — is a no-op). [tool.flet.android] extract_packages = [__EXTRACT_PACKAGES__] diff --git a/tests/recipe-tester/read_extract_packages.py b/tests/recipe-tester/read_extract_packages.py new file mode 100644 index 00000000..dd71354a --- /dev/null +++ b/tests/recipe-tester/read_extract_packages.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Print a recipe meta.yaml's top-level ``extract_packages`` list, one per line. + +Used by ``stage_recipe.sh`` to feed ``[tool.flet.android].extract_packages`` in +the generated recipe-tester ``pyproject.toml``. The meta is Jinja-then-YAML like +forge loads it (``forge/package.py``), so a version-templated meta still parses. +Run hermetically, e.g.:: + + uv run --no-project --with jinja2 --with pyyaml python3 read_extract_packages.py +""" +import sys + +import jinja2 +import yaml + +meta_path = sys.argv[1] +text = open(meta_path, encoding="utf-8").read() + +# Same render inputs forge uses; extract_packages is SDK-independent, but pass a +# concrete context so metas that Jinja-branch on sdk/arch/version still render. +rendered = jinja2.Template(text).render( + sdk="android", + sdk_version="24", + arch="arm64-v8a", + version=None, + py_version=sys.version_info, +) +meta = yaml.safe_load(rendered) or {} + +for entry in meta.get("extract_packages") or []: + print(entry) diff --git a/tests/recipe-tester/stage_recipe.sh b/tests/recipe-tester/stage_recipe.sh index c668ccf1..12e48600 100755 --- a/tests/recipe-tester/stage_recipe.sh +++ b/tests/recipe-tester/stage_recipe.sh @@ -80,27 +80,27 @@ fi # Path-hungry packages to ship EXTRACTED to disk instead of inside Flet 0.86's # compressed sitepackages.zip — those that read bundled data via a real __file__ # path (rather than importlib.resources) and otherwise crash on-device with -# NotADirectoryError. One relative path per line in recipes//extract_packages.txt -# (blanks and full-line comments skipped); emitted into -# [tool.flet.android].extract_packages as a TOML array. Absent file => [] (no-op). -EXTRACT_FILE="$RECIPE_DIR/extract_packages.txt" +# NotADirectoryError. Declared as the recipe's `extract_packages:` list in +# meta.yaml; emitted into [tool.flet.android].extract_packages as a TOML array. +# Read via read_extract_packages.py (Jinja-then-YAML, like forge loads meta), run +# hermetically with uv so jinja2/pyyaml are present regardless of the caller env. EXTRACT_ENTRIES=() -if [ -f "$EXTRACT_FILE" ]; then - while IFS= read -r line || [ -n "$line" ]; do - line="${line#"${line%%[![:space:]]*}"}" - line="${line%"${line##*[![:space:]]}"}" - [ -z "$line" ] && continue - [ "${line:0:1}" = "#" ] && continue - # Entries become double-quoted TOML strings; a literal double quote in - # a path would end the string. Package names / relative paths never have one. - if [[ "$line" == *'"'* ]]; then - echo "::error::double quotes are not supported in $EXTRACT_FILE: $line" >&2 - exit 1 - fi - EXTRACT_ENTRIES+=("$line") - done < "$EXTRACT_FILE" +if ! EXTRACT_RAW="$(uv run --no-project --quiet --with jinja2 --with pyyaml \ + python3 "$SCRIPT_DIR/read_extract_packages.py" "$RECIPE_DIR/meta.yaml")"; then + echo "::error::failed to read extract_packages from $RECIPE_DIR/meta.yaml" >&2 + exit 1 fi -# Comma-joined TOML array body, e.g. "matplotlib", "thinc" (empty when no file). +while IFS= read -r line || [ -n "$line" ]; do + [ -z "$line" ] && continue + # Entries become double-quoted TOML strings; a literal double quote in a path + # would end the string. Package names / relative paths never have one. + if [[ "$line" == *'"'* ]]; then + echo "::error::double quotes are not supported in extract_packages: $line" >&2 + exit 1 + fi + EXTRACT_ENTRIES+=("$line") +done <<< "$EXTRACT_RAW" +# Comma-joined TOML array body, e.g. "matplotlib", "thinc" (empty when none declared). EXTRACT_LIST="" for e in ${EXTRACT_ENTRIES[@]+"${EXTRACT_ENTRIES[@]}"}; do [ -n "$EXTRACT_LIST" ] && EXTRACT_LIST="$EXTRACT_LIST, " @@ -141,7 +141,7 @@ if [ ${#TEST_DEPS[@]} -gt 0 ]; then echo " test-only deps (tests/requirements.txt): ${TEST_DEPS[*]}" fi if [ ${#EXTRACT_ENTRIES[@]} -gt 0 ]; then - echo " extract-to-disk (extract_packages.txt): ${EXTRACT_ENTRIES[*]}" + echo " extract-to-disk (meta.yaml extract_packages): ${EXTRACT_ENTRIES[*]}" fi echo "" echo "Next:" From 5aef19863a4b4aa33e073cd4cee0f76c0e0bdb46 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 16:22:33 +0200 Subject: [PATCH 06/49] recipe-tester: move test-only deps into meta.yaml `test.requires` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a conda-style `test:` section to the recipe schema (forge's schema is "a subset of conda syntax", which has a `test:` block) and reads test-only deps from `test.requires` instead of a per-recipe `tests/requirements.txt` — one declarative config surface (meta.yaml), with `tests/` left as pure test code. - schema/meta-schema.yaml: new `test.requires` (array of PEP 508 specs, default []). - read_extract_packages.py -> read_meta_list.py: generalized to print any list-valued meta field by dotted key (`extract_packages`, `test.requires`), rendered Jinja->YAML the way forge loads meta. - stage_recipe.sh: read both lists via a shared `meta_list()` helper; a leftover tests/requirements.txt now errors (pointing at test.requires) instead of being silently ignored. No recipes currently declare test deps (no tests/requirements.txt existed), so nothing to migrate. Verified: schema validates with and without the field; the generated pyproject injects test.requires into `dependencies` and keeps extract_packages, structurally identical to the prior requirements.txt path; the legacy guard fires. extract_packages stays TOP-LEVEL — it is a deployment property of the wheel (a real app that bundles the package needs it too), not a test-only concern. --- src/forge/schema/meta-schema.yaml | 21 +++++++ tests/recipe-tester/pyproject.toml.tpl | 2 +- tests/recipe-tester/read_extract_packages.py | 31 ---------- tests/recipe-tester/read_meta_list.py | 40 +++++++++++++ tests/recipe-tester/stage_recipe.sh | 61 ++++++++++---------- 5 files changed, 93 insertions(+), 62 deletions(-) delete mode 100644 tests/recipe-tester/read_extract_packages.py create mode 100644 tests/recipe-tester/read_meta_list.py diff --git a/src/forge/schema/meta-schema.yaml b/src/forge/schema/meta-schema.yaml index 36f8ec60..5db919aa 100644 --- a/src/forge/schema/meta-schema.yaml +++ b/src/forge/schema/meta-schema.yaml @@ -201,6 +201,27 @@ properties: additionalProperties: false + test: + type: object + default: {} + properties: + requires: + type: array + default: [] + items: + type: string + description: >- + Test-only dependencies: packages the recipe's tests import but the + package itself does NOT declare in Requires-Dist (e.g. numpy for a + recipe whose numpy integration is extra-gated upstream). PEP 508 + specs; each must resolve for the MOBILE target (pure-Python from PyPI, + or a recipe published on pypi.flet.dev / seeded into dist/) — the same + constraint a real app faces. Injected into the recipe-tester's + generated pyproject `dependencies` by + `tests/recipe-tester/stage_recipe.sh`. Replaces the former + per-recipe `tests/requirements.txt`. + additionalProperties: false + about: type: object default: {} diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 08f78713..2e233dd1 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -9,7 +9,7 @@ dependencies = [ "pytest", # `stage_recipe.sh` rewrites the line below to pin the recipe under test (e.g. `"numpy==2.2.2"`), # and replaces the token line after it with any test-only deps declared in - # the recipe's tests/requirements.txt (nothing emitted when the file is absent). + # the recipe's meta.yaml `test.requires` (nothing emitted when none declared). "__RECIPE_DEP__", __TEST_DEPS__ ] diff --git a/tests/recipe-tester/read_extract_packages.py b/tests/recipe-tester/read_extract_packages.py deleted file mode 100644 index dd71354a..00000000 --- a/tests/recipe-tester/read_extract_packages.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Print a recipe meta.yaml's top-level ``extract_packages`` list, one per line. - -Used by ``stage_recipe.sh`` to feed ``[tool.flet.android].extract_packages`` in -the generated recipe-tester ``pyproject.toml``. The meta is Jinja-then-YAML like -forge loads it (``forge/package.py``), so a version-templated meta still parses. -Run hermetically, e.g.:: - - uv run --no-project --with jinja2 --with pyyaml python3 read_extract_packages.py -""" -import sys - -import jinja2 -import yaml - -meta_path = sys.argv[1] -text = open(meta_path, encoding="utf-8").read() - -# Same render inputs forge uses; extract_packages is SDK-independent, but pass a -# concrete context so metas that Jinja-branch on sdk/arch/version still render. -rendered = jinja2.Template(text).render( - sdk="android", - sdk_version="24", - arch="arm64-v8a", - version=None, - py_version=sys.version_info, -) -meta = yaml.safe_load(rendered) or {} - -for entry in meta.get("extract_packages") or []: - print(entry) diff --git a/tests/recipe-tester/read_meta_list.py b/tests/recipe-tester/read_meta_list.py new file mode 100644 index 00000000..6afef8a7 --- /dev/null +++ b/tests/recipe-tester/read_meta_list.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Print a list-valued field from a recipe meta.yaml, one item per line. + +Usage: + read_meta_list.py + +e.g. `extract_packages` or `test.requires`. The meta is rendered +Jinja-then-YAML the way forge loads it (`forge/package.py`), so a +version-templated meta still parses. A missing key (at any level) or a non-list +value prints nothing. Run hermetically so jinja2/pyyaml are present regardless of +the caller's environment:: + + uv run --no-project --with jinja2 --with pyyaml python3 read_meta_list.py +""" + +import sys + +import jinja2 +import yaml + +meta_path, dotted_key = sys.argv[1], sys.argv[2] +text = open(meta_path, encoding="utf-8").read() + +# Same render inputs forge uses; the fields we read are SDK-independent, but pass +# a concrete context so metas that Jinja-branch on sdk/arch/version still render. +rendered = jinja2.Template(text).render( + sdk="android", + sdk_version="24", + arch="arm64-v8a", + version=None, + py_version=sys.version_info, +) + +node = yaml.safe_load(rendered) or {} +for key in dotted_key.split("."): + node = node.get(key) if isinstance(node, dict) else None + +if isinstance(node, list): + for item in node: + print(item) diff --git a/tests/recipe-tester/stage_recipe.sh b/tests/recipe-tester/stage_recipe.sh index 12e48600..8e02a317 100755 --- a/tests/recipe-tester/stage_recipe.sh +++ b/tests/recipe-tester/stage_recipe.sh @@ -47,49 +47,50 @@ fi # 2. Generate pyproject.toml from the template (gitignored): pin the recipe # under test (__RECIPE_DEP__) and expand test-only deps (__TEST_DEPS__) -# from the recipe's optional tests/requirements.txt. +# from the recipe's meta.yaml `test.requires`. DEP="$RECIPE" [ -n "$VERSION" ] && DEP="$RECIPE==$VERSION" +# Declarative lists live in the recipe's meta.yaml; read them the way forge loads +# meta (Jinja -> YAML) via read_meta_list.py, run hermetically with uv so +# jinja2/pyyaml are present regardless of the caller's environment. +meta_list() { # $1 = dotted key -> one item per line on stdout + uv run --no-project --quiet --with jinja2 --with pyyaml \ + python3 "$SCRIPT_DIR/read_meta_list.py" "$RECIPE_DIR/meta.yaml" "$1" +} + # Test-only deps: packages the tests import that are NOT in the recipe's -# Requires-Dist (e.g. numpy for a zero-runtime-dep recipe like safetensors, -# whose numpy integration is extra-gated upstream). One PEP 508 spec per -# line; blanks and full-line comments are skipped. Each dep must resolve for +# Requires-Dist (e.g. numpy for a recipe whose numpy integration is extra-gated +# upstream). PEP 508 specs from meta.yaml `test.requires`; each must resolve for # the MOBILE target — pure-Python from PyPI, or a recipe published on # pypi.flet.dev (or seeded into dist/) — the same constraint a real app faces. -REQS_FILE="$TEST_DIR/requirements.txt" -TEST_DEPS=() -if [ -f "$REQS_FILE" ]; then - while IFS= read -r line || [ -n "$line" ]; do - # ltrim/rtrim, then skip blanks and full-line comments - line="${line#"${line%%[![:space:]]*}"}" - line="${line%"${line##*[![:space:]]}"}" - [ -z "$line" ] && continue - [ "${line:0:1}" = "#" ] && continue - # Deps are emitted as TOML literal (single-quoted) strings so PEP 508 - # markers — which legitimately contain double quotes — pass through - # verbatim; a single quote inside the spec would end the TOML string. - if [[ "$line" == *"'"* ]]; then - echo "::error::single quotes are not supported in $REQS_FILE: $line" >&2 - exit 1 - fi - TEST_DEPS+=("$line") - done < "$REQS_FILE" +if ! TEST_REQ_RAW="$(meta_list test.requires)"; then + echo "::error::failed to read test.requires from $RECIPE_DIR/meta.yaml" >&2 + exit 1 fi +TEST_DEPS=() +while IFS= read -r line || [ -n "$line" ]; do + [ -z "$line" ] && continue + # Deps are emitted as TOML literal (single-quoted) strings so PEP 508 markers + # — which legitimately contain double quotes — pass through verbatim; a single + # quote inside the spec would end the TOML string. + if [[ "$line" == *"'"* ]]; then + echo "::error::single quotes are not supported in test.requires: $line" >&2 + exit 1 + fi + TEST_DEPS+=("$line") +done <<< "$TEST_REQ_RAW" # Path-hungry packages to ship EXTRACTED to disk instead of inside Flet 0.86's # compressed sitepackages.zip — those that read bundled data via a real __file__ # path (rather than importlib.resources) and otherwise crash on-device with -# NotADirectoryError. Declared as the recipe's `extract_packages:` list in -# meta.yaml; emitted into [tool.flet.android].extract_packages as a TOML array. -# Read via read_extract_packages.py (Jinja-then-YAML, like forge loads meta), run -# hermetically with uv so jinja2/pyyaml are present regardless of the caller env. -EXTRACT_ENTRIES=() -if ! EXTRACT_RAW="$(uv run --no-project --quiet --with jinja2 --with pyyaml \ - python3 "$SCRIPT_DIR/read_extract_packages.py" "$RECIPE_DIR/meta.yaml")"; then +# NotADirectoryError. Declared as the recipe's top-level `extract_packages:` list +# in meta.yaml; emitted into [tool.flet.android].extract_packages as a TOML array. +if ! EXTRACT_RAW="$(meta_list extract_packages)"; then echo "::error::failed to read extract_packages from $RECIPE_DIR/meta.yaml" >&2 exit 1 fi +EXTRACT_ENTRIES=() while IFS= read -r line || [ -n "$line" ]; do [ -z "$line" ] && continue # Entries become double-quoted TOML strings; a literal double quote in a path @@ -138,7 +139,7 @@ echo " recipe_tests/:" ls -1 "$TEST_DIR" | sed 's/^/ /' echo " pyproject.toml: generated (gitignored)" if [ ${#TEST_DEPS[@]} -gt 0 ]; then - echo " test-only deps (tests/requirements.txt): ${TEST_DEPS[*]}" + echo " test-only deps (meta.yaml test.requires): ${TEST_DEPS[*]}" fi if [ ${#EXTRACT_ENTRIES[@]} -gt 0 ]; then echo " extract-to-disk (meta.yaml extract_packages): ${EXTRACT_ENTRIES[*]}" From 25a2f29036c3f183a4f70c2b7b8aa2cf81dc2e34 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 17:07:51 +0200 Subject: [PATCH 07/49] forge: ABI-tag bare CPython extension modules in fix_wheel (Android) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flet 0.86's serious_python Android packaging only relocates a native module into jniLibs and writes the `.soref` marker its on-device importer needs for extensions whose filename carries a CPython ABI tag (`*.cpython-*.so` / `*.abi3.so`). A bare `NAME.so` is treated as a plain dependency library, gets no `.soref`, and fails to import on-device with ModuleNotFoundError. CMake / SWIG / Cython / nanobind builds routinely emit un-tagged extensions because they can't derive the target SOABI when cross-compiling — this broke ncnn (ncnn.ncnn), faiss-cpu (_swigfaiss), coolprop (CoolProp/_constants) and onnxruntime (x86_64 leg only) on 0.86. fix_wheel now renames each bare `.so` that exports `PyInit_` (a genuine CPython extension) to `.cpython-3X.so`, using llvm-nm to distinguish extensions from real dependency libraries (left untouched). One fix covers the whole class — present and future — instead of per-recipe CMake patches. Verified: - on-device (android arm64): a manually ABI-tagged ncnn wheel -> serious_python relocates it (lib/.../libncnn-ncnn.so + ncnn/ncnn.soref) and `import ncnn` works; recipe-tester ncnn 3/3 passed (was ModuleNotFoundError: ncnn.ncnn). - the tagging logic, run standalone over the published wheels, tags exactly ncnn/ncnn.so, faiss/_swigfaiss.so and CoolProp/{CoolProp,_constants}.so, and SKIPS flet-libcpp-shared's opt/lib/libc++_shared.so (no PyInit_). --- src/forge/build.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/forge/build.py b/src/forge/build.py index ece855e7..a3153595 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -809,6 +809,40 @@ def fix_wheel(self, wheel_dir: Path): if self.cross_venv.sdk == "android": env = self.compile_env() + # ABI-tag bare CPython extension modules so serious_python's Android + # packaging recognizes them. That packaging only relocates a native + # module into jniLibs and writes the `.soref` marker its on-device + # importer resolves for extensions whose filename carries a CPython + # ABI tag (`*.cpython-*.so` / `*.abi3.so`); a bare `NAME.so` is + # treated as a plain dependency library, gets no `.soref`, and so + # fails to import on-device (ModuleNotFoundError). CMake / SWIG / + # Cython / nanobind builds routinely emit un-tagged extensions + # (ncnn, faiss, coolprop, ...) because they can't derive the target + # SOABI when cross-compiling. Rename ONLY genuine extension modules — + # those exporting `PyInit_` — so real dependency + # libraries bundled alongside them are left untouched. + ext_tag_re = re.compile(r"\.(cpython-[^/]+|abi3)\.so$") + ext_suffix = f".cpython-3{sys.version_info.minor}.so" + nm = Path(env["STRIP"]).with_name("llvm-nm") + for so in wheel_dir.glob("**/*.so"): + if ext_tag_re.search(so.name): + continue # already ABI-tagged + module = so.name[: -len(".so")] + try: + symbols = subprocess.check_output( + [str(nm), "-D", "--defined-only", str(so)], text=True + ) + except (subprocess.CalledProcessError, OSError): + continue # not an analyzable ELF; leave it alone + if f"PyInit_{module}" in symbols.split(): + tagged = so.with_name(module + ext_suffix) + log( + self.log_file, + f"[{self.cross_venv}] ABI-tagging extension " + f"{so.name} -> {tagged.name}", + ) + so.rename(tagged) + for so in wheel_dir.glob("**/*.so"): log(self.log_file, f"[{self.cross_venv}] Stripping {so}") self.cross_venv.run( From 25be755ff5041cb3effc94b919c3191765d8864b Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 17:36:59 +0200 Subject: [PATCH 08/49] =?UTF-8?q?recipe:=20pysodium=20+=20opaque=20?= =?UTF-8?q?=E2=80=94=20load=20bundled=20lib=20by=20soname=20(Flet=200.86?= =?UTF-8?q?=20Android)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both packages load their native lib via ctypes.util.find_library('sodium' / 'opaque'), which returns None on Android (no ld cache — find_library relies on gcc/ld/ldconfig) and on iOS (the lib is framework-ized). LoadLibrary(None) then yields a nameless handle and __init__ raises "Unable to find libsodium/libopaque". flet-libsodium / flet-libopaque bundle the libraries and serious-python surfaces them into the APK's jniLibs, so fall back to loading the bundled soname (libX.so / libX.fwork / libX.dylib) when find_library() comes up empty. - pysodium: new mobile.patch (recipe had none) + register it in meta.yaml. - opaque: extend the existing mobile.patch (the setup.py install_requires fix) with the same loader accommodation; opaque imports pysodium, so both are fixed together. Verified on-device (android arm64, flet 0.86.0.dev2): pysodium 2/2 passed, opaque 2/2 passed (import + a real registration/credential roundtrip exercising both libopaque and libsodium). Was: ValueError: Unable to find libsodium/libopaque. --- recipes/opaque/patches/mobile.patch | 41 +++++++++++++++++++++++++++ recipes/pysodium/meta.yaml | 3 ++ recipes/pysodium/patches/mobile.patch | 37 ++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 recipes/pysodium/patches/mobile.patch diff --git a/recipes/opaque/patches/mobile.patch b/recipes/opaque/patches/mobile.patch index d3402ac5..9ec5c7d0 100644 --- a/recipes/opaque/patches/mobile.patch +++ b/recipes/opaque/patches/mobile.patch @@ -1,3 +1,16 @@ +opaque 1.0.0 ships two mobile-hostile things this patch fixes: + +1. setup.py declares `install_requires = ("pysodium")` — a bare string, not a + tuple (missing trailing comma), which setuptools treats inconsistently; + rewritten to a proper list so pip reliably pulls the pysodium dependency. + +2. opaque/__init__.py loads libopaque via ctypes.util.find_library('opaque'), + which returns None on Android (no ld cache) and iOS (the library is + framework-ized), so LoadLibrary(None) yields a nameless handle and __init__ + raises "Unable to find libopaque". flet-libopaque bundles the library; fall + back to the bundled sonames serious-python surfaces (identical to the + pysodium fix). + diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -10,3 +23,31 @@ diff --git a/setup.py b/setup.py classifiers=["Development Status :: 4 - Beta", "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", "Topic :: Security :: Cryptography", +diff --git a/opaque/__init__.py b/opaque/__init__.py +--- a/opaque/__init__.py ++++ b/opaque/__init__.py +@@ -26,9 +26,21 @@ + import ctypes.util + from ctypes import c_uint16 + +-opaquelib = ctypes.cdll.LoadLibrary(ctypes.util.find_library('opaque') or ctypes.util.find_library('libopaque')) +- +-if not opaquelib._name: ++# mobile-forge: ctypes.util.find_library() cannot locate the bundled libopaque ++# on Android (no ld cache) or iOS (framework-ized), returning None; fall back to ++# the bundled sonames flet-libopaque surfaces (see the identical pysodium fix). ++_found = ctypes.util.find_library('opaque') or ctypes.util.find_library('libopaque') ++if _found: ++ opaquelib = ctypes.cdll.LoadLibrary(_found) ++else: ++ opaquelib = None ++ for _cand in ('libopaque.so', 'libopaque.fwork', 'libopaque.dylib'): ++ try: ++ opaquelib = ctypes.cdll.LoadLibrary(_cand) ++ break ++ except OSError: ++ opaquelib = None ++if opaquelib is None or not opaquelib._name: + raise ValueError('Unable to find libopaque') + + from pysodium import (crypto_core_ristretto255_SCALARBYTES, crypto_scalarmult_SCALARBYTES, diff --git a/recipes/pysodium/meta.yaml b/recipes/pysodium/meta.yaml index 6178fe01..1923d0e8 100644 --- a/recipes/pysodium/meta.yaml +++ b/recipes/pysodium/meta.yaml @@ -5,6 +5,9 @@ package: build: number: 10 +patches: + - mobile.patch + requirements: host: - flet-libsodium 1.0.20 \ No newline at end of file diff --git a/recipes/pysodium/patches/mobile.patch b/recipes/pysodium/patches/mobile.patch new file mode 100644 index 00000000..4229e4e9 --- /dev/null +++ b/recipes/pysodium/patches/mobile.patch @@ -0,0 +1,37 @@ +pysodium loads libsodium via ctypes.util.find_library('sodium'), which returns +None on Android (no ld cache) and on iOS (the library is framework-ized), so +LoadLibrary(None) yields a nameless handle and __init__ raises "Unable to find +libsodium". flet-libsodium bundles the library and serious-python surfaces it +under a bundled soname (Android jniLibs libsodium.so / iOS libsodium.fwork / +desktop libsodium.dylib); fall back to loading those directly when +find_library() comes up empty. + +diff --git a/pysodium/__init__.py b/pysodium/__init__.py +--- a/pysodium/__init__.py ++++ b/pysodium/__init__.py +@@ -30,8 +30,23 @@ + import ctypes + import ctypes.util + +-sodium = ctypes.cdll.LoadLibrary(ctypes.util.find_library('sodium') or ctypes.util.find_library('libsodium')) +-if not sodium._name: ++# mobile-forge: on Android/iOS ctypes.util.find_library() cannot locate the ++# bundled libsodium (Android has no ld cache; iOS framework-izes it), returning ++# None -> LoadLibrary(None) yields a nameless handle and the check below raises. ++# flet-libsodium surfaces the lib under these bundled names (Android jniLibs / ++# iOS .fwork shim / desktop .dylib), so fall back to loading them directly. ++_found = ctypes.util.find_library('sodium') or ctypes.util.find_library('libsodium') ++if _found: ++ sodium = ctypes.cdll.LoadLibrary(_found) ++else: ++ sodium = None ++ for _cand in ('libsodium.so', 'libsodium.fwork', 'libsodium.dylib'): ++ try: ++ sodium = ctypes.cdll.LoadLibrary(_cand) ++ break ++ except OSError: ++ sodium = None ++if sodium is None or not sodium._name: + raise ValueError('Unable to find libsodium') + + sodium.sodium_version_string.restype = ctypes.c_char_p From b2b54a8fcf162b760b9692fc386964ade1ab360a Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 17:47:47 +0200 Subject: [PATCH 09/49] =?UTF-8?q?recipe:=20pycryptodome=20+=20pycryptodome?= =?UTF-8?q?x=20=E2=80=94=20resolve=20native=20module=20via=20importlib=20(?= =?UTF-8?q?0.86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pycryptodome ctypes-loads its native modules (Crypto/Util/_raw_api.py: load_pycryptodome_raw_lib) by probing os.path.isfile() for the .so next to the package's __file__. Under Flet 0.86 the package lives inside sitepackages.zip, so every probe misses and it raises "Cannot load native module 'Crypto.Util._cpuid_c'". The extensions ARE relocated into the APK's jniLibs (they are .abi3.so, which serious_python tags and writes a .soref for), but the loader ctypes-loads by path, not via import, so it never finds them. Extend the existing mobile.patch loader: after the on-disk probes miss, ask the import system (importlib.util.find_spec) for the resolved on-device origin — the serious_python .soref finder returns a spec whose origin is the loadable jniLibs / base.apk!/lib path — and dlopen that. No hardcoded soname mangling; it reuses serious_python's own resolution through the public importlib API. The iOS (.fwork) and desktop paths are unchanged (they succeed in the first loop, before this fallback is reached). pycryptodomex gets the identical fix under the Cryptodome namespace. Verified on-device (android arm64, flet 0.86.0.dev2), extensions resolving from their .soref markers: - pycryptodome 3/3 passed (AES import + AES-CBC roundtrip + SHA-256 vector) - pycryptodomex 2/2 passed (AES-GCM roundtrip + SHA-256 vector) Was: OSError: Cannot load native module ... Not found _cpuid_c.cpython-312.so/.abi3.so/.so/.fwork. --- recipes/pycryptodome/patches/mobile.patch | 39 ++++++++++++++++++-- recipes/pycryptodomex/patches/mobile.patch | 41 ++++++++++++++++++++-- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/recipes/pycryptodome/patches/mobile.patch b/recipes/pycryptodome/patches/mobile.patch index acd0f670..da6cc938 100644 --- a/recipes/pycryptodome/patches/mobile.patch +++ b/recipes/pycryptodome/patches/mobile.patch @@ -1,16 +1,32 @@ +pycryptodome's native modules are ctypes-loaded (not imported) by +Crypto/Util/_raw_api.py:load_pycryptodome_raw_lib, which probes os.path.isfile() +for the .so next to the package's __file__. This patch makes that work on mobile: + +1. setup.py: add install_requires=['cffi']. Upstream declares none, so pip does + not pull cffi and _raw_api falls back to ctypes.pythonapi.PyObject_GetBuffer, + which fails on Android ("undefined symbol: PyObject_GetBuffer" — Flet loads + libpython with RTLD_LOCAL). See recipes/pycryptodome/meta.yaml. + +2. _raw_api.py: locate the native module when it is NOT a plain file on disk: + - iOS (serious-python frameworks): a `.fwork` text marker names the embedded + framework binary next to sys.executable. + - Android (Flet 0.86): the package ships inside sitepackages.zip, so every + os.path.isfile() probe misses; serious-python relocated the extension into + the APK's jniLibs and left a `.soref` marker resolved by a sys.meta_path + finder. Ask importlib for the resolved on-device origin and dlopen it. + diff --git a/lib/Crypto/Util/_raw_api.py b/lib/Crypto/Util/_raw_api.py -index e0065c3..3b14e00 100644 --- a/lib/Crypto/Util/_raw_api.py +++ b/lib/Crypto/Util/_raw_api.py @@ -30,6 +30,7 @@ - + import os import abc +import pathlib import sys from Crypto.Util.py3compat import byte_string from Crypto.Util._file_system import pycryptodome_filename -@@ -302,13 +303,17 @@ def load_pycryptodome_raw_lib(name, cdecl): +@@ -302,16 +303,34 @@ split = name.split(".") dir_comps, basename = split[:-1], split[-1] attempts = [] @@ -29,6 +45,23 @@ index e0065c3..3b14e00 100644 return load_lib(full_name, cdecl) except OSError as exp: attempts.append("Cannot load '%s': %s" % (filename, str(exp))) ++ # mobile-forge (Android/Flet 0.86): the extension ships inside sitepackages.zip, ++ # so the os.path.isfile() probes above all miss. serious_python relocated it into ++ # the APK's jniLibs and left a ``.soref`` marker resolved by a sys.meta_path ++ # finder -- ask the import system for the resolved on-device origin and load it. ++ try: ++ import importlib.util ++ spec = importlib.util.find_spec(name) ++ except Exception: ++ spec = None ++ if spec is not None and getattr(spec, "origin", None): ++ try: ++ return load_lib(spec.origin, cdecl) ++ except OSError as exp: ++ attempts.append("Cannot load '%s': %s" % (spec.origin, str(exp))) + raise OSError("Cannot load native module '%s': %s" % (name, ", ".join(attempts))) + + diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py diff --git a/recipes/pycryptodomex/patches/mobile.patch b/recipes/pycryptodomex/patches/mobile.patch index 0cf315c4..047ecfa8 100644 --- a/recipes/pycryptodomex/patches/mobile.patch +++ b/recipes/pycryptodomex/patches/mobile.patch @@ -1,16 +1,34 @@ +pycryptodomex is the sister package of pycryptodome (same code under the +Cryptodome.* namespace); this patch mirrors recipes/pycryptodome/patches/mobile.patch. +Its native modules are ctypes-loaded (not imported) by +Cryptodome/Util/_raw_api.py:load_pycryptodome_raw_lib, which probes os.path.isfile() +for the .so next to the package's __file__. This makes that work on mobile: + +1. setup.py: add install_requires=['cffi']. Upstream declares none, so pip does + not pull cffi and _raw_api falls back to ctypes.pythonapi.PyObject_GetBuffer, + which fails on Android ("undefined symbol: PyObject_GetBuffer" — Flet loads + libpython with RTLD_LOCAL). See recipes/pycryptodomex/meta.yaml. + +2. _raw_api.py: locate the native module when it is NOT a plain file on disk: + - iOS (serious-python frameworks): a `.fwork` text marker names the embedded + framework binary next to sys.executable. + - Android (Flet 0.86): the package ships inside sitepackages.zip, so every + os.path.isfile() probe misses; serious-python relocated the extension into + the APK's jniLibs and left a `.soref` marker resolved by a sys.meta_path + finder. Ask importlib for the resolved on-device origin and dlopen it. + diff --git a/lib/Cryptodome/Util/_raw_api.py b/lib/Cryptodome/Util/_raw_api.py -index e0065c3..3b14e00 100644 --- a/lib/Cryptodome/Util/_raw_api.py +++ b/lib/Cryptodome/Util/_raw_api.py @@ -30,6 +30,7 @@ - + import os import abc +import pathlib import sys from Cryptodome.Util.py3compat import byte_string from Cryptodome.Util._file_system import pycryptodome_filename -@@ -302,13 +303,17 @@ def load_pycryptodome_raw_lib(name, cdecl): +@@ -302,16 +303,34 @@ split = name.split(".") dir_comps, basename = split[:-1], split[-1] attempts = [] @@ -29,6 +47,23 @@ index e0065c3..3b14e00 100644 return load_lib(full_name, cdecl) except OSError as exp: attempts.append("Cannot load '%s': %s" % (filename, str(exp))) ++ # mobile-forge (Android/Flet 0.86): the extension ships inside sitepackages.zip, ++ # so the os.path.isfile() probes above all miss. serious_python relocated it into ++ # the APK's jniLibs and left a ``.soref`` marker resolved by a sys.meta_path ++ # finder -- ask the import system for the resolved on-device origin and load it. ++ try: ++ import importlib.util ++ spec = importlib.util.find_spec(name) ++ except Exception: ++ spec = None ++ if spec is not None and getattr(spec, "origin", None): ++ try: ++ return load_lib(spec.origin, cdecl) ++ except OSError as exp: ++ attempts.append("Cannot load '%s': %s" % (spec.origin, str(exp))) + raise OSError("Cannot load native module '%s': %s" % (name, ", ".join(attempts))) + + diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py From 98b036ac522c38ed012773b7aee049b2a29344cd Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 18:00:07 +0200 Subject: [PATCH 10/49] =?UTF-8?q?recipe:=20llama-cpp-python=20=E2=80=94=20?= =?UTF-8?q?load=20bundled=20libs=20by=20soname=20on=20Android=20(Flet=200.?= =?UTF-8?q?86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama_cpp/_ctypes_extensions.py:load_shared_library builds candidate paths under base_path = dirname(__file__)/lib and gates each on Path.exists() before ctypes.CDLL. Under Flet 0.86 site-packages ships as a zip, so base_path is inside sitepackages.zip and never exists on disk; serious-python relocated the bundled libllama/libggml* into the APK's jniLibs. Every probe missed -> FileNotFoundError: Shared library with base name 'llama' not found. Extend the existing mobile.patch loader: when no candidate path exists on disk, fall back to ctypes.CDLL by bare soname (lib.so) for both the ggml dependency preload and the main libllama load, so the Android linker resolves them from jniLibs (loaded RTLD_GLOBAL, so the preloaded deps satisfy libllama's DT_NEEDED). iOS (.fwork) and desktop paths are unchanged — their base_path is a real directory, so the existing Path.exists() branch still wins first. Verified on-device (android arm64, flet 0.86.0.dev2): llama-cpp-python 1/1 passed (test_native_lib_callable), with libllama/libggml{,-base,-cpu}.so resolving from jniLibs. Was: FileNotFoundError: Shared library with base name 'llama' not found. llama on iOS is a separate blocker (serious-python #223 — framework-izing the interdependent ctypes .dylib libs — not this Android fix). --- recipes/llama-cpp-python/patches/mobile.patch | 57 ++++++++++++++----- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/recipes/llama-cpp-python/patches/mobile.patch b/recipes/llama-cpp-python/patches/mobile.patch index 06fce5fb..7f793469 100644 --- a/recipes/llama-cpp-python/patches/mobile.patch +++ b/recipes/llama-cpp-python/patches/mobile.patch @@ -1,4 +1,4 @@ -It does four things: +It does five things: 1. Stops llama-cpp-python's CMakeLists from FORCE-enabling GGML_METAL on all Apple platforms (it would override our -DGGML_METAL=OFF for the iOS cross @@ -10,12 +10,16 @@ It does four things: three copies (and three colliding frameworks on iOS). 4. Teaches the ctypes loader to find the bundled lib under its iOS framework name (lib.fwork) and on the "ios" platform. +5. Android/Flet 0.86: site-packages ships as a zip and serious-python relocates + the bundled libs (libllama/libggml*) into the APK's jniLibs, so the loader's + Path.exists() probes against the in-zip base_path all miss. Add a bare-soname + fallback (ctypes.CDLL("lib.so")) for both the dependency preload and the + main load, so the Android linker resolves them from jniLibs. diff --git a/CMakeLists.txt b/CMakeLists.txt -index 5feaaca..db16837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -67,8 +67,11 @@ if (LLAMA_BUILD) +@@ -67,8 +67,11 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set(CMAKE_SKIP_RPATH FALSE) @@ -29,7 +33,7 @@ index 5feaaca..db16837 100644 # Disable building curl support set(LLAMA_CURL OFF CACHE BOOL "llama.cpp: enable curl" FORCE) -@@ -100,7 +103,11 @@ if (LLAMA_BUILD) +@@ -100,7 +103,11 @@ endif() # Architecture detection and settings for Apple platforms @@ -43,10 +47,9 @@ index 5feaaca..db16837 100644 execute_process( COMMAND uname -m diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py -index 02cee8a..9274b5d 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py -@@ -25,30 +25,33 @@ _EMSCRIPTEN_SIDE_MODULE_SUFFIX = ".cpython-00-wasm32-emscripten.so" +@@ -25,30 +25,33 @@ # Load the library def load_shared_library(lib_base_name: str, base_path: pathlib.Path): """Platform independent shared library loader""" @@ -104,7 +107,7 @@ index 02cee8a..9274b5d 100644 raise RuntimeError("Unsupported platform") cdll_args = dict() # type: ignore -@@ -67,9 +70,14 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path): +@@ -67,9 +70,14 @@ os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) cdll_args["winmode"] = ctypes.RTLD_GLOBAL @@ -120,7 +123,7 @@ index 02cee8a..9274b5d 100644 lib_dir = str(base_path) ld_library_path = os.environ.get("LD_LIBRARY_PATH", "") if lib_dir not in ld_library_path.split(os.pathsep): -@@ -79,24 +87,26 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path): +@@ -79,30 +87,54 @@ else f"{lib_dir}{os.pathsep}{ld_library_path}" ) @@ -142,6 +145,7 @@ index 02cee8a..9274b5d 100644 + "llava": ("ggml-base", "ggml-cpu", "ggml", "llama"), + } + for dependency in dependencies.get(lib_base_name, ()): ++ loaded = False + for dependency_path in _candidate_paths(dependency): if dependency_path.exists(): try: @@ -152,7 +156,19 @@ index 02cee8a..9274b5d 100644 - ) + except Exception: + pass ++ loaded = True + break ++ if not loaded: ++ # Android: the bundled lib is not a file on disk (Flet 0.86 ships ++ # site-packages as a zip and serious-python relocated it into the ++ # APK's jniLibs); load it by bare soname so the platform linker ++ # resolves it from there. Best-effort, like the on-disk branch. ++ for dependency_path in _candidate_paths(dependency): ++ try: ++ ctypes.CDLL(dependency_path.name, **cdll_args) # type: ignore ++ break ++ except OSError: ++ continue # Try to load the shared library, handling potential errors - for lib_path in lib_paths: @@ -160,11 +176,25 @@ index 02cee8a..9274b5d 100644 if lib_path.exists(): try: return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore + except Exception as e: + raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") + ++ # Android: the library was relocated into the APK's jniLibs and is not a file ++ # on disk (Flet 0.86 ships site-packages as a zip); load it by bare soname so ++ # the platform linker resolves it from jniLibs. ++ for lib_path in _candidate_paths(lib_base_name): ++ try: ++ return ctypes.CDLL(lib_path.name, **cdll_args) # type: ignore ++ except OSError: ++ continue ++ + raise FileNotFoundError( + f"Shared library with base name '{lib_base_name}' not found" + ) diff --git a/vendor/llama.cpp/ggml/src/CMakeLists.txt b/vendor/llama.cpp/ggml/src/CMakeLists.txt -index 89e5180..d4a6657 100644 --- a/vendor/llama.cpp/ggml/src/CMakeLists.txt +++ b/vendor/llama.cpp/ggml/src/CMakeLists.txt -@@ -208,10 +208,13 @@ add_library(ggml-base +@@ -208,10 +208,13 @@ ggml-quants.h gguf.cpp) @@ -182,7 +212,7 @@ index 89e5180..d4a6657 100644 target_include_directories(ggml-base PRIVATE .) if (GGML_BACKEND_DL) -@@ -244,10 +247,10 @@ add_library(ggml +@@ -244,10 +247,10 @@ ggml-backend-reg.cpp) add_library(ggml::ggml ALIAS ggml) @@ -197,7 +227,7 @@ index 89e5180..d4a6657 100644 if (GGML_BACKEND_DIR) if (NOT GGML_BACKEND_DL) -@@ -291,10 +294,11 @@ function(ggml_add_backend_library backend) +@@ -291,10 +294,11 @@ # Set versioning properties for all backend libraries # Building a MODULE library with a version is not supported on macOS (https://gitlab.kitware.com/cmake/cmake/-/issues/20782) if (NOT (APPLE AND GGML_BACKEND_DL)) @@ -214,10 +244,9 @@ index 89e5180..d4a6657 100644 if(NOT GGML_AVAILABLE_BACKENDS) diff --git a/vendor/llama.cpp/src/CMakeLists.txt b/vendor/llama.cpp/src/CMakeLists.txt -index d15ccfd..84769d1 100644 --- a/vendor/llama.cpp/src/CMakeLists.txt +++ b/vendor/llama.cpp/src/CMakeLists.txt -@@ -43,8 +43,9 @@ add_library(llama +@@ -43,8 +43,9 @@ ) set_target_properties(llama PROPERTIES From 6186f3a3434370b5c4a6aaefbdf92233ff5b5fcb Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 18:28:57 +0200 Subject: [PATCH 11/49] =?UTF-8?q?recipe:=20jq=20=E2=80=94=20static-link=20?= =?UTF-8?q?libjq=20so=20the=20extension=20doesn't=20collide=20on=20Android?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `jq` Python binding is a top-level extension module `jq`, which serious_python (Flet 0.86 Android) relocates into jniLibs under the mangled name `libjq.so` — the same basename as flet-libjq's shared library. Both mapped to lib//libjq.so, the extension clobbered the C lib, and `import jq` failed with "cannot locate symbol jq_teardown referenced by libjq.so" (the C lib carrying jq_teardown was overwritten by the extension). NOT a missing symbol export — libjq.so exports all 177 jq_/jv_ symbols; it was a jniLibs name collision. Fix: static-link jq + oniguruma into the extension so it is self-contained and no libjq.so ships at runtime. - flet-libjq: configure --with-pic (PIC static archives), keep libjq.a/libonig.a, drop the shared libraries. build 10 -> 11. - jq: requirements host -> host_build (flet-libjq linked at build time, not a runtime dep) and mobile.patch links -l:libjq.a/-l:libonig.a (static) instead of -ljq/-lonig (dynamic). build 1 -> 2. Verified on-device (android arm64, flet 0.86.0.dev2): jq 2/2 passed (filter + first). The APK ships a single self-contained lib/arm64-v8a/libjq.so (the ~1 MB extension; DT_NEEDED lists no libjq.so/libonig.so) — no collision. Was: ImportError: dlopen failed: cannot locate symbol "jq_teardown". --- recipes/flet-libjq/build.sh | 16 +++++++++++----- recipes/flet-libjq/meta.yaml | 2 +- recipes/jq/meta.yaml | 13 ++++++++++--- recipes/jq/patches/mobile.patch | 26 ++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 recipes/jq/patches/mobile.patch diff --git a/recipes/flet-libjq/build.sh b/recipes/flet-libjq/build.sh index a546dbc4..b769ea51 100755 --- a/recipes/flet-libjq/build.sh +++ b/recipes/flet-libjq/build.sh @@ -1,13 +1,19 @@ #!/bin/bash set -eu -./configure --host=$HOST_TRIPLET --prefix=$PREFIX --with-oniguruma=builtin +# --with-pic: build PIC objects so the static archives (libjq.a / libonig.a) +# can be linked into the jq Python extension, which is itself a shared object. +# The jq recipe static-links them (requirements.host_build + -l:libjq.a) so the +# extension is self-contained. This is deliberate: on Android the `jq` Python +# module mangles to jniLibs `libjq.so` — the SAME basename this library would +# ship — so a bundled libjq.so would collide with (and be clobbered by) the +# extension. Static-linking avoids shipping libjq.so at all. +./configure --host=$HOST_TRIPLET --prefix=$PREFIX --with-oniguruma=builtin --with-pic make -j $CPU_COUNT make install rm -r $PREFIX/{bin,share} rm -r $PREFIX/lib/{*.la,pkgconfig} - -if [ $CROSS_VENV_SDK == "android" ]; then - rm -r $PREFIX/lib/*.a -fi \ No newline at end of file +# Keep the static archives (libjq.a / libonig.a); drop the shared libraries so +# nothing can ship/relocate a colliding libjq.so at runtime. +rm -f $PREFIX/lib/*.so* diff --git a/recipes/flet-libjq/meta.yaml b/recipes/flet-libjq/meta.yaml index e3d26a36..b3c3b4ca 100644 --- a/recipes/flet-libjq/meta.yaml +++ b/recipes/flet-libjq/meta.yaml @@ -8,7 +8,7 @@ source: url: https://github.com/jqlang/jq/releases/download/jq-{{ version }}/jq-{{ version }}.tar.gz build: - number: 10 + number: 11 requirements: build: diff --git a/recipes/jq/meta.yaml b/recipes/jq/meta.yaml index 72313637..5514ef1d 100644 --- a/recipes/jq/meta.yaml +++ b/recipes/jq/meta.yaml @@ -3,10 +3,17 @@ package: version: "1.11.0" requirements: - host: + # host_build (not host): flet-libjq's static archives are linked INTO the jq + # extension and must NOT be shipped as a runtime dependency — a bundled + # libjq.so would collide with the `jq` extension's own jniLibs name on Android + # (see recipes/jq/patches/mobile.patch and recipes/flet-libjq/build.sh). + host_build: - flet-libjq 1.7.1 build: - number: 1 + number: 2 script_env: - JQPY_USE_SYSTEM_LIBS: 1 \ No newline at end of file + JQPY_USE_SYSTEM_LIBS: 1 + +patches: + - mobile.patch \ No newline at end of file diff --git a/recipes/jq/patches/mobile.patch b/recipes/jq/patches/mobile.patch new file mode 100644 index 00000000..4f01db61 --- /dev/null +++ b/recipes/jq/patches/mobile.patch @@ -0,0 +1,26 @@ +The jq Python extension is the top-level module `jq`, which serious-python +mangles to jniLibs `libjq.so` on Android — the same basename as a shared libjq. +A dynamic `-ljq` link therefore bundles a `libjq.so` that collides with (and is +clobbered by) the extension's own relocated `libjq.so`, so `import jq` loses +jq_teardown. Static-link jq + oniguruma into the extension instead (flet-libjq +ships PIC static archives — see recipes/flet-libjq/build.sh): the extension is +self-contained and no libjq.so is bundled at runtime. JQPY_USE_SYSTEM_LIBS still +selects flet-libjq's prebuilt libs, now via `-l:libjq.a` (static) not `-ljq`. + +diff --git a/setup.py b/setup.py +--- a/setup.py ++++ b/setup.py +@@ -103,7 +103,12 @@ + + if use_system_libs: + jq_build_ext = build_ext +- link_args_deps = ["-ljq", "-lonig"] ++ # mobile-forge: static-link flet-libjq's prebuilt archives so the extension ++ # is self-contained. The `jq` module mangles to jniLibs `libjq.so` on Android ++ # -- the same basename as a shared libjq -- so a dynamic `-ljq` would bundle a ++ # colliding libjq.so. `-l:NAME.a` forces the static archives from flet-libjq's ++ # opt/lib (wired via requirements.host_build). ++ link_args_deps = ["-l:libjq.a", "-l:libonig.a"] + extra_objects = [] + else: + jq_build_ext = jq_with_deps_build_ext From c87658e86ac09da8f76cfc49f3f03e47df84faf5 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 18:43:47 +0200 Subject: [PATCH 12/49] skills: document the Flet 0.86 sitepackages.zip failure class + meta fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold this session's findings into the git-tracked skills so they're not re-derived: - forge-error-catalogue: new umbrella entry (the 0.86 Android packaging change and why __file__-relative loaders/data reads that worked under 0.85 fail now) plus the specific symptom -> fix entries — NotADirectoryError -> extract_packages; untagged native .so ModuleNotFoundError / "circular import" -> forge fix_wheel ABI-tag; pycryptodome "Cannot load native module" -> importlib.find_spec().origin; llama FileNotFoundError / ctypes find_library-None -> bare-soname load from jniLibs; a top-level ext colliding with a flet-lib* at lib.so -> static-link. - new-mobile-recipe + local-recipe-testing: the new meta.yaml `extract_packages` and `test.requires` fields (test.requires replaced tests/requirements.txt), the Flet 0.86 build-toolchain pin (uvx --prerelease from pypi.flet.dev) for the local loop, and an APK-contents verification gotcha. --- .claude/skills/forge-error-catalogue/SKILL.md | 16 ++- .../references/failure-catalogue.md | 135 +++++++++++++++++- .claude/skills/local-recipe-testing/SKILL.md | 22 ++- .claude/skills/new-mobile-recipe/SKILL.md | 7 +- 4 files changed, 168 insertions(+), 12 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 6fb06daf..9ba6ade1 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -15,7 +15,11 @@ description: >- duplicated patch hunks, iOS flatc MACOSX_BUNDLE, .dylib-vs-.so staging, libc++_shared, config.sub apple-ios, ctypes "Unable to find ... shared library", PT_LOAD 16KB alignment, lazy_loader "non-existent stub" crashes - (stripped *.pyi), hidden runtime deps, or a stale on-device cache. Sibling + (stripped *.pyi), the Flet 0.86 Android sitepackages.zip class (NotADirectoryError + on a bundled data file -> extract_packages, untagged native .so + ModuleNotFoundError -> forge ABI-tag, ctypes-by-__file__ loaders -> find_spec / + bare soname, jniLibs lib.so name collision -> static-link), hidden runtime + deps, or a stale on-device cache. Sibling of `new-mobile-recipe` (authoring), `native-recipe-bumps` (version bumps), `local-recipe-testing` (on-device testing), and `forge-ci` (CI triage); this one is the dedicated error -> fix reference. @@ -69,7 +73,15 @@ instead of re-deriving it. hand-written + gen2.py-generated code), **opencv-5 KleidiCV `armv8-a` on x86_64** and **hardcoded `CMAKE_SYSTEM_PROCESSOR` → ARM asm on the x86_64 sim** (per-arch fix), **Rust crate with no `target_os="ios"` backend** (`mac_address`, cfg-gate it). -- **Runtime failures** (device/emulator/simulator) — `libc++_shared.so` not found, +- **Runtime failures** (device/emulator/simulator) — **the Flet 0.86 Android + `sitepackages.zip` class** (its umbrella entry explains "why only now"): + `NotADirectoryError` on a bundled data file → **`extract_packages`** meta field; + untagged native `.so` `ModuleNotFoundError`/"circular import" → **forge `fix_wheel` + ABI-tags `PyInit_*` `.so`**; pycryptodome `Cannot load native module …` → + **`importlib.find_spec().origin`**; llama `FileNotFoundError` / ctypes + `find_library`-None → **bare-soname load from jniLibs**; a top-level extension + colliding with a `flet-lib*` at `lib.so` → **static-link (`host_build` + + `-l:lib.a`)**. Plus: `libc++_shared.so` not found, **CMake OBJECT-lib not linked into the iOS `.framework` → undefined symbol at dlopen** (opencv-5 MLAS; green build ≠ loadable — verify with `nm -u`/`otool -L`), ctypes **"Unable to find … shared library"** (Pattern H loader / lib delivery), diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 0ca4252d..9af10ce3 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -943,6 +943,136 @@ uuid-utils 0.17.0 `mobile.patch`. ## Runtime failures (on device/emulator/simulator) +### Flet 0.86 changed Android packaging — `sitepackages.zip` + jniLibs relocation (the umbrella behind a whole class of "worked under 0.85, fails now" on-device failures) + +**Cause:** the recipe-tester migration to Flet 0.86 (flet-dev/flet#104) stopped +extracting site-packages to disk. Now, on **Android**: +- pure Python ships in a stored **`sitepackages.zip`** imported via `zipimport`; +- native extensions are **relocated into the APK's `jniLibs//lib.so`** + and resolved by a `sys.meta_path` finder (`_sp_bootstrap._SorefFinder`) through a + per-module **`.soref`** marker left in the zip — written **only** for filenames + matching `\.(cpython-[^/]+|abi3)\.so$`; +- `opt/` trees from `flet-lib*` wheels are copied to `jniLibs` by `copyOpt`, which + takes **only `**/*.so`** (non-`.so` data is dropped); +- packages are compiled to `.pyc` and the `.py` source is stripped (except the app, + which sets `[tool.flet.compile] app = false`). + +Under 0.85 site-packages was a real directory (the `useLegacyPackaging` extract), so +any loader/data-access that built a filesystem path from `__file__` worked; under +0.86 `dirname(__file__)` is *inside* the zip and every such path misses — that's the +"why only now." **iOS is unaffected** (its site-packages stay a real dir in the app +bundle; iOS has its own `.fwork`/framework story). Match the specific symptom below; +after a build, `unzip -l` the APK and check `assets/sitepackages.zip` / +`assets/extract.zip` / `lib//`. + +--- + +### `NotADirectoryError: [Errno 20] Not a directory: '.../sitepackages.zip//'` (Android, at import) + +**Cause:** the package reads a bundled **data file** (not a `.py`) via a real +`__file__`-relative path — matplotlib `mpl-data/matplotlibrc`, astropy `CITATION`, +scikit-learn `estimator.css`, thinc `backends/_custom_kernels.cu` (spacy pulls +thinc). Under 0.86 the parent is `sitepackages.zip` (a file), so `open()`/ +`read_text()` on the child path raises NotADirectoryError. + +**Fix:** ship the package **extracted to disk** via serious_python's Android hatch. +Declare it in the recipe's meta.yaml as a top-level list: +```yaml +extract_packages: + - matplotlib # the IMPORT name (sklearn not scikit-learn; cv2 not opencv-python) +``` +`stage_recipe.sh` reads it (via `read_meta_list.py`) into +`[tool.flet.android].extract_packages`; serious_python unpacks those packages to +disk (on `sys.path` before the zip) so `__file__` resolves to a real dir. Field is +in `src/forge/schema/meta-schema.yaml`. Verified: matplotlib, astropy. Two +look-alikes this does NOT fix: **python-magic** (`could not find any valid magic +files!` — `magic.mgc` lives in flet-libmagic's `opt/`, dropped by `copyOpt` which +copies only `**/*.so`) and **opencv-python** (`missing configuration file: +['config.py']` — cv2's loader needs the `.py` *source*, stripped by 0.86's `.pyc` +compile). + +--- + +### `ModuleNotFoundError: No module named '.'` / `cannot import name '<_ext>' … (most likely due to a circular import)` for a NATIVE submodule (Android) + +**Cause:** the extension `.so` ships **untagged** — `ncnn/ncnn.so`, +`faiss/_swigfaiss.so`, `CoolProp/{CoolProp,_constants}.so` — rather than +`*.cpython-*.so`/`*.abi3.so`. serious_python only relocates + writes a `.soref` for +ABI-tagged names; a bare `NAME.so` is treated as a plain dependency lib, gets no +`.soref`, and the import finder can't resolve it. CMake / SWIG / Cython / nanobind +builds routinely emit untagged extensions (they can't derive the target SOABI when +cross-compiling; setuptools/maturin tag theirs, which is why numpy/pandas/pyarrow are +fine). The x86_64 leg can break while arm64 is fine if the build tags +nondeterministically per arch (onnxruntime). The misleading "circular import" +message is just Python's wording when a `from . import _ext` can't find the `.so`. + +**Fix:** none per-recipe — **forge `fix_wheel` (build.py, Android branch) ABI-tags +any bare `.so` exporting `PyInit_`** (a genuine extension; `llvm-nm -D` +discriminates it from a dependency lib, which is left untouched) to +`.cpython-3X.so`, so serious_python writes its `.soref`. Just rebuild; the +fix is generic and covers future CMake/SWIG/Cython recipes. If you hit this, confirm +via `unzip -l` that the wheel ships a bare `.so` exporting `PyInit_*`. Verified: +ncnn, faiss-cpu, coolprop; onnxruntime x86_64. + +--- + +### `OSError: Cannot load native module '.': Not found .cpython-3X.so, .abi3.so, .so, .fwork` (pycryptodome/pycryptodomex, Android) + +**Cause:** the `.abi3.so` extensions ARE relocated + `.soref`'d (a plain `import` +would work), but pycryptodome's `Crypto/Util/_raw_api.py:load_pycryptodome_raw_lib` +**ctypes-loads by a `__file__`-relative path** (`os.path.isfile` next to the module, +inside the zip), never consulting the import system. General shape: any package with +a custom dlopen-by-`__file__` loader for extensions that *are* correctly tagged. + +**Fix:** after the on-disk probes miss, ask the import system for the resolved +on-device origin and dlopen that — no hardcoded soname mangling: +```python +import importlib.util +spec = importlib.util.find_spec(name) # name = "Crypto.Util._cpuid_c" +if spec is not None and spec.origin: + return load_lib(spec.origin, cdecl) # _SorefFinder set origin to the jniLibs/apk path +``` +Extend the recipe's `mobile.patch` loader; keep the existing iOS `.fwork` branch (it +wins first on iOS, where the path is real). Verified: pycryptodome, pycryptodomex. + +--- + +### `FileNotFoundError: Shared library with base name '' not found` (ctypes-by-`__file__` loader, Android) + +**Cause:** the loader gates each candidate on `Path.exists()` for a +`dirname(__file__)/lib` path — inside `sitepackages.zip` under 0.86, so every probe +misses. The bundled libs (llama-cpp-python's libllama/libggml*) were relocated to +`jniLibs` and are loadable by bare soname. Same 0.86 class as the `find_library() +→ None` case (pysodium/opaque) below, different loader shape. + +**Fix:** after the on-disk probes miss, fall back to `ctypes.CDLL("lib.so")` +(bare soname → the Android linker resolves it from jniLibs), for both the dependency +preload and the main lib; load `RTLD_GLOBAL` so preloaded deps satisfy the main +lib's DT_NEEDED. Verified: llama-cpp-python (android). See also "Unable to find +… shared library" (the `find_library`-returns-None sibling). + +--- + +### `ImportError: dlopen failed: cannot locate symbol "" referenced by ".../lib//lib.so"` where `lib.so` is the wheel's OWN extension (Android) + +**Cause:** a **jniLibs name collision**. A *top-level* Python extension module named +`` (e.g. `jq`) mangles to `lib.so`; a bundled `flet-lib*` C library with +the same base name (`libjq.so`) *also* lands at `lib//lib.so`. One clobbers +the other — the extension (which references the C lib's symbols) overwrites the C +lib, so its symbols vanish (`cannot locate symbol jq_teardown`). NOT a missing +export — `llvm-nm -D` shows the C lib exports it fine; check `lib//lib.so`'s +size to see which one won. + +**Fix:** make the extension **self-contained** so no colliding `lib.so` ships. +Static-link the flet-lib* into the extension: build the flet-lib* `--with-pic` and +keep its `.a` (drop the `.so`); in the consumer switch `requirements.host` → +**`host_build`** (build-time only, not a runtime dep) and patch the link +`-l` → `-l:lib.a` (static). The extension's DT_NEEDED then lists no +`lib.so`. Verified: jq (static-links flet-libjq's libjq.a/libonig.a; flet-libjq +build 11, jq build 2). + +--- + ### iOS: `import ` → `symbol not found in flat namespace '……'` at dlopen (a CMake OBJECT library never made it into the `.framework`) **Cause:** the build is GREEN but a CMake `OBJECT` library's objects — meant to link @@ -990,7 +1120,10 @@ iOS doesn't need this — Apple's clang resolves the C++ runtime to system libc+ **Cause:** a pure-Python `ctypes` wrapper called `ctypes.util.find_library()`, which returns `None` on mobile (no ldconfig/compiler), so it gave up before -trying to load the lib. This is the Pattern H case (pyzbar, python-magic, …). +trying to load the lib. This is the Pattern H case (pyzbar, python-magic, +pysodium, opaque, …); under Flet 0.86 it is the `find_library`-returns-None +member of the `sitepackages.zip` class (umbrella entry at the top of this +section). **Fix:** this needs the full Pattern H treatment, not a one-liner: 1. the `flet-lib*` must be built **shared** (`lib.so`), not static; diff --git a/.claude/skills/local-recipe-testing/SKILL.md b/.claude/skills/local-recipe-testing/SKILL.md index f0337856..14e812be 100644 --- a/.claude/skills/local-recipe-testing/SKILL.md +++ b/.claude/skills/local-recipe-testing/SKILL.md @@ -1,6 +1,6 @@ --- name: local-recipe-testing -description: Run a mobile-forge recipe's wheel ON-DEVICE locally — Android emulator and/or iOS simulator — instead of waiting ~1 hour for a CI mobile-test cycle. Covers the recipe-tester app loop (build wheel → stage → flet build → install → read console.log), and the non-obvious gotchas that each cost a wasted cycle: use forge's stripped dist/ wheel, build the recipe against the SAME Python flet bundles, clear flet's build cache between rebuilds, use a rootable (google_apis, not playstore) arm64 AVD to read the app-private console.log (it's in the app's cache/ dir), give the emulator enough RAM/disk, build ALL THREE iOS slices before `flet build ios-simulator`, use explicit simulator UDIDs when more than one sim is booted, and verify the staged-test COUNT so a silently-failed staging can't replay stale tests as false passes. Also covers forge slice syntax, bundling model assets next to recipe tests, test-only deps via tests/requirements.txt, desktop pre-validation via a sys.modules alias shim, and consumer verify-apps for beyond-pytest validation. USE THIS SKILL when iterating on a recipe's on-device behaviour (import works? functions run? crashes?), reproducing or debugging a CI mobile-test failure locally, or whenever someone says the CI mobile test is too slow to iterate on. Sibling of `new-mobile-recipe` (authoring), `forge-ci` (CI runs), `forge-error-catalogue` (build errors), and `native-recipe-bumps` (version bumps); this one is specifically the fast on-device validation loop. macOS + Apple Silicon assumed (the host this was developed on). +description: Run a mobile-forge recipe's wheel ON-DEVICE locally — Android emulator and/or iOS simulator — instead of waiting ~1 hour for a CI mobile-test cycle. Covers the recipe-tester app loop (build wheel → stage → flet build → install → read console.log), and the non-obvious gotchas that each cost a wasted cycle: use forge's stripped dist/ wheel, build the recipe against the SAME Python flet bundles, clear flet's build cache between rebuilds, use a rootable (google_apis, not playstore) arm64 AVD to read the app-private console.log (it's in the app's cache/ dir), give the emulator enough RAM/disk, build ALL THREE iOS slices before `flet build ios-simulator`, use explicit simulator UDIDs when more than one sim is booted, and verify the staged-test COUNT so a silently-failed staging can't replay stale tests as false passes. Also covers forge slice syntax, bundling model assets next to recipe tests, test-only deps via the meta.yaml test.requires field, desktop pre-validation via a sys.modules alias shim, and consumer verify-apps for beyond-pytest validation. USE THIS SKILL when iterating on a recipe's on-device behaviour (import works? functions run? crashes?), reproducing or debugging a CI mobile-test failure locally, or whenever someone says the CI mobile test is too slow to iterate on. Sibling of `new-mobile-recipe` (authoring), `forge-ci` (CI runs), `forge-error-catalogue` (build errors), and `native-recipe-bumps` (version bumps); this one is specifically the fast on-device validation loop. macOS + Apple Silicon assumed (the host this was developed on). --- # Testing a mobile-forge recipe locally @@ -36,10 +36,14 @@ rm -rf /tmp/rt_dist && mkdir /tmp/rt_dist cp dist/-*-android_24_arm64_v8a.whl /tmp/rt_dist/ # forge's dist/ wheel = STRIPPED ./tests/recipe-tester/stage_recipe.sh -# 3. Clear flet's stale bundle (gotcha #3), then build the app +# 3. Clear flet's stale bundle (gotcha #3), then build the app. +# The recipe-tester targets Flet 0.86 (only there since flet#104), which is NOT +# on PyPI yet — pull it from pypi.flet.dev and pin the prerelease (gotcha #13). rm -rf tests/recipe-tester/build/site-packages tests/recipe-tester/build/.hash cd tests/recipe-tester -PIP_FIND_LINKS=/tmp/rt_dist uvx --with flet-cli flet build apk --arch arm64-v8a --yes +PIP_FIND_LINKS=/tmp/rt_dist \ + uvx --prerelease allow --default-index https://pypi.flet.dev --index https://pypi.org/simple \ + --from flet-cli flet build apk --arch arm64-v8a --yes --python-version 3.12 cd "$REPO" # 4. Boot the rootable AVD (gotcha #4/#5), install, launch @@ -70,7 +74,9 @@ forge iphoneos:arm64 ; forge iphonesimulator:arm64 ; forge iph ./tests/recipe-tester/stage_recipe.sh rm -rf tests/recipe-tester/build/site-packages tests/recipe-tester/build/.hash cd tests/recipe-tester -PIP_FIND_LINKS="$(realpath ../../dist)" uvx --with flet-cli flet build ios-simulator --yes +PIP_FIND_LINKS="$(realpath ../../dist)" \ + uvx --prerelease allow --default-index https://pypi.flet.dev --index https://pypi.org/simple \ + --from flet-cli flet build ios-simulator --yes --python-version 3.12 # 0.86 pin — gotcha #13 # 3. Boot any available iPhone sim, install, launch — ALWAYS by explicit UDID # (gotcha #11: `booted` is ambiguous the moment two sims are booted) @@ -122,7 +128,9 @@ for i in $(seq 1 30); do grep EXIT "$DATA/Library/Caches/console.log" 2>/dev/nul 11. **Two booted simulators make `simctl booted` ambiguous.** With more than one sim booted, `simctl install booted …` targets one device and your subsequent `get_app_container booted …` may query the OTHER — the app "isn't installed" / the container is empty despite a successful install. Use the explicit `$UDID` for every simctl call (as the loop above does); never rely on `booted` unless you've verified exactly one device is booted (`xcrun simctl list devices | grep -c Booted`). -12. **Verify the staged tests + the on-device test COUNT — staging can fail silently.** `stage_recipe.sh` wipes and re-stages `recipe_tests/`; if the invocation ever fails without you noticing (a scripted loop with a bad variable — zsh does NOT word-split unquoted `$VAR` like bash, so a `for r in $RECIPES`-style loop can pass the whole list as ONE argument), the PREVIOUS recipe's tests are still staged and run happily, reporting "N passed" for the wrong package. Two cheap checks after staging: `ls tests/recipe-tester/recipe_tests/` shows YOUR test files, and the "N passed" in console.log matches your recipe's test count. (Bit during the h5py→keras loop: the same 4 stale h5py tests "passed" three times.) +12. **Verify the staged tests + the on-device test COUNT — staging can fail silently.** `stage_recipe.sh` wipes and re-stages `recipe_tests/`; if the invocation ever fails without you noticing (a scripted loop with a bad variable — zsh does NOT word-split unquoted `$VAR` like bash, so a `for r in $RECIPES`-style loop can pass the whole list as ONE argument), the PREVIOUS recipe's tests are still staged and run happily, reporting "N passed" for the wrong package. Two cheap checks after staging: `ls tests/recipe-tester/recipe_tests/` shows YOUR test files, and the "N passed" in console.log matches your recipe's test count. (Bit during the h5py→keras loop: the same 4 stale h5py tests "passed" three times.) **Stronger still — verify the built APK's CONTENTS, not just `recipe_tests/`:** a build that *fails* can leave a STALE `build/apk/recipe-tester.apk` that installs the wrong app entirely. `unzip -l build/apk/recipe-tester.apk` should show your recipe's test `.py` inside `app.zip` AND (for a native recipe) `lib//lib*.so` for its libs. Caught an opaque run that silently installed a stale pysodium APK and reported "2 passed" for the wrong package. When in doubt nuke `build/apk` too, not just `build/site-packages`. + +13. **The recipe-tester now targets Flet 0.86 (since flet#104), which isn't on PyPI.** Plain `uvx --with flet-cli flet build` pulls 0.85 from PyPI — the OLD packaging model (extracted site-packages), so you'd test the wrong thing. Pull 0.86 from pypi.flet.dev with a prerelease pin: `uvx --prerelease allow --default-index https://pypi.flet.dev --index https://pypi.org/simple --from flet-cli flet build apk … --python-version 3.12`. 0.86 ships site-packages as `sitepackages.zip` and relocates native `.so` to jniLibs — a whole class of on-device loader/data-file failures lives there (`forge-error-catalogue` § the `sitepackages.zip` class). ## Model assets & test-only deps @@ -131,7 +139,9 @@ for i in $(seq 1 30); do grep EXIT "$DATA/Library/Caches/console.log" 2>/dev/nul - **Big models (MBs): drop next to the test locally; the test discovers it by presence and skips otherwise.** Precedent sherpa-onnx `silero_vad.onnx` (2.2 MB): `if not os.path.exists(model): pytest.skip("silero_vad.onnx not bundled")`. CI (no asset) skips; your local loop runs REAL inference. `.gitignore` has `recipes/*/tests/*.onnx` so the asset can never be committed (that would silently flip the CI skip into a real run and embed MBs in git history) — extend the pattern for other formats. - **Tiny models (~KB): COMMIT them so CI runs real inference too.** Precedent tflite-runtime's 1 KB `dense_relu.tflite` (generated with desktop TF at a fixed seed, expected outputs asserted). -Test-only deps — packages the tests import that are NOT in the recipe's Requires-Dist (e.g. numpy) — go in `recipes//tests/requirements.txt`; `stage_recipe.sh` injects them into the generated tester `pyproject.toml` (merged to main, #98). Each must resolve for the MOBILE target: pure-Python from PyPI, or a recipe on pypi.flet.dev / seeded into `dist/`. +Test-only deps — packages the tests import that are NOT in the recipe's Requires-Dist (e.g. numpy) — go in the recipe's meta.yaml `test.requires` (a list of PEP 508 specs; this replaced the former `recipes//tests/requirements.txt`, and `stage_recipe.sh` now errors on a stale one). `stage_recipe.sh` injects them into the generated tester `pyproject.toml` `dependencies` (read via `read_meta_list.py`). Each must resolve for the MOBILE target: pure-Python from PyPI, or a recipe on pypi.flet.dev / seeded into `dist/`. + +Path-hungry packages (read bundled DATA via `__file__`) also need a top-level meta.yaml `extract_packages:` list on Android — see `forge-error-catalogue` § the `sitepackages.zip` class. ## Desktop pre-validation when no desktop wheel exists diff --git a/.claude/skills/new-mobile-recipe/SKILL.md b/.claude/skills/new-mobile-recipe/SKILL.md index 253b1a15..3034c73c 100644 --- a/.claude/skills/new-mobile-recipe/SKILL.md +++ b/.claude/skills/new-mobile-recipe/SKILL.md @@ -215,7 +215,8 @@ Open `tests/test_.py` and replace the placeholder with a real smoke test: For ML/inference recipes, raise the bar from import-only to real compute: -- **Test-only deps** go in `tests/requirements.txt` (PEP 508 lines; the recipe-tester expands them into its pyproject — merged in #98). E.g. tflite-runtime's tests need numpy without numpy being a recipe dep of the test app. +- **Test-only deps** go in meta.yaml `test.requires` (a list of PEP 508 specs; the recipe-tester expands them into its pyproject `dependencies`). E.g. tflite-runtime's tests need numpy without numpy being a recipe dep of the test app. (This replaced the former per-recipe `tests/requirements.txt` file — `stage_recipe.sh` now errors if a stale one is present.) +- **Path-hungry packages need `extract_packages`** (Android): if the package reads a bundled DATA file via a real `__file__` path (not `importlib.resources`), add a top-level meta.yaml `extract_packages:` list of the IMPORT name(s) to ship extracted to disk. Under Flet 0.86 site-packages ship as a compressed `sitepackages.zip`, so a `__file__`-relative `open()`/`read_text()` raises `NotADirectoryError` otherwise. Examples: matplotlib, astropy, `sklearn`, thinc, `[spacy, thinc]`. (Full class + look-alikes it does NOT fix in `forge-error-catalogue` § the `sitepackages.zip` class.) - **A tiny committed model asset (~1KB) beats an import-only test** — CI's emulator then runs REAL inference: tflite's `dense_relu.tflite` (1KB, generated with desktop TF at a fixed seed), onnxruntime's `tiny_mlp.onnx` (170 bytes, hand-built relu graph). - **Big assets use the discovered-by-presence pattern**: the test looks for the model file next to itself — dropped there for the local on-device loop, absent in CI → the test skips (sherpa-onnx's silero VAD, 2.2MB). These MUST stay gitignored (the `recipes/*/tests/*.onnx` rule exists) — committing one flips the CI skip into a mandatory download-free run and embeds megabytes in the repo. - **No desktop wheel to test against? Validate the test LOGIC on desktop via a module-alias shim**: tflite's test ran on desktop TF with the `tflite_runtime.interpreter` module aliased to `tf.lite` — that caught a real math bug before ever touching a device. @@ -363,8 +364,8 @@ xcrun simctl launch booted com.flet.recipe-tester `PIP_FIND_LINKS` pointing at `dist/` is what makes the app bundle YOUR freshly-built wheels instead of pypi.flet.dev's — don't skip it. `tests/recipe-tester/README.md` documents the -runner itself (EXIT sentinel, `tests/requirements.txt` handling, what's committed vs -generated). +runner itself (EXIT sentinel, meta.yaml `test.requires` / `extract_packages` handling, +what's committed vs generated). `>>>>>>>>>> EXIT 0 <<<<<<<<<<` in console.log on both platforms = ready to ship. From 24ad035bb754e21783c86ad7a4e5a5da29ec5711 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 18:51:01 +0200 Subject: [PATCH 13/49] =?UTF-8?q?recipe:=20selectolax=20=E2=80=94=20drop?= =?UTF-8?q?=20vestigial=20`modest`=20namespace=20import=20(Flet=200.86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit selectolax/__init__.py did `from . import lexbor, modest, parser`, but `selectolax/modest/` is only a directory of Cython `.pxi` includes (parser.pyx does `include "modest/*.pxi"`, so they compile into parser.cpython-*.so) — there is no runtime `modest` module. Under normal CPython `from . import modest` resolves as an empty PEP 420 namespace package; under Flet 0.86 Android, site-packages ship as sitepackages.zip and serious-python only synthesizes an __init__ for dirs with an importable member, so include-only modest/ is invisible to zipimport and the import fails ("cannot import name 'modest' from partially initialized module 'selectolax'"). Drop the vestigial import — selectolax.modest has no runtime API; the engines are selectolax.parser (Modest backend) and selectolax.lexbor, both unaffected. Verified on-device (android arm64, flet 0.86.0.dev2): selectolax 2/2 passed (modest + lexbor parsers). Was: ImportError: cannot import name 'modest'. Also documents the symptom in forge-error-catalogue (the include-only namespace-dir entry). --- .../references/failure-catalogue.md | 21 ++++++++++++++++++ recipes/selectolax/meta.yaml | 5 ++++- recipes/selectolax/patches/mobile.patch | 22 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 recipes/selectolax/patches/mobile.patch diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 9af10ce3..6c1b69a4 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1037,6 +1037,27 @@ wins first on iOS, where the path is real). Verified: pycryptodome, pycryptodome --- +### `cannot import name '' from partially initialized module ''` for an include-only namespace dir (Android) + +**Cause:** `/__init__.py` does `from . import `, but `//` +has **no importable member** — only Cython `.pxi` includes / `.h`/`.c` / data +(compiled into a sibling `.so` at build time). Under normal CPython it's an empty +PEP 420 namespace package; under 0.86 serious_python's `synthesizePackageInits()` +only injects a synthetic `__init__` for dirs containing a `.py`/`.pyc`/`.soref`, so +the include-only dir is invisible to `zipimport`. selectolax: +`from . import lexbor, modest, parser` — `modest/` holds only `.pxi` (`include`d by +`parser.pyx`); `lexbor`/`parser` are real `.so` and load fine. + +**Fix:** if the import is **vestigial** (no runtime API — the real modules are the +sibling `.so`), drop it from `__init__` via `mobile.patch` (selectolax: +`from . import lexbor, parser`). If the dir genuinely must be importable, ship an +empty `//__init__.py` — but verify the build's `find_packages` / +`package_data` actually lands it in the wheel (selectolax's +`find_packages(include=["selectolax"])` would NOT, which is why the drop was cleaner +there). Verified: selectolax (drop). + +--- + ### `FileNotFoundError: Shared library with base name '' not found` (ctypes-by-`__file__` loader, Android) **Cause:** the loader gates each candidate on `Path.exists()` for a diff --git a/recipes/selectolax/meta.yaml b/recipes/selectolax/meta.yaml index 139d8565..645ce068 100644 --- a/recipes/selectolax/meta.yaml +++ b/recipes/selectolax/meta.yaml @@ -3,4 +3,7 @@ package: version: "0.4.10" build: - number: 10 + number: 11 + +patches: + - mobile.patch diff --git a/recipes/selectolax/patches/mobile.patch b/recipes/selectolax/patches/mobile.patch new file mode 100644 index 00000000..6465ccd2 --- /dev/null +++ b/recipes/selectolax/patches/mobile.patch @@ -0,0 +1,22 @@ +selectolax/__init__.py imports the `modest` submodule (`from . import lexbor, +modest, parser`), but `selectolax/modest/` is only a directory of Cython `.pxi` +include files, compiled into parser.cpython-*.so at build time (parser.pyx does +`include "modest/*.pxi"`). There is no runtime `modest` module. Under normal +CPython `from . import modest` resolves as an empty PEP 420 namespace package; +under Flet 0.86 Android, site-packages ship as `sitepackages.zip` and +serious-python only synthesizes an `__init__` for dirs that have an importable +member (`.py`/`.pyc`/`.soref`), so the include-only `modest/` is invisible to +zipimport and the import fails with "cannot import name 'modest' from partially +initialized module 'selectolax'". Drop the vestigial import — `selectolax.modest` +carries no runtime API; the engines are `selectolax.parser` (Modest backend) and +`selectolax.lexbor`, both unaffected. + +diff --git a/selectolax/__init__.py b/selectolax/__init__.py +--- a/selectolax/__init__.py ++++ b/selectolax/__init__.py +@@ -5,4 +5,4 @@ + __email__ = "me@rushter.com" + __version__ = "0.4.10" + +-from . import lexbor, modest, parser ++from . import lexbor, parser From b89676bce54541efbd5e62404d56e205a0226d6e Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 19:08:36 +0200 Subject: [PATCH 14/49] dev: add default for `PYTHON_BUILD_RUN_ID` if input is empty --- .github/workflows/build-wheels-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index da529007..672e469c 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -273,7 +273,7 @@ jobs: FORGE_PACKAGES: ${{ matrix.forge_packages }} PREBUILD_RECIPES: ${{ inputs.prebuild_recipes }} PLATFORM: ${{ matrix.platform }} - PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id }} + PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id != '' && inputs.python_build_run_id || '29193662574' }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PIP_FIND_LINKS: ${{ github.workspace }}/dist UV_PYTHON: ${{ inputs.python_version }} From fdb7743373ae99067a1eae1056dcb14e64bd55d1 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 19:45:31 +0200 Subject: [PATCH 15/49] =?UTF-8?q?recipe:=20opencv-python=20=E2=80=94=20loa?= =?UTF-8?q?d=20native=20cv2=20under=20top-level=20name=20on=20Android=20(F?= =?UTF-8?q?let=200.86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under Flet 0.86 Android the native cv2 extension is relocated to jniLibs (importable only as the submodule cv2.cv2 via a cv2/cv2.soref marker) and config.py is compiled to config.pyc, so cv2's stock loader dies two ways: "missing configuration file: ['config.py']", and — if the native is loaded as cv2.cv2 — "Submodule name should always start with a parent module name. Parent name: cv2.cv2. Submodule name: cv2" (OpenCV's compiled bindings require the top-level name "cv2"). mobile.patch note 3 inserts a fast-path in cv2/__init__.py after the recursion guard: resolve the relocated extension via find_spec('cv2.cv2') and load it through a fresh ExtensionFileLoader("cv2", origin) so its runtime __name__ is the required top-level "cv2", relink, and load the extra pure-python submodules. On desktop/non-0.86 find_spec is None → stock loader runs unchanged. Pairs with extract_packages: [cv2] (the extra-submodule scan os.listdir()s the package dir). Verified on-device (arm64) 4/4 against the byte-identical published 4.10.0.84 loader. Skills: dedicated cv2 entry + cross-refs in forge-error-catalogue. --- .claude/skills/forge-error-catalogue/SKILL.md | 12 +++- .../references/failure-catalogue.md | 56 ++++++++++++++-- recipes/opencv-python/meta.yaml | 8 ++- recipes/opencv-python/patches/mobile.patch | 67 +++++++++++++++++++ setup.sh | 2 +- 5 files changed, 134 insertions(+), 11 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 9ba6ade1..6dbcc898 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -18,8 +18,10 @@ description: >- (stripped *.pyi), the Flet 0.86 Android sitepackages.zip class (NotADirectoryError on a bundled data file -> extract_packages, untagged native .so ModuleNotFoundError -> forge ABI-tag, ctypes-by-__file__ loaders -> find_spec / - bare soname, jniLibs lib.so name collision -> static-link), hidden runtime - deps, or a stale on-device cache. Sibling + bare soname, jniLibs lib.so name collision -> static-link, a package whose + loader re-imports its own native under a fixed top-level name -> opencv cv2 + "missing configuration file: ['config.py']" / "Submodule name should always start + with a parent module name"), hidden runtime deps, or a stale on-device cache. Sibling of `new-mobile-recipe` (authoring), `native-recipe-bumps` (version bumps), `local-recipe-testing` (on-device testing), and `forge-ci` (CI triage); this one is the dedicated error -> fix reference. @@ -81,7 +83,11 @@ instead of re-deriving it. **`importlib.find_spec().origin`**; llama `FileNotFoundError` / ctypes `find_library`-None → **bare-soname load from jniLibs**; a top-level extension colliding with a `flet-lib*` at `lib.so` → **static-link (`host_build` + - `-l:lib.a`)**. Plus: `libc++_shared.so` not found, + `-l:lib.a`)**; opencv-python cv2 `missing configuration file: ['config.py']` / + `Submodule name should always start with a parent module name. Parent name: + cv2.cv2` → **load the native under top-level name `cv2` via + `ExtensionFileLoader("cv2", find_spec('cv2.cv2').origin)` + `extract_packages`**. + Plus: `libc++_shared.so` not found, **CMake OBJECT-lib not linked into the iOS `.framework` → undefined symbol at dlopen** (opencv-5 MLAS; green build ≠ loadable — verify with `nm -u`/`otool -L`), ctypes **"Unable to find … shared library"** (Pattern H loader / lib delivery), diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 6c1b69a4..9884d077 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -984,12 +984,13 @@ extract_packages: `stage_recipe.sh` reads it (via `read_meta_list.py`) into `[tool.flet.android].extract_packages`; serious_python unpacks those packages to disk (on `sys.path` before the zip) so `__file__` resolves to a real dir. Field is -in `src/forge/schema/meta-schema.yaml`. Verified: matplotlib, astropy. Two -look-alikes this does NOT fix: **python-magic** (`could not find any valid magic -files!` — `magic.mgc` lives in flet-libmagic's `opt/`, dropped by `copyOpt` which -copies only `**/*.so`) and **opencv-python** (`missing configuration file: -['config.py']` — cv2's loader needs the `.py` *source*, stripped by 0.86's `.pyc` -compile). +in `src/forge/schema/meta-schema.yaml`. Verified: matplotlib, astropy. **opencv-python** +also needs `extract_packages: [cv2]` — but that alone is NOT enough for it; its +loader has a separate config.py/native-name break (see the dedicated cv2 entry +below). One look-alike `extract_packages` does NOT fix at all: **python-magic** +(`could not find any valid magic files!` — `magic.mgc` lives in flet-libmagic's +`opt/`, dropped by `copyOpt` which copies only `**/*.so`, so extracting the python +package doesn't bring the data file back). --- @@ -1058,6 +1059,49 @@ there). Verified: selectolax (drop). --- +### `ImportError: OpenCV loader: missing configuration file: ['config.py']` / `Bindings generation error. Submodule name should always start with a parent module name. Parent name: cv2.cv2. Submodule name: cv2` (opencv-python, Android) + +**Cause:** a *two-part* 0.86 break in cv2's stock loader +(`cv2/__init__.py::bootstrap`). Stock OpenCV: (1) `exec()`s a `config.py` / +`config-3.py` to discover a directory of `.so` files, then (2) pops the `cv2` +package from `sys.modules` and **re-imports it as the top-level module `cv2`** so the +native binary's `__name__` is exactly `cv2` — its compiled-in bindings register +submodules (`cv2.dnn`, `cv2.gapi`, …) and assert each *starts with the parent name*. +Under 0.86 Android both halves break: `config.py` is compiled to `config.pyc`, so +`os.path.exists('config.py')` is False → **"missing configuration file"**; and the +native `cv2.cv2` extension is relocated to `jniLibs` (importable **only** as the +submodule `cv2.cv2` via a `cv2/cv2.soref` marker, never as top-level `cv2`). If you +naively `import cv2.cv2`, OpenCV's C init sees parent name `cv2.cv2`, the compiled +submodule `cv2` no longer starts with it, and it aborts with **"Submodule name +should always start with a parent module name."** (The native `.so`'s DT_NEEDED are +all system libs — `libcamera2ndk`/`libmediandk`/`liblog`/… — so this is **not** a +dlopen failure; don't chase a missing lib.) + +**Fix:** patch `cv2/__init__.py` to insert a fast-path right after the +`sys.OpenCV_LOADER = True` recursion guard that bypasses the config machinery and +loads the native under the **required top-level name `cv2`**: +```python +import importlib.util as _ilu, importlib.machinery as _ilm +_sub = _ilu.find_spec(__name__ + ".cv2") # soref finder → real jniLibs .so path +if _sub is not None and getattr(_sub, "origin", None): + _loader = _ilm.ExtensionFileLoader("cv2", _sub.origin) # name MUST be "cv2", not "cv2.cv2" + _native = _ilu.module_from_spec(_ilu.spec_from_loader("cv2", _loader)) + _loader.exec_module(_native) # runtime __name__ == "cv2" → C init happy + # relink _native.__dict__ into globals(), del sys.OpenCV_LOADER, + # then run __collect_extra_submodules()/__load_extra_py_code_for_module() + return +# else fall through to the stock loader (desktop / non-0.86) +``` +Pair with **`extract_packages: [cv2]`** in meta.yaml (the extra-submodule scan does +`os.listdir()` on the package dir, so cv2 must ship extracted, not zipped). On +desktop `find_spec('cv2.cv2')` is None → stock loader runs unchanged. General tell: +any package whose loader **re-imports its own native under a specific top-level +name** — reproduce the exact name via a fresh `ExtensionFileLoader(name, origin)`, +never via `import pkg.pkg`. Verified on-device (arm64) 4/4 against the byte-identical +4.10 loader; ported to the 5.0.0.93 recipe. + +--- + ### `FileNotFoundError: Shared library with base name '' not found` (ctypes-by-`__file__` loader, Android) **Cause:** the loader gates each candidate on `Path.exists()` for a diff --git a/recipes/opencv-python/meta.yaml b/recipes/opencv-python/meta.yaml index 72c86348..8e5fff96 100644 --- a/recipes/opencv-python/meta.yaml +++ b/recipes/opencv-python/meta.yaml @@ -2,8 +2,14 @@ package: name: opencv-python version: "5.0.0.93" +# Ship cv2 as a real extracted directory (not inside sitepackages.zip) so the +# loader's extra-submodule scan can os.listdir() the package; see mobile.patch +# note 3 for the Flet 0.86 native-relocation loader fix this pairs with. +extract_packages: + - cv2 + build: - number: 1 + number: 2 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/opencv-python/patches/mobile.patch b/recipes/opencv-python/patches/mobile.patch index 631c9c42..32a1faae 100644 --- a/recipes/opencv-python/patches/mobile.patch +++ b/recipes/opencv-python/patches/mobile.patch @@ -20,6 +20,27 @@ Make opencv-python 5.0 cross-compile AND load on Flet's mobile targets. compile out and opencv_dnn falls back to its built-in SGEMM. Android keeps MLAS (it static-links fine; verified on-device). +3. Teach the cv2 loader (modules/python/package/cv2/__init__.py) to boot under + Flet 0.86. OpenCV's stock bootstrap exec()s a config.py to discover a directory + of `.so` files, then pops the cv2 package and re-imports it as the top-level + module "cv2" so the native binary loads under that exact name (its compiled-in + bindings register submodules like `cv2.dnn` and assert each starts with the + parent name). Under Flet 0.86 Android neither half survives: `config.py` is + compiled to `config.pyc` (so `os.path.exists('config.py')` is False -> + "missing configuration file"), and the native `cv2.cv2` extension is relocated + to jniLibs (importable only as the submodule `cv2.cv2` via a `cv2/cv2.soref` + marker, never as top-level `cv2`). Loading it as `cv2.cv2` makes OpenCV abort + with "Submodule name should always start with a parent module name. Parent + name: cv2.cv2. Submodule name: cv2". The fix inserts a fast-path right after + the recursion guard: resolve the relocated extension via `find_spec('cv2.cv2')` + (the soref finder yields its real jniLibs path), then load it through a fresh + `ExtensionFileLoader("cv2", origin)` so its runtime `__name__` is the required + top-level "cv2", relink its symbols onto the package, and load the extra pure- + python submodules. On desktop / non-0.86 builds `find_spec('cv2.cv2')` returns + None and the stock loader runs unchanged. Pairs with `extract_packages: [cv2]` + in meta.yaml so the package ships as a real dir (the extra-submodule scan does + `os.listdir` on it). Verified on-device (arm64) 4/4. + diff --git a/opencv/CMakeLists.txt b/opencv/CMakeLists.txt index d6812aa..7555914 100644 --- a/opencv/CMakeLists.txt @@ -472,3 +493,49 @@ index a4bc8e5..b86efd8 100644 if (pyopencv_VideoWriter_getp(self, obj_getp) && obj_getp && *obj_getp) { (*obj_getp)->release(); } + +diff --git a/opencv/modules/python/package/cv2/__init__.py b/opencv/modules/python/package/cv2/__init__.py +--- a/opencv/modules/python/package/cv2/__init__.py ++++ b/opencv/modules/python/package/cv2/__init__.py +@@ -75,6 +75,41 @@ def bootstrap(): + print(sys.path) + raise ImportError('ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.') + sys.OpenCV_LOADER = True ++ # --- mobile-forge fast-path ------------------------- ++ # Under Flet 0.86 the native cv2 extension is relocated out of the wheel into ++ # jniLibs (resolved via a cv2/cv2.soref marker) and config.py is compiled to ++ # config.pyc, so OpenCV's stock loader -- which exec()s config.py to find a ++ # directory of .so files and re-imports it as the top-level module "cv2" -- ++ # cannot run. Load the native module directly under the top-level name "cv2" ++ # (OpenCV's compiled bindings hard-require that exact name) and relink. ++ try: ++ import importlib.util as _ilu ++ import importlib.machinery as _ilm ++ _sub_spec = _ilu.find_spec(__name__ + ".cv2") ++ except Exception: ++ _sub_spec = None ++ if _sub_spec is not None and getattr(_sub_spec, "origin", None): ++ _dbg = hasattr(sys, "OpenCV_LOADER_DEBUG") ++ _loader = _ilm.ExtensionFileLoader("cv2", _sub_spec.origin) ++ _spec = _ilu.spec_from_loader("cv2", _loader) ++ _native = _ilu.module_from_spec(_spec) ++ _loader.exec_module(_native) ++ _g = globals() ++ setattr(sys.modules[__name__], "_native", _native) ++ for _k, _v in _native.__dict__.items(): ++ if _k not in ("__file__", "__loader__", "__spec__", "__name__", "__package__") and _k not in _g: ++ _g[_k] = _v ++ try: ++ del sys.OpenCV_LOADER ++ except Exception: ++ pass ++ try: ++ for _s in __collect_extra_submodules(_dbg): ++ __load_extra_py_code_for_module("cv2", _s, _dbg) ++ except Exception: ++ pass ++ return ++ # --- end mobile-forge fast-path ----------------------------------------- + + DEBUG = False + if hasattr(sys, 'OpenCV_LOADER_DEBUG'): diff --git a/setup.sh b/setup.sh index 6efc9aba..7da11d8c 100755 --- a/setup.sh +++ b/setup.sh @@ -34,7 +34,7 @@ if ! command -v uv &> /dev/null; then fi # Pinned flet-dev/python-build release to consume (date-keyed YYYYMMDD, PBS-style). -PYTHON_BUILD_RELEASE="${PYTHON_BUILD_RELEASE:-20260701}" +PYTHON_BUILD_RELEASE="${PYTHON_BUILD_RELEASE:-20260712}" # Resolve a full X.Y.Z from a bare X.Y minor using the pinned release's # manifest.json (downloaded + cached under downloads/). Echoes the full version; From d05bf43cfbd5781384542c1e2f3a231904531e09 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 20:30:42 +0200 Subject: [PATCH 16/49] =?UTF-8?q?recipe:=20python-magic=20=E2=80=94=20ship?= =?UTF-8?q?=20magic.mgc=20in-wheel=20+=20load=20from=20memory=20(Flet=200.?= =?UTF-8?q?86=20Android)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under Flet 0.86 Android the compiled magic database never reached the device: magic.mgc ships in flet-libmagic's opt/share/misc/ tree, but serious-python's copyOpt copies only **/*.so out of a flet-lib* opt/ tree (and splitSitePackages skips opt/), so it landed in neither jniLibs nor sitepackages.zip. The loader pointed magic_file at /opt/share/misc/magic.mgc, os.path.exists() was False, and magic_load(cookie, None) failed "could not find any valid magic files!". Fix (mobile.patch note 2 + meta script_env), two parts: (a) setup.py copies magic.mgc INTO the `magic` package at build time (path fed via the new FLET_MAGIC_MGC script_env, pointing at flet-libmagic's cross-env opt/) and adds it to package_data, so the DB ships inside python-magic's own wheel and rides 0.86's all-files sitepackages.zip. Arch-neutral -> wheel stays pure; flet-libmagic remains the per-platform Requires-Dist for the .so. (b) magic/__init__.py loads it: real path when one exists (iOS/desktop/ extract_packages), else bind magic_load_buffers (public in file>=5.32) and load the DB from an in-memory buffer read via importlib.resources (reads through zipimport). Buffer stashed on the Magic instance (kept alive by the module-level _instances cache) since libmagic references it in place. Zero consumer config (no extract_packages needed). Verified on-device (arm64) 2/2: libmagic loads, magic.from_buffer(png) -> image/png. Built libmagic.so confirmed to export magic_load_buffers; magic.mgc (10.3 MB) confirmed in the wheel's sitepackages.zip. --- recipes/python-magic/meta.yaml | 15 +- recipes/python-magic/patches/mobile.patch | 177 +++++++++++++++++++--- 2 files changed, 166 insertions(+), 26 deletions(-) diff --git a/recipes/python-magic/meta.yaml b/recipes/python-magic/meta.yaml index 92a46d63..2feaab9a 100644 --- a/recipes/python-magic/meta.yaml +++ b/recipes/python-magic/meta.yaml @@ -3,14 +3,21 @@ package: version: "0.4.27" build: - number: 1 + number: 2 + script_env: + # flet-libmagic (the host dep below) installs the compiled magic database + # into the cross-env at opt/share/misc/magic.mgc. Hand setup.py that path so + # it copies magic.mgc INTO the python-magic wheel (see mobile.patch note 2): + # Flet 0.86 Android's copyOpt drops non-.so files from a flet-lib* opt/ tree, + # so the DB must ride python-magic's own (all-files) sitepackages.zip instead. + FLET_MAGIC_MGC: "{platlib}/opt/share/misc/magic.mgc" requirements: host: # python-magic is a pure-Python ctypes wrapper that dlopens libmagic at - # runtime (loader + magic-db default patched by mobile.patch). Ship libmagic - # and its magic.mgc database on both platforms; fix_wheel turns this into a - # Requires-Dist so pip pulls it. + # runtime (loader + magic-db patched by mobile.patch). Ship libmagic (the .so + # it dlopens) and, via FLET_MAGIC_MGC above, its magic.mgc database; fix_wheel + # turns this into a Requires-Dist so pip pulls the platform .so. - flet-libmagic 5.46 patches: diff --git a/recipes/python-magic/patches/mobile.patch b/recipes/python-magic/patches/mobile.patch index 9fcb51f0..b5cc5a89 100644 --- a/recipes/python-magic/patches/mobile.patch +++ b/recipes/python-magic/patches/mobile.patch @@ -1,8 +1,130 @@ +Make python-magic find its libmagic AND its magic database on Flet's mobile +targets under Flet 0.86. + +1. libmagic loader (magic/loader.py). flet-libmagic ships libmagic under + /opt/lib and serious-python surfaces it (Android jniLibs / an + iOS framework). ctypes' find_library('magic') returns None on iOS/Android, so + also yield the bundled opt/lib path plus the bare sonames the platform linker + resolves ("libmagic.so" from jniLibs on Android, "libmagic.fwork" on iOS). + +2. Ship + load the magic database (setup.py + magic/__init__.py). The compiled + magic database (magic.mgc) is built by flet-libmagic and installed into its + opt/share/misc/ tree. Under Flet 0.86 Android, serious-python's copyOpt copies + ONLY **/*.so out of a flet-lib* opt/ tree (and splitSitePackages skips opt/ + entirely), so magic.mgc reaches neither jniLibs nor the sitepackages.zip -- the + old loader pointed magic_file at /opt/share/misc/magic.mgc, + os.path.exists() was False on device, and magic_load(cookie, None) failed with + "could not find any valid magic files!". Fix, in two parts: + + (a) setup.py copies magic.mgc INTO the `magic` package at build time (path fed + via the FLET_MAGIC_MGC script_env, pointing at flet-libmagic's cross-env + opt/) and declares it as package_data, so it ships inside python-magic's + own wheel -- which rides 0.86's stored sitepackages.zip (that keeps all + package files, unlike an opt/ tree). magic.mgc is arch-neutral, so the + wheel stays py3-none-any; flet-libmagic remains the per-platform + Requires-Dist for the .so. + + (b) magic/__init__.py loads it. iOS / desktop / Android extract_packages -> + magic.mgc is a real file, loaded by path (magic_load). Flet 0.86 Android -> + the `magic` package is inside a stored zip with no real filesystem path, so + magic_load()'s fopen() can't reach it; bind magic_load_buffers (public in + file >= 5.32) and load the DB from an in-memory buffer whose bytes are read + via importlib.resources (which reads through zipimport). libmagic + references the buffer in place (MAP_TYPE_USER, no copy), so it is stashed on + the Magic instance to outlive the cookie. The binding is guarded so import + still works against an older libmagic lacking the symbol. + +diff --git a/magic/__init__.py b/magic/__init__.py +index bab7c7b..f0032db 100644 +--- a/magic/__init__.py ++++ b/magic/__init__.py +@@ -70,7 +70,22 @@ class Magic: + self.cookie = magic_open(self.flags) + self.lock = threading.Lock() + +- magic_load(self.cookie, magic_file) ++ # mobile-forge: resolve the magic database. iOS / desktop / Android ++ # extract_packages -> magic.mgc is a real file (load by path). Flet 0.86 ++ # Android -> the `magic` package ships inside a stored sitepackages.zip ++ # with no real filesystem path, so magic_load()'s fopen() can't reach ++ # magic.mgc; load the bundled DB from memory instead. libmagic references ++ # the buffer IN PLACE (no copy), so it is stashed on self and must outlive ++ # the cookie (closed in __del__ before self is released). ++ if magic_file is None: ++ magic_file = _bundled_magic_db_path() ++ try: ++ magic_load(self.cookie, magic_file) ++ except MagicException: ++ if magic_file is not None: ++ raise ++ self._magic_db_buffer = magic_load_buffers( ++ self.cookie, _bundled_magic_db_bytes()) + + # MAGIC_EXTENSION was added in 523 or 524, so bail if + # it doesn't appear to be available +@@ -332,6 +347,57 @@ def magic_load(cookie, filename): + return _magic_load(cookie, coerce_filename(filename)) + + ++# mobile-forge: bind magic_load_buffers (file >= 5.32) so a compiled magic DB can ++# be loaded from memory. Under Flet 0.86 Android the `magic` package ships inside a ++# stored sitepackages.zip with no real path, so magic.mgc cannot be fopen()'d by ++# magic_load(); we read its bytes via importlib.resources and hand them to libmagic ++# directly. Guarded so `import magic` still works against an older libmagic that ++# lacks the symbol. ++try: ++ _magic_load_buffers = libmagic.magic_load_buffers ++ _magic_load_buffers.restype = c_int ++ _magic_load_buffers.argtypes = [magic_t, POINTER(c_void_p), POINTER(c_size_t), c_size_t] ++ _magic_load_buffers.errcheck = errorcheck_negative_one ++except AttributeError: ++ _magic_load_buffers = None ++ ++ ++def magic_load_buffers(cookie, data): ++ """Load a compiled magic database from the bytes ``data``. ++ ++ libmagic references the buffer IN PLACE (MAP_TYPE_USER, no copy), so the ++ returned ctypes buffer MUST be kept alive as long as ``cookie`` is used. ++ """ ++ if _magic_load_buffers is None: ++ raise MagicException(b"libmagic has no magic_load_buffers") ++ buf = ctypes.create_string_buffer(data, len(data)) ++ buffers = (c_void_p * 1)(ctypes.cast(buf, c_void_p)) ++ sizes = (c_size_t * 1)(len(data)) ++ _magic_load_buffers(cookie, buffers, sizes, 1) ++ return buf ++ ++ ++def _bundled_magic_db_path(): ++ """A real on-disk magic.mgc path if one exists (iOS real dir, desktop, or ++ Android extract_packages), else None (Flet 0.86 Android sitepackages.zip).""" ++ import os ++ here = os.path.dirname(os.path.abspath(__file__)) ++ for p in ( ++ os.path.join(here, "magic.mgc"), # bundled in this package ++ os.path.join(os.path.dirname(here), "opt", "share", "misc", "magic.mgc"), # legacy flet-libmagic opt/ ++ ): ++ if os.path.exists(p): ++ return p ++ return None ++ ++ ++def _bundled_magic_db_bytes(): ++ """Bytes of the bundled magic/magic.mgc -- works whether `magic` is a real dir ++ or inside Flet 0.86's stored sitepackages.zip (read via zipimport.get_data).""" ++ from importlib.resources import files ++ return files(__package__).joinpath("magic.mgc").read_bytes() ++ ++ + magic_setflags = libmagic.magic_setflags + magic_setflags.restype = c_int + magic_setflags.argtypes = [magic_t, c_int] diff --git a/magic/loader.py b/magic/loader.py -index 931f161..60479a0 100644 +index 931f161..7d4cd08 100644 --- a/magic/loader.py +++ b/magic/loader.py -@@ -8,6 +8,16 @@ def _lib_candidates(): +@@ -8,6 +8,17 @@ def _lib_candidates(): yield find_library('magic') @@ -15,29 +137,40 @@ index 931f161..60479a0 100644 + yield os.path.join(_optlib, 'libmagic.so') # generic shipped path + yield 'libmagic.so' # Android jniLibs + yield 'libmagic.dylib' # desktop fallback ++ + if sys.platform == 'darwin': paths = [ -diff --git a/magic/__init__.py b/magic/__init__.py -index bab7c7b..e389603 100644 ---- a/magic/__init__.py -+++ b/magic/__init__.py -@@ -70,6 +70,17 @@ class Magic: - self.cookie = magic_open(self.flags) - self.lock = threading.Lock() +diff --git a/setup.py b/setup.py +index 06386c3..9784d89 100644 +--- a/setup.py ++++ b/setup.py +@@ -12,6 +12,18 @@ def read(file_name): + encoding='utf-8') as f: + return f.read() -+ # mobile-forge: on iOS/Android there is no system magic database, so -+ # default to the magic.mgc shipped by the flet-libmagic wheel at -+ # /opt/share/misc/magic.mgc when no explicit db is given. -+ if magic_file is None: -+ import os -+ _bundled_db = os.path.join( -+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), -+ 'opt', 'share', 'misc', 'magic.mgc') -+ if os.path.exists(_bundled_db): -+ magic_file = _bundled_db ++# mobile-forge: bundle the compiled magic database (magic.mgc) INSIDE this wheel. ++# flet-libmagic (a host build dep) installs magic.mgc into the cross-env opt/ ++# tree; forge hands us its path via FLET_MAGIC_MGC. Copy it into the `magic` ++# package so it rides Flet 0.86 Android's stored sitepackages.zip (which keeps ++# all package files) -- serious-python's copyOpt drops non-.so files from a ++# flet-lib* opt/ tree, so magic.mgc never reaches the device otherwise. ++_mgc = os.environ.get("FLET_MAGIC_MGC") ++if _mgc and os.path.exists(_mgc): ++ import shutil ++ shutil.copyfile( ++ _mgc, os.path.join(os.path.dirname(__file__) or ".", "magic", "magic.mgc")) + - magic_load(self.cookie, magic_file) - - # MAGIC_EXTENSION was added in 523 or 524, so bail if + setuptools.setup( + name='python-magic', + description='File type identification using libmagic', +@@ -23,7 +35,7 @@ setuptools.setup( + long_description_content_type='text/markdown', + packages=['magic'], + package_data={ +- 'magic': ['py.typed', '*.pyi', '**/*.pyi'], ++ 'magic': ['py.typed', '*.pyi', '**/*.pyi', 'magic.mgc'], + }, + keywords="mime magic file", + license="MIT", From 87e958ecb5c3603c7267b3354e0ab041e9dd955c Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 20:33:08 +0200 Subject: [PATCH 17/49] =?UTF-8?q?skills:=20python-magic=200.86=20fix=20(da?= =?UTF-8?q?ta=20file=20in=20flet-lib=20opt/=20=E2=86=92=20ship=20in-wheel?= =?UTF-8?q?=20+=20load=20from=20memory)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/forge-error-catalogue/SKILL.md | 8 +++- .../references/failure-catalogue.md | 47 +++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 6dbcc898..63ec2133 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -86,8 +86,12 @@ instead of re-deriving it. `-l:lib.a`)**; opencv-python cv2 `missing configuration file: ['config.py']` / `Submodule name should always start with a parent module name. Parent name: cv2.cv2` → **load the native under top-level name `cv2` via - `ExtensionFileLoader("cv2", find_spec('cv2.cv2').origin)` + `extract_packages`**. - Plus: `libc++_shared.so` not found, + `ExtensionFileLoader("cv2", find_spec('cv2.cv2').origin)` + `extract_packages`**; + a runtime DATA file that lives in a sibling `flet-lib*` `opt/` tree (dropped by + `copyOpt`, which copies only `.so`) → python-magic `could not find any valid magic + files!` → **ship the data file in the consumer's own wheel (`script_env` + `{platlib}/opt/…` copy in setup.py + `package_data`) + load from memory + (`importlib.resources` bytes → `magic_load_buffers`)**. Plus: `libc++_shared.so` not found, **CMake OBJECT-lib not linked into the iOS `.framework` → undefined symbol at dlopen** (opencv-5 MLAS; green build ≠ loadable — verify with `nm -u`/`otool -L`), ctypes **"Unable to find … shared library"** (Pattern H loader / lib delivery), diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 9884d077..feb9d28f 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -987,10 +987,12 @@ disk (on `sys.path` before the zip) so `__file__` resolves to a real dir. Field in `src/forge/schema/meta-schema.yaml`. Verified: matplotlib, astropy. **opencv-python** also needs `extract_packages: [cv2]` — but that alone is NOT enough for it; its loader has a separate config.py/native-name break (see the dedicated cv2 entry -below). One look-alike `extract_packages` does NOT fix at all: **python-magic** -(`could not find any valid magic files!` — `magic.mgc` lives in flet-libmagic's -`opt/`, dropped by `copyOpt` which copies only `**/*.so`, so extracting the python -package doesn't bring the data file back). +below). And it does NOT help when the missing data file lives in a *flet-lib\** +`opt/` tree rather than the python package itself: **python-magic** +(`could not find any valid magic files!` — `magic.mgc` is dropped by `copyOpt` +because `copyOpt` copies only `**/*.so`); the fix there is to ship the data file +**inside the python package's own wheel** + load it from memory — see the dedicated +python-magic entry below. --- @@ -1102,6 +1104,43 @@ never via `import pkg.pkg`. Verified on-device (arm64) 4/4 against the byte-iden --- +### `magic.MagicException: could not find any valid magic files!` (python-magic, Android) — a data file that lives in a `flet-lib*` `opt/` tree + +**Cause:** the runtime data file a package needs ships in a **separate `flet-lib*` +wheel's `opt/` tree**, not in the package itself, and 0.86's `copyOpt` copies **only +`.so`** out of an `opt/` tree (and `splitSitePackages` skips `opt/` entirely), so the +data file reaches **neither** jniLibs **nor** `sitepackages.zip`. python-magic: +libmagic's compiled DB `magic.mgc` ships in `flet-libmagic`'s +`opt/share/misc/magic.mgc`; the `.so` (`libmagic.so`) is relocated to jniLibs fine, +but `magic.mgc` is dropped, so `magic_load(cookie, )` / +`magic_load(cookie, None)` (falls back to libmagic's compiled-in default, absent on +device) fails. **`extract_packages` does NOT help** — extracting the *python* package +to disk doesn't bring back a file that was never in it. iOS is unaffected (real dir). + +**Fix (two parts):** ship the data file **inside the consuming package's own wheel** +(which rides 0.86's all-files `sitepackages.zip`, unlike an `opt/` tree), then load it +in a way that survives the stored zip. +- *Delivery:* forge's `PythonPackageBuilder` has no copy hook and `patch -p1` can't + carry a multi-MB binary, so copy it in from `setup.py` at build time, fed the path + via a meta `build.script_env` that templates the `flet-lib*` host dep's cross-env + path — `FLET_MAGIC_MGC: "{platlib}/opt/share/misc/magic.mgc"` (the `{platlib}/opt/…` + idiom; the `flet-lib*` host dep installs there). Add the file to `package_data`. +- *Load:* real path when one exists (iOS/desktop/`extract_packages`), else read the + file's **bytes** via `importlib.resources.files(__package__).joinpath(...).read_bytes()` + (routes through `zipimport.get_data`, so it works from a stored zip) and hand them to + a **from-memory** API — for libmagic, bind `magic_load_buffers` (public in file + ≥ 5.32) and keep the ctypes buffer alive past the cookie (libmagic references it in + place; stash it on the object the module caches). Guard the binding + (`except AttributeError`) so import survives an older lib. +Zero consumer config (no `extract_packages`). General tell: a package's runtime data +lives in a sibling `flet-lib*` `opt/` and you see a "can't find " at import → +relocate the data into the consumer's wheel + load from memory. Verify with +`nm -D lib.so | grep ` (must be exported) and `unzip -l` the +APK's `sitepackages.zip` for the data file. Verified on-device (arm64) 2/2: +python-magic (`magic_load_buffers`, `magic.mgc` 10.3 MB in `sitepackages.zip`). + +--- + ### `FileNotFoundError: Shared library with base name '' not found` (ctypes-by-`__file__` loader, Android) **Cause:** the loader gates each candidate on `Path.exists()` for a From ce098c12f1d43e81f53f07a69fcd5dbe4df0ccac Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 21:19:22 +0200 Subject: [PATCH 18/49] recipe: bump build numbers for 0.86 loader-patch changes (llama-cpp-python, opaque, pycryptodome, pycryptodomex, pysodium) These recipes' mobile.patch changed (Flet 0.86 ctypes-loader fixes) but their build number was left unchanged, so a published wheel at the old build number would shadow the fix. Bump so the patched wheel wins. --- recipes/llama-cpp-python/meta.yaml | 2 +- recipes/opaque/meta.yaml | 2 +- recipes/pycryptodome/meta.yaml | 2 +- recipes/pycryptodomex/meta.yaml | 2 +- recipes/pysodium/meta.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index c2fd70c6..28fee473 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -3,7 +3,7 @@ package: version: "0.3.32" build: - number: 1 + number: 2 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/opaque/meta.yaml b/recipes/opaque/meta.yaml index 9cea9d7f..e496f708 100644 --- a/recipes/opaque/meta.yaml +++ b/recipes/opaque/meta.yaml @@ -3,7 +3,7 @@ package: version: "1.0.0" build: - number: 1 + number: 2 requirements: host: diff --git a/recipes/pycryptodome/meta.yaml b/recipes/pycryptodome/meta.yaml index cb097041..2bca12ef 100644 --- a/recipes/pycryptodome/meta.yaml +++ b/recipes/pycryptodome/meta.yaml @@ -3,7 +3,7 @@ package: version: "3.23.0" build: - number: 1 + number: 2 # pycryptodome's internal Crypto/Util/_raw_api.py tries a cffi-based # fast path first, and only falls back to ctypes.pythonapi.PyObject_GetBuffer diff --git a/recipes/pycryptodomex/meta.yaml b/recipes/pycryptodomex/meta.yaml index 5cea9b16..958a500e 100644 --- a/recipes/pycryptodomex/meta.yaml +++ b/recipes/pycryptodomex/meta.yaml @@ -3,7 +3,7 @@ package: version: "3.23.0" build: - number: 1 + number: 2 # Same fix rationale as recipes/pycryptodome/meta.yaml — pycryptodomex is # the sister package (same code under `Cryptodome.*` namespace). Without diff --git a/recipes/pysodium/meta.yaml b/recipes/pysodium/meta.yaml index 1923d0e8..b4f6e098 100644 --- a/recipes/pysodium/meta.yaml +++ b/recipes/pysodium/meta.yaml @@ -3,7 +3,7 @@ package: version: "0.7.18" build: - number: 10 + number: 11 patches: - mobile.patch From 033cfd8643ba4cfa62b38601d820bf20e99ad0b4 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Sun, 12 Jul 2026 21:53:50 +0200 Subject: [PATCH 19/49] skills: apsw 0.86 = serious_python _SorefFinder gap (package whose __init__ IS the native ext) + local SP-override test technique --- .../references/failure-catalogue.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index feb9d28f..91663727 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1104,6 +1104,38 @@ never via `import pkg.pkg`. Verified on-device (arm64) 4/4 against the byte-iden --- +### `AttributeError: module 'apsw' has no attribute 'Connection'` / a package whose `__init__` IS the native extension imports empty (Android) — a serious_python `_SorefFinder` gap + +**Cause:** the package's `__init__` is *itself* a C extension — apsw's setup builds +`Extension("apsw.__init__", …)` → `apsw/__init__.cpython-*.so` (dotted import name +`apsw`). Under 0.86 that `.so` is relocated to jniLibs and its marker is written at +**`apsw/__init__.soref`**, but `_sp_bootstrap._SorefFinder.find_spec("apsw")` only +probed `".soref"` (= `apsw.soref`) → miss → `None`. Meanwhile +`synthesizePackageInits()` injects an **empty `apsw/__init__.py`**, which then wins, +so `import apsw` yields an empty module and any `apsw.Connection` / `apsw.apswversion()` +raises AttributeError. **This is a serious_python bug, not a recipe/forge bug** — the +recipe's mobile.patch (SQLite amalgamation cross-compile) is unrelated; the `.so` +exports `PyInit_apsw` (not `PyInit___init__`) on non-Windows, so it *can* be loaded +under the top-level name. + +**Fix (in serious_python, `serious_python_android/python/_sp_bootstrap.py`):** when +`".soref"` misses, fall back to **`"/__init__.soref"`**; load via +`ExtensionFileLoader(fullname, origin)` (its `PyInit_` matches) and mark the +result a **package** — set `spec.submodule_search_locations = [/]` so pure-Python submodules (`apsw.ext`, `apsw._unicode`, …) resolve +via the normal zipimport/FileFinder machinery (`_read_marker` must also return which +entry it hit). The `_SorefFinder` sits at `meta_path[0]`, so once it resolves `apsw` +it beats the synthesized empty `__init__`. Validated on-device (Android arm64) with a +local serious_python override: apsw 3.53.2.0 imports + runs an in-memory SQLite +round-trip 2/2. **Caveat:** the fix must land in a *released* serious_python before +CI / real consumers get it — until then apsw stays red in CI (do not include it in a +recipe CI dispatch). To test an unreleased serious_python locally, point the app at +your checkout via `[tool.flet.flutter.pubspec.dependency_overrides]` (path deps for +`serious_python` + `serious_python_android` + `serious_python_platform_interface`) in +the generated pyproject. + +--- + ### `magic.MagicException: could not find any valid magic files!` (python-magic, Android) — a data file that lives in a `flet-lib*` `opt/` tree **Cause:** the runtime data file a package needs ships in a **separate `flet-lib*` From 413951cf135c1c30d565ce61b4b69d2e4026c6af Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 00:50:30 +0200 Subject: [PATCH 20/49] =?UTF-8?q?recipe:=20jq=20=E2=80=94=20gate=20the=20s?= =?UTF-8?q?tatic-link=20syntax=20per=20platform=20(fix=20iOS=20build)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The static-link change used GNU ld's `-l:libjq.a`/`-l:libonig.a` for BOTH platforms, but Apple's ld64 has no `-l:` syntax, so the iOS build failed with `ld: library ':libjq.a' not found` on every Python. The jniLibs `libjq.so` name collision the `-l:` form guards against is Android-only; flet-libjq ships only static `.a` on both platforms, so on iOS a plain `-ljq`/`-lonig` resolves to the static archive just the same. Gate the link flags on `sysconfig.get_platform()`: keep `-l:libjq.a` on Android (the on-device-verified form), use `-ljq`/`-lonig` on iOS. Verified locally on BOTH: iphonesimulator arm64 builds and statically embeds 107 jq_* symbols with no external libjq/libonig dylib dep; android arm64 unchanged (no libjq DT_NEEDED). Build 2 -> 3. --- recipes/jq/meta.yaml | 2 +- recipes/jq/patches/mobile.patch | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/recipes/jq/meta.yaml b/recipes/jq/meta.yaml index 5514ef1d..01fc5d93 100644 --- a/recipes/jq/meta.yaml +++ b/recipes/jq/meta.yaml @@ -11,7 +11,7 @@ requirements: - flet-libjq 1.7.1 build: - number: 2 + number: 3 script_env: JQPY_USE_SYSTEM_LIBS: 1 diff --git a/recipes/jq/patches/mobile.patch b/recipes/jq/patches/mobile.patch index 4f01db61..d16bed75 100644 --- a/recipes/jq/patches/mobile.patch +++ b/recipes/jq/patches/mobile.patch @@ -8,19 +8,27 @@ self-contained and no libjq.so is bundled at runtime. JQPY_USE_SYSTEM_LIBS still selects flet-libjq's prebuilt libs, now via `-l:libjq.a` (static) not `-ljq`. diff --git a/setup.py b/setup.py +index c79e917..7c2dd1b 100644 --- a/setup.py +++ b/setup.py -@@ -103,7 +103,12 @@ - +@@ -103,7 +103,19 @@ use_system_libs = bool(os.environ.get("JQPY_USE_SYSTEM_LIBS")) + if use_system_libs: jq_build_ext = build_ext - link_args_deps = ["-ljq", "-lonig"] -+ # mobile-forge: static-link flet-libjq's prebuilt archives so the extension -+ # is self-contained. The `jq` module mangles to jniLibs `libjq.so` on Android -+ # -- the same basename as a shared libjq -- so a dynamic `-ljq` would bundle a -+ # colliding libjq.so. `-l:NAME.a` forces the static archives from flet-libjq's -+ # opt/lib (wired via requirements.host_build). -+ link_args_deps = ["-l:libjq.a", "-l:libonig.a"] ++ # mobile-forge: static-link flet-libjq's prebuilt archives so the extension is ++ # self-contained (flet-libjq ships ONLY static .a -- see recipes/flet-libjq/ ++ # build.sh). On Android the `jq` module mangles to jniLibs `libjq.so` -- the ++ # same basename a shared libjq would take -- so force the exact static archive ++ # via GNU ld's `-l:NAME.a` to be certain nothing bundles a colliding libjq.so. ++ # Apple's ld64 has no `-l:` syntax (and iOS has no jniLibs name collision), so ++ # there use plain `-ljq`/`-lonig`, which resolve to flet-libjq's static .a since ++ # no shared lib is shipped. ++ import sysconfig as _sysconfig ++ if "ios" in _sysconfig.get_platform(): ++ link_args_deps = ["-ljq", "-lonig"] ++ else: ++ link_args_deps = ["-l:libjq.a", "-l:libonig.a"] extra_objects = [] else: jq_build_ext = jq_with_deps_build_ext From af7ad8b137652b17ba18de71b8b4585e35c072b8 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 00:51:17 +0200 Subject: [PATCH 21/49] =?UTF-8?q?skills:=20jq=20iOS=20-l:NAME.a=20gotcha?= =?UTF-8?q?=20(GNU=20ld=20only;=20Apple=20ld64=20needs=20-lX)=20=E2=80=94?= =?UTF-8?q?=20gate=20static-link=20per=20platform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../references/failure-catalogue.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 91663727..33354170 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1205,7 +1205,18 @@ keep its `.a` (drop the `.so`); in the consumer switch `requirements.host` → **`host_build`** (build-time only, not a runtime dep) and patch the link `-l` → `-l:lib.a` (static). The extension's DT_NEEDED then lists no `lib.so`. Verified: jq (static-links flet-libjq's libjq.a/libonig.a; flet-libjq -build 11, jq build 2). +build 11, jq build 3). + +**iOS caveat — `-l:NAME.a` is a GNU ld extension; Apple's ld64 does NOT support it** +→ the iOS build dies `ld: library ':libjq.a' not found` on every Python. The jniLibs +name collision that motivates `-l:` is **Android-only**, and since the `flet-lib*` +ships *only* static `.a` (no `.so`/`.dylib`), a plain `-l` resolves to the +static archive just the same on iOS. So gate the flag by target platform in the +consumer's `setup.py`: `-l:libX.a` on Android, `-lX` on iOS. Detect with +`"ios" in sysconfig.get_platform()` (the crossenv reports the *target* platform, e.g. +`ios-13.0-arm64-iphonesimulator`). Verify the iOS extension is self-contained with +`otool -L` (no `libX` dylib dep) + `nm` (the `libX` symbols are `T`, defined). Any +recipe that borrows this static-link pattern must apply the same per-platform gate. --- From 7aa573b57013b7129efe9ba5dfdceedb64f6ee82 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 01:07:00 +0200 Subject: [PATCH 22/49] ci: temporarily disable concurrency cancel-in-progress (parallel targeted dispatches) --- .github/workflows/build-wheels.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 620fca3c..4f26898c 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -72,9 +72,11 @@ on: required: false # Cancel in-flight runs when a newer event arrives for the same logical branch. -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} - cancel-in-progress: true +# TEMPORARILY DISABLED: we fire several targeted workflow_dispatch runs on the same +# branch in parallel (one per prebuild set) and must not let them cancel each other. +# concurrency: +# group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} +# cancel-in-progress: true env: DEFAULT_PYTHONS: "3.12,3.13,3.14" From 06f4f718a63bd6bff26d85bf5c5c3a9f5681f64b Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 01:54:57 +0200 Subject: [PATCH 23/49] forge: convert iOS MH_BUNDLE extensions to MH_DYLIB in fix_wheel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serious_python's darwin packaging turns every site-packages native .so into a .framework binary that SwiftPM LINKS (a Package.swift binaryTarget the plugin depends on). ld only links MH_OBJECT/MH_DYLIB, so any extension that is an MH_BUNDLE aborts the iOS-sim app build with "Unsupported mach-o filetype (only MH_OBJECT and MH_DYLIB can be linked) in .framework/". iOS CPython's sysconfig sets LDSHARED='...-dynamiclib -F . -framework Python', which setuptools/Cython/meson/maturin honor (-> MH_DYLIB), but CMake/scikit-build recipes link a Python MODULE with Apple's default -bundle (-> MH_BUNDLE), ignoring LDSHARED. So the failing set is exactly the CMake ones: opencv (cv2), ncnn, coolprop, faiss (_swigfaiss); the setuptools/Cython/meson ones already ship MH_DYLIB and pass. (llama-cpp-python is a DIFFERENT bug — ctypes interdependent SHARED dylibs, sp #223 — not MH_BUNDLE; this is a no-op there.) fix_wheel's new iOS branch mirrors the Android block: for each MH_BUNDLE .so it injects an LC_ID_DYLIB into the header's free padding and flips the filetype to MH_DYLIB, then ad-hoc re-signs (the edit invalidates the linker signature). dlopen (the import path) works on both filetypes, so nothing regresses; serious_python's own `install_name_tool -id` overwrites the placeholder id. Validated: converting cv2.abi3.so + ncnn.so makes `ld` go from the exact error to a clean link (exit 0); a real forge build of ncnn (iphonesimulator arm64) runs the new branch (MH_BUNDLE -> MH_DYLIB, codesign ok) and the recipe-tester iOS-sim app build gets past the cv2.cv2.framework link (Packaged Python app OK). --- src/forge/build.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/forge/build.py b/src/forge/build.py index a3153595..86e28a74 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -793,6 +793,71 @@ def _check_elf_alignment(self, so_path: Path): ) log(self.log_file, f"[{self.cross_venv}] {so_path.name}: 16KB alignment OK") + def _bundle_to_dylib(self, so_path: Path) -> bool: + """Convert a thin ``MH_BUNDLE`` Mach-O extension to ``MH_DYLIB`` in place. + + Injects an ``LC_ID_DYLIB`` load command into the header's free padding and + flips the filetype byte. Returns True if a conversion was made, False if + the file is not a thin little-endian ``MH_BUNDLE`` (already a dylib, a fat + binary, or not Mach-O — all left untouched). The caller must re-sign + (``codesign --force --sign -``) afterwards, since the edit invalidates the + linker's ad-hoc signature. + """ + MH_MAGIC_64 = 0xFEEDFACF + MH_BUNDLE = 0x8 + MH_DYLIB = 0x6 + LC_ID_DYLIB = 0xD + LC_SEGMENT_64 = 0x19 + + data = bytearray(so_path.read_bytes()) + if len(data) < 32 or struct.unpack_from(" leave alone) + (_magic, _cpu, _sub, filetype, ncmds, sizeofcmds, flags, _res) = ( + struct.unpack_from(" MH_DYLIB in place." + ) + + # dylib_command: cmd, cmdsize, name.offset(24), timestamp, cur_ver, compat_ver + idcmd = ( + struct.pack(" MH_DYLIB); but CMake / + # scikit-build recipes (opencv, ncnn, coolprop, faiss, ...) link a + # Python MODULE with Apple's default -bundle (-> MH_BUNDLE), ignoring + # LDSHARED. serious_python turns every site-packages .so into a + # `.framework` binary that SwiftPM LINKS (a Package.swift binaryTarget + # the plugin target depends on), and `ld` rejects a bundle with + # "Unsupported mach-o filetype (only MH_OBJECT and MH_DYLIB can be + # linked)". Injecting an LC_ID_DYLIB and flipping the header filetype + # makes the extension a linkable dylib; dlopen (the import path) works + # on both filetypes, so nothing downstream regresses. serious_python's + # own `install_name_tool -id` then overwrites the placeholder id. + for so in wheel_dir.glob("**/*.so"): + if self._bundle_to_dylib(so): + log( + self.log_file, + f"[{self.cross_venv}] MH_BUNDLE -> MH_DYLIB: {so.name}", + ) + # The header edit invalidates the linker's ad-hoc signature; + # re-sign ad-hoc so dyld/codesign accept the dylib. + self.cross_venv.run( + self.log_file, + ["codesign", "--force", "--sign", "-", str(so)], + ) + # Normalize a dotted distribution name (e.g. "zope.interface" -> "zope-interface") # to its PEP 503 canonical form in the wheel METADATA. pypi.flet.dev (Gemfury) # returns 409 "version already exists" for every wheel after the first when the From b805269592358b303c1f49c85d98d9d2d4d44447 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 01:58:41 +0200 Subject: [PATCH 24/49] skills: iOS MH_BUNDLE->MH_DYLIB fix (CMake extensions ignore LDSHARED=-dynamiclib; forge fix_wheel converts) --- .claude/skills/forge-error-catalogue/SKILL.md | 5 ++- .../references/failure-catalogue.md | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 63ec2133..7a772f84 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -71,7 +71,10 @@ instead of re-deriving it. `install_data rename:` / `-include` sanity-check breakage, **build.sh `cmake: command not found` → `requirements.build`**, **duplicated patch hunks from concatenated diffs**, iOS `flatc` MACOSX_BUNDLE, **`.dylib` staged under - the `.so` name**, **Apple `MacTypes.h` `Ptr` vs `cv::Ptr` ambiguity** (iOS-only; + the `.so` name**, **iOS `Unsupported mach-o filetype (only MH_OBJECT and + MH_DYLIB can be linked)` → forge `fix_wheel` converts a CMake extension's + `MH_BUNDLE` `.so` to `MH_DYLIB`** (inject `LC_ID_DYLIB` + flip filetype + ad-hoc + re-sign; setuptools/Cython/meson already ship dylib), **Apple `MacTypes.h` `Ptr` vs `cv::Ptr` ambiguity** (iOS-only; hand-written + gen2.py-generated code), **opencv-5 KleidiCV `armv8-a` on x86_64** and **hardcoded `CMAKE_SYSTEM_PROCESSOR` → ARM asm on the x86_64 sim** (per-arch fix), **Rust crate with no `target_os="ios"` backend** (`mac_address`, cfg-gate it). diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 33354170..be147b50 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -219,6 +219,41 @@ The reported arch must match the tag — `*-ios_13_0_x86_64_iphonesimulator.whl` --- +### iOS `flet build ios-simulator`: `Error (Xcode): Unsupported mach-o filetype (only MH_OBJECT and MH_DYLIB can be linked) in .framework/` (a CMake/scikit-build extension is an MH_BUNDLE) + +**Cause:** serious_python's darwin packaging wraps **every** site-packages native +`.so` into a `.framework` binary that SwiftPM **links** (a `Package.swift` +`binaryTarget` the plugin target depends on). `ld` links only `MH_OBJECT`/ +`MH_DYLIB`, so an extension that is an **`MH_BUNDLE`** aborts the app build. iOS +CPython's sysconfig sets `LDSHARED='...-dynamiclib -F . -framework Python'`, which +**setuptools / Cython / meson / maturin honor** (→ `MH_DYLIB`, pass); but +**CMake / scikit-build** link a Python `MODULE` with Apple's default `-bundle` +(→ `MH_BUNDLE`), *ignoring* `LDSHARED`. So the failing set is exactly the CMake +ones — **opencv (`cv2`), ncnn, coolprop, faiss (`_swigfaiss`)**, plus latent +sherpa_onnx/onnx. The flet-lib dependency is a **red herring** (cv2/ncnn link only +system `Accelerate`/`libc++`). Discriminate with `otool -hv ` (`BUNDLE` vs +`DYLIB`). (Note: **llama-cpp-python's** iOS failure is a *different* bug — ctypes +interdependent SHARED dylibs, sp #223 — its libs are already dylibs, so this is a +no-op there.) + +**Fix:** none per-recipe — **forge `fix_wheel` (build.py, iOS branch) converts each +`MH_BUNDLE` `.so` to `MH_DYLIB`** in place: inject an `LC_ID_DYLIB` load command +into the Mach-O header's free padding (bounded by the lowest `__TEXT` section file +offset — the `section_64.offset` field is at +48), flip the header `filetype` +`0x8`→`0x6` (and `ncmds`/`sizeofcmds`), then **ad-hoc re-sign** (`codesign --force +--sign -` — the header edit invalidates the linker signature). `dlopen` (the import +path) works on both filetypes so nothing regresses; serious_python's own +`install_name_tool -id` overwrites the placeholder `@rpath/` id. Generic +— rebuild and it catches every CMake extension with zero per-recipe CMake knowledge. +Per-recipe fallback (what onnxruntime/tflite/lightgbm did): force `-dynamiclib` ++ `-undefined dynamic_lookup` in the CMake link flags. **Validate** by `otool -hv` += `DYLIB` + `otool -l | grep LC_ID_DYLIB` on the built wheel's `.so`, and that +`ld ... ` links (exit 0) where the original bundle gave the exact error above. +Verified: ncnn (real forge build → `MH_BUNDLE -> MH_DYLIB`, codesign ok) + cv2 +(link test + iOS-sim app build past the `cv2.cv2.framework` link). + +--- + ### iOS: a flet-lib*'s native lib is silently absent from the app bundle **Symptom:** `flet build ios-simulator` succeeds, but at runtime the consumer From d01a26903711482473a658c0ffeecd9a4ea273bf Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 02:16:18 +0200 Subject: [PATCH 25/49] skills(forge-ci): transitive patched-dep resolves to unpatched published copy -> runtime error (opaque->pysodium); prebuild the dep recipe --- .claude/skills/forge-ci/SKILL.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.claude/skills/forge-ci/SKILL.md b/.claude/skills/forge-ci/SKILL.md index 25228c1f..e9038180 100644 --- a/.claude/skills/forge-ci/SKILL.md +++ b/.claude/skills/forge-ci/SKILL.md @@ -112,6 +112,19 @@ Both of these situations are chains — the second is easy to miss: (onnx's Requires-Dist includes ml_dtypes; ml-dtypes runs as both a package and a prebuild — that's fine and cheap). + **Silent variant — the dep resolves to an UNPATCHED published copy.** If B is + a *pure-Python* recipe that ALSO exists on pypi.flet.dev/PyPI, A's job won't + error at resolution — it silently pulls the published B (which lacks B's + not-yet-published 0.86 fix) and then fails at **runtime on device** with B's + own loader error, not `No matching distribution`. Seen with opaque → + `pysodium`: opaque's app pulled upstream pysodium, whose `__init__` raised + `ValueError: Unable to find libsodium` (no bare-soname fallback), even though + `flet-libsodium` was prebuilt. Fix: prebuild the **patched dep recipe** too so + its `-9999` wheel wins — `packages="opaque:" + prebuild_recipes="flet-libopaque,flet-libsodium,pysodium"`. Tell: a device + traceback whose failing frame is in a *dependency* package, not the one under + test → that dep is resolving to its unpatched published build. + If a chain dep lives on a *different* branch, merge that branch in first (`git merge machine/`) so the recipe dir exists for the prebuild. From d917c3f1aaba491f1b3aeba3893dc763b253da61 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 03:28:48 +0200 Subject: [PATCH 26/49] =?UTF-8?q?recipe:=20opencv-python=20=E2=80=94=20ext?= =?UTF-8?q?end=20cv2=20loader=20to=20iOS=20framework-ized=20native=20(Flet?= =?UTF-8?q?=200.86)=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Flet 0.86 cv2 loader fast-path only handled Android (find_spec('cv2.cv2') origin is a real .so from the soref finder). On iOS the native is framework-ized: serious_python leaves a cv2/cv2.fwork marker whose text is the app-relative path to the framework binary, so find_spec's origin ends in .fwork, the old fast-path fell through, and the stock loader recursed ("recursion is detected during loading of cv2 binary extensions"). Unify the fast-path: when the resolved origin ends in .fwork, read it and join with dirname(sys.executable) to get the framework binary (exactly as CPython's AppleFrameworkLoader.create_module does), then load it under the required top-level name "cv2" via ExtensionFileLoader. Android (.so origin) is unchanged. Verified: Android arm64 4/4 (unchanged path); iOS bundle->dylib link + framework packaging validated locally, loader .fwork resolution mirrors AppleFrameworkLoader — CI-confirmed next to the coolprop iOS pass. Build 2 -> 3. --- recipes/opencv-python/meta.yaml | 2 +- recipes/opencv-python/patches/mobile.patch | 48 +++++++++++++++------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/recipes/opencv-python/meta.yaml b/recipes/opencv-python/meta.yaml index 8e5fff96..8d91fe43 100644 --- a/recipes/opencv-python/meta.yaml +++ b/recipes/opencv-python/meta.yaml @@ -9,7 +9,7 @@ extract_packages: - cv2 build: - number: 2 + number: 3 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/opencv-python/patches/mobile.patch b/recipes/opencv-python/patches/mobile.patch index 32a1faae..1b140fd7 100644 --- a/recipes/opencv-python/patches/mobile.patch +++ b/recipes/opencv-python/patches/mobile.patch @@ -31,15 +31,24 @@ Make opencv-python 5.0 cross-compile AND load on Flet's mobile targets. to jniLibs (importable only as the submodule `cv2.cv2` via a `cv2/cv2.soref` marker, never as top-level `cv2`). Loading it as `cv2.cv2` makes OpenCV abort with "Submodule name should always start with a parent module name. Parent - name: cv2.cv2. Submodule name: cv2". The fix inserts a fast-path right after + name: cv2.cv2. Submodule name: cv2". iOS breaks the same way but relocates the + native into a `.framework` instead: serious_python leaves a `cv2/cv2.fwork` + marker whose text is the app-relative path to the framework binary, so the + stock loader recurses ("recursion is detected during loading of cv2 binary + extensions") on the pop-and-reimport. The fix inserts a fast-path right after the recursion guard: resolve the relocated extension via `find_spec('cv2.cv2')` - (the soref finder yields its real jniLibs path), then load it through a fresh + (Android: the soref finder yields the real jniLibs `.so`; iOS: the FileFinder + yields the `.fwork` marker path -> read it and join with + `dirname(sys.executable)` to get the framework binary, exactly as CPython's + AppleFrameworkLoader does), then load it through a fresh `ExtensionFileLoader("cv2", origin)` so its runtime `__name__` is the required top-level "cv2", relink its symbols onto the package, and load the extra pure- python submodules. On desktop / non-0.86 builds `find_spec('cv2.cv2')` returns None and the stock loader runs unchanged. Pairs with `extract_packages: [cv2]` in meta.yaml so the package ships as a real dir (the extra-submodule scan does - `os.listdir` on it). Verified on-device (arm64) 4/4. + `os.listdir` on it). Verified on-device: Android arm64 4/4. iOS bundle->dylib + link + framework packaging validated locally; loader `.fwork` resolution + mirrors AppleFrameworkLoader (CI-confirmed alongside the coolprop iOS pass). diff --git a/opencv/CMakeLists.txt b/opencv/CMakeLists.txt index d6812aa..7555914 100644 @@ -497,26 +506,37 @@ index a4bc8e5..b86efd8 100644 diff --git a/opencv/modules/python/package/cv2/__init__.py b/opencv/modules/python/package/cv2/__init__.py --- a/opencv/modules/python/package/cv2/__init__.py +++ b/opencv/modules/python/package/cv2/__init__.py -@@ -75,6 +75,41 @@ def bootstrap(): +@@ -75,6 +75,52 @@ def bootstrap(): print(sys.path) raise ImportError('ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.') sys.OpenCV_LOADER = True -+ # --- mobile-forge fast-path ------------------------- -+ # Under Flet 0.86 the native cv2 extension is relocated out of the wheel into -+ # jniLibs (resolved via a cv2/cv2.soref marker) and config.py is compiled to -+ # config.pyc, so OpenCV's stock loader -- which exec()s config.py to find a -+ # directory of .so files and re-imports it as the top-level module "cv2" -- -+ # cannot run. Load the native module directly under the top-level name "cv2" -+ # (OpenCV's compiled bindings hard-require that exact name) and relink. ++ # --- mobile-forge (Flet 0.86) fast-path ------------------------------- ++ # Under Flet 0.86 the native cv2 extension is relocated out of the wheel: ++ # Android -> jniLibs (a cv2/cv2.soref marker, origin is a real .so); iOS -> ++ # a framework (a cv2/cv2.fwork marker whose text is the app-relative path to ++ # the framework binary). config.py is also compiled to config.pyc, so the ++ # stock loader (which exec()s config.py and re-imports as top-level "cv2") ++ # recurses/fails. Resolve the relocated native and load it under the required ++ # top-level name "cv2" (OpenCV's compiled bindings hard-require that name). + try: + import importlib.util as _ilu + import importlib.machinery as _ilm + _sub_spec = _ilu.find_spec(__name__ + ".cv2") + except Exception: + _sub_spec = None -+ if _sub_spec is not None and getattr(_sub_spec, "origin", None): ++ _origin = getattr(_sub_spec, "origin", None) if _sub_spec is not None else None ++ if _origin and _origin.endswith(".fwork"): ++ # iOS: the .fwork holds the app-relative framework-binary path; resolve it ++ # exactly as CPython's AppleFrameworkLoader does. ++ try: ++ with open(_origin, "rb") as _fwf: ++ _rel = _fwf.read().decode("utf-8").strip() ++ _origin = os.path.join(os.path.dirname(sys.executable), _rel) ++ except Exception: ++ _origin = None ++ if _origin: + _dbg = hasattr(sys, "OpenCV_LOADER_DEBUG") -+ _loader = _ilm.ExtensionFileLoader("cv2", _sub_spec.origin) ++ _loader = _ilm.ExtensionFileLoader("cv2", _origin) + _spec = _ilu.spec_from_loader("cv2", _loader) + _native = _ilu.module_from_spec(_spec) + _loader.exec_module(_native) @@ -535,7 +555,7 @@ diff --git a/opencv/modules/python/package/cv2/__init__.py b/opencv/modules/pyth + except Exception: + pass + return -+ # --- end mobile-forge fast-path ----------------------------------------- ++ # --- end mobile-forge fast-path --------------------------------------- DEBUG = False if hasattr(sys, 'OpenCV_LOADER_DEBUG'): From 9a0667d2fe9893d2f7a3b911394f5007e8bce8c2 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 03:31:19 +0200 Subject: [PATCH 27/49] skills: opencv iOS cv2 loader (.fwork framework resolution) + iOS-sim local gotchas (per-arch staging ignores find-links; device_info_plus old-Xcode override) [skip ci] --- .../references/failure-catalogue.md | 17 +++++++++++++++++ .claude/skills/local-recipe-testing/SKILL.md | 10 +++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index be147b50..865ed0d2 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1137,6 +1137,23 @@ name** — reproduce the exact name via a fresh `ExtensionFileLoader(name, origi never via `import pkg.pkg`. Verified on-device (arm64) 4/4 against the byte-identical 4.10 loader; ported to the 5.0.0.93 recipe. +**iOS has the SAME break, different relocation → the fast-path must handle both.** +On iOS the native is framework-ized, not soref'd: serious_python leaves a +`cv2/cv2.fwork` marker whose text is the **app-relative path to the framework +binary** (e.g. `Frameworks/cv2.cv2.framework/cv2.cv2`). So `find_spec('cv2.cv2')` +returns an origin ending in **`.fwork`** (not `.so`) — an Android-only fast-path +that assumes a `.so` origin falls through and the stock loader recurses +(`ERROR: recursion is detected during loading of "cv2" binary extensions`). Resolve +it exactly as CPython's `AppleFrameworkLoader.create_module` does — read the +`.fwork` text and `os.path.join(os.path.dirname(sys.executable), )` — to +get the framework binary, then `ExtensionFileLoader("cv2", )`. +(This is separate from, and downstream of, the forge `fix_wheel` MH_BUNDLE→MH_DYLIB +conversion that makes cv2 *linkable* in the first place.) One local-repro gotcha: +serious_python's per-arch native staging pulls the iphoneos-slice wheel from the +index (NOT honoring `--find-links`/dist-test locally), so a hand-patched dist-test +wheel won't take on a local `flet build ios-simulator` — verify opencv iOS in CI +(where the freshly-built slice wheels are used), as coolprop iOS did. + --- ### `AttributeError: module 'apsw' has no attribute 'Connection'` / a package whose `__init__` IS the native extension imports empty (Android) — a serious_python `_SorefFinder` gap diff --git a/.claude/skills/local-recipe-testing/SKILL.md b/.claude/skills/local-recipe-testing/SKILL.md index 14e812be..76cccfe8 100644 --- a/.claude/skills/local-recipe-testing/SKILL.md +++ b/.claude/skills/local-recipe-testing/SKILL.md @@ -124,7 +124,15 @@ for i in $(seq 1 30); do grep EXIT "$DATA/Library/Caches/console.log" 2>/dev/nul 9. **Android `console.log` lives in the app's CACHE dir — `/data/data/com.flet.recipe_tester/cache/console.log` — NOT under `files/flet/app/`** (that's the app code; `python_site_packages` is a SIBLING under `files/flet/`). Polling the wrong dir looks like "the app never wrote a result" and cost ~10 min during the sherpa-onnx validation. Root is still required to read it (gotcha #4): `adb root` then `adb shell cat …`, or `adb shell su 0 cat …` on a google_apis image. -10. **`flet build ios-simulator` resolves the `iphoneos` (device) wheel AS WELL as both simulator ones.** It configures pip for `iphoneos.arm64` + `iphonesimulator.arm64` + `iphonesimulator.x86_64` and needs a wheel for EACH — a partial local matrix fails with `No matching distribution found`. Build all three iOS slices first (for the recipe AND every `flet-lib*` host dep). CI never hits this because it dumps all of `dist/*.whl` into its find-links dir. (`flet build apk` needs only the one `--arch` slice — the asymmetry is iOS-only.) Long-standing gotcha; re-hit during the onnxruntime iOS spike. +10. **`flet build ios-simulator` resolves the `iphoneos` (device) wheel AS WELL as both simulator ones.** It configures pip for `iphoneos.arm64` + `iphonesimulator.arm64` + `iphonesimulator.x86_64` and needs a wheel for EACH — a partial local matrix fails with `No matching distribution found`. Build all three iOS slices first (for the recipe AND every `flet-lib*` host dep). CI never hits this because it dumps all of `dist/*.whl` into its find-links dir. (`flet build apk` needs only the one `--arch` slice — the asymmetry is iOS-only.) Long-standing gotcha; re-hit during the onnxruntime iOS spike. **Worse: serious_python's PER-ARCH native staging (`build/site-packages//`) resolves those slice wheels from the INDEX directly and does NOT honor `PIP_FIND_LINKS`/dist-test locally** — so a hand-patched `-9999` wheel in your find-links dir is used for the initial pip install but the staged PYTHON code (e.g. `cv2/__init__.py`, whichever slice it picks — often `iphoneos.arm64`) still comes from the published wheel. Net: you cannot validate a *hand-patched loader* on a local `ios-simulator` build; use a real `forge` build of all slices, or verify in CI (where the freshly-built slice wheels ARE used — this is why coolprop iOS passed in CI but a hand-patched opencv wouldn't locally). + +10a. **Old local Xcode can't compile newer Flutter plugins** — e.g. Xcode 16.4 dies on `device_info_plus` 12.4.0 with `ARC Semantic Issue: No visible @interface for 'NSProcessInfo' declares the selector 'isiOSAppOnVision'` (a visionOS selector added in a newer SDK). This is a LOCAL toolchain gap, not your recipe (CI's Xcode 26.5 is fine). Pin the offending plugins older in the generated app pyproject before `flet build ios-simulator`: + ```toml + [tool.flet.flutter.pubspec.dependency_overrides] + connectivity_plus = "6.1.5" + device_info_plus = "12.3.0" + ``` + (`stage_recipe.sh` regenerates the pyproject, so append this AFTER staging.) 11. **Two booted simulators make `simctl booted` ambiguous.** With more than one sim booted, `simctl install booted …` targets one device and your subsequent `get_app_container booted …` may query the OTHER — the app "isn't installed" / the container is empty despite a successful install. Use the explicit `$UDID` for every simctl call (as the loop above does); never rely on `booted` unless you've verified exactly one device is booted (`xcrun simctl list devices | grep -c Booted`). From 60ec983d9c59f08fc0d56da5f090061079ff9173 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 12:13:46 +0200 Subject: [PATCH 28/49] recipe: pyobjus strip py2 examples + pyarrow rebuild for iOS bundle->dylib [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pyobjus iOS: Flet 0.86 compiles every bundled .py -> .pyc, and pyobjus ships Python-2 example scripts (share/pyobjus-examples/*.py) whose `print` statements raise SyntaxError, failing the iOS app build. Drop the `examples` data tree from setup.py (demos, no runtime role). Build 1 -> 2. pyarrow iOS: its _json/_pyarrow_cpp_tests/lib extensions are CMake MODULE MH_BUNDLE binaries that Xcode refused to link ("Unsupported mach-o filetype") — the same class the forge fix_wheel MH_BUNDLE->MH_DYLIB conversion now handles. Bump build 1 -> 2 so the forge-fixed rebuild wins over any published wheel. --- recipes/pyarrow/meta.yaml | 5 ++- recipes/pyobjus/meta.yaml | 2 +- recipes/pyobjus/patches/mobile.patch | 50 ++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/recipes/pyarrow/meta.yaml b/recipes/pyarrow/meta.yaml index 61a3a9aa..241d646c 100644 --- a/recipes/pyarrow/meta.yaml +++ b/recipes/pyarrow/meta.yaml @@ -15,7 +15,10 @@ package: # x86_64 -> x86 and arm64/arm64-v8a -> aarch64 covers every shipped slice. # {% set arrow_cpu = 'x86' if 'x86' in arch else 'aarch64' %} build: - number: 1 + # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB + # conversion (pyarrow's _json/_pyarrow_cpp_tests/lib extensions are CMake + # MODULE bundles that Xcode refused to link into the app's frameworks). + number: 2 script_env: # Belt-and-suspenders: hard-lock every OPTIONAL component OFF regardless of # what the flet-libarrow Arrow C++ build enables. pyarrow's define_option macro diff --git a/recipes/pyobjus/meta.yaml b/recipes/pyobjus/meta.yaml index fee726b6..0075173d 100644 --- a/recipes/pyobjus/meta.yaml +++ b/recipes/pyobjus/meta.yaml @@ -4,7 +4,7 @@ package: platforms: [ios] build: - number: 1 + number: 2 patches: - mobile.patch diff --git a/recipes/pyobjus/patches/mobile.patch b/recipes/pyobjus/patches/mobile.patch index 9eaa7a61..0bfe736f 100644 --- a/recipes/pyobjus/patches/mobile.patch +++ b/recipes/pyobjus/patches/mobile.patch @@ -1,4 +1,24 @@ +Make pyobjus 1.2.4 build and package on Flet's iOS target. + +1. libffi header path: the vendored sources include , but the + flet-libffi host dep installs the header as ; rewrite the include in + _runtime.h and common.pxi. + +2. Python-3 fixes in the Cython sources: drop the removed py2 `unicode` builtin + (isinstance(arg, (str, unicode)) -> isinstance(arg, str)). + +3. iOS build config: pyobjus's setup.py hard-requires a pre-built + `ios-deps-install/` tree (from its own .ci script) for libffi; mobile-forge + supplies libffi via the recipe's `libffi` host dep instead, so drop that block. + +4. Drop the bundled `examples` data tree. pyobjus ships Python-2 example scripts + under share/pyobjus-examples/ (`print` statements); Flet 0.86 compiles every + bundled .py to .pyc and those py2 files raise `SyntaxError: Missing parentheses + in call to 'print'`, failing the iOS app build. The examples are demos with no + runtime role, so they are removed from setup.py's data_files. + diff --git a/pyobjus/_runtime.h b/pyobjus/_runtime.h +index c31b7ba..6d0cf85 100644 --- a/pyobjus/_runtime.h +++ b/pyobjus/_runtime.h @@ -1,6 +1,6 @@ @@ -10,21 +30,23 @@ diff --git a/pyobjus/_runtime.h b/pyobjus/_runtime.h #include #include diff --git a/pyobjus/common.pxi b/pyobjus/common.pxi +index 61a835d..42eaf81 100644 --- a/pyobjus/common.pxi +++ b/pyobjus/common.pxi -@@ -110,7 +110,7 @@ +@@ -110,7 +110,7 @@ cdef extern from "objc/runtime.h": objc_method_description* protocol_copyMethodDescriptionList(Protocol *p, BOOL isRequiredMethod, BOOL isInstanceMethod, unsigned int *outCount) - - + + -cdef extern from "ffi/ffi.h": +cdef extern from "ffi.h": ctypedef unsigned long ffi_arg ctypedef signed long ffi_sarg ctypedef enum: FFI_TYPE_STRUCT diff --git a/pyobjus/pyobjus_conversions.pxi b/pyobjus/pyobjus_conversions.pxi +index 303038a..b3f33a3 100644 --- a/pyobjus/pyobjus_conversions.pxi +++ b/pyobjus/pyobjus_conversions.pxi -@@ -428,7 +428,7 @@ +@@ -428,7 +428,7 @@ cpdef object convert_py_to_nsobject(arg): return arg elif arg in (True, False): return autoclass('NSNumber').alloc().initWithBool_(int(arg)) @@ -34,11 +56,12 @@ diff --git a/pyobjus/pyobjus_conversions.pxi b/pyobjus/pyobjus_conversions.pxi elif isinstance(arg, int): return autoclass('NSNumber').alloc().initWithLong_(arg) diff --git a/setup.py b/setup.py +index f0b22da..a34d428 100644 --- a/setup.py +++ b/setup.py -@@ -43,23 +43,10 @@ +@@ -43,23 +43,10 @@ extra_link_args = [] include_dirs = [] - + if sys.platform == "ios": - ios_ver = platform.ios_ver() - ios_deps_path = join(dirname(__file__), "ios-deps-install") @@ -62,5 +85,18 @@ diff --git a/setup.py b/setup.py + # the build-time ios-deps-install/ check pyobjus 1.2.4 added; let + # forge's env do the libffi wiring. libraries.append('objc') - + depends = [join('pyobjus', x) for x in ( +@@ -101,8 +88,11 @@ setup( + ext_package='pyobjus', + data_files=[ + item ++ # mobile-forge: DROP the `examples` tree. pyobjus ships Python-2 example ++ # scripts (share/pyobjus-examples/*.py with `print` statements); Flet 0.86 ++ # compiles every bundled .py -> .pyc and those py2 files raise SyntaxError, ++ # failing the iOS app build. The examples are demos, not runtime code. + for data in [ +- list(tree('examples').items()), + list(tree('objc_classes', tree_name='objc_classes/').items()) + ] + for item in data From cb2ad99654d985de26bc1968052f9b0f075cc2e7 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 12:25:16 +0200 Subject: [PATCH 29/49] skills: iOS _posixshmem ModuleNotFoundError (multiprocessing) = python-build gap; re-enable in build_ios.py [skip ci] --- .../references/failure-catalogue.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 865ed0d2..3246fbee 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1404,6 +1404,30 @@ scikit-image `mobile.patch`. --- +### `ModuleNotFoundError: No module named '_posixshmem'` (iOS, at import of a multiprocessing user) + +**Cause:** a python-build gap, not a recipe bug. iOS CPython (flet's python-build) +re-enables `_multiprocessing` (`build_ios.py` flips `py_cv_module__multiprocessing=n/a` +→ `yes` — SemLock/sockets build fine on Darwin) but left **`_posixshmem` n/a**. +`multiprocessing/resource_tracker.py` does an **unconditional `import _posixshmem` +on posix**, so with `_multiprocessing` present but `_posixshmem` absent, *any* +transitive `import multiprocessing` takes the consumer down on iOS. scikit-learn: +`sklearn.utils.validation` → joblib → multiprocessing → `resource_tracker` → +`ModuleNotFoundError: _posixshmem`. Affects **any** package that reaches +multiprocessing (not sklearn-specific). + +**Fix (in python-build, not the recipe):** flip `py_cv_module__posixshmem=n/a` → +`yes` in `darwin/build_ios.py`, right beside the existing `_multiprocessing` flip. +`shm_open`/`shm_unlink` build fine on Darwin/iOS (the sandbox restricts *use*, not +the build; nothing here uses shared memory), and upstream 3.13's official iOS +support ships `_posixshmem`. Leave `_posixsubprocess` n/a (genuinely needs +fork/exec). **Needs an iOS python-build rebuild + a new support run-id for CI to +pick it up.** Tell: any iOS import traceback whose deepest frame is +`multiprocessing/resource_tracker.py` importing a missing `_posix*` → it's this +python-build gap, fix it there, don't patch the consuming recipe. + +--- + ### `ModuleNotFoundError` on device for a dep the package never declared (hidden dependency) **Cause:** upstream's `Requires-Dist` is incomplete for the code path mobile From fb0ec2f582b1d717e10e4de0338677ff8a522296 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 12:31:52 +0200 Subject: [PATCH 30/49] =?UTF-8?q?recipe:=20scikit-learn=20=E2=80=94=20drop?= =?UTF-8?q?=20obsolete=20ios-guard-multiprocessing-import=20patch=20[skip?= =?UTF-8?q?=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The patch guarded `from multiprocessing.connection import ...` in sklearn/callback/_transport.py on the (false) premise that iOS disables _multiprocessing. It's dead + ineffective: - python-build re-enables _multiprocessing on iOS (build_ios.py), so the guarded import succeeds and the except branch never fires; - the actual sklearn iOS failure is sklearn/utils/validation -> multiprocessing/resource_tracker -> `import _posixshmem` (a DIFFERENT module, imported earlier), which this patch never touched — the CI run failed with it applied. The real fix is re-enabling _posixshmem in python-build (build_ios.py). Build 1 -> 2. --- recipes/scikit-learn/meta.yaml | 3 +-- .../ios-guard-multiprocessing-import.patch | 26 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 recipes/scikit-learn/patches/ios-guard-multiprocessing-import.patch diff --git a/recipes/scikit-learn/meta.yaml b/recipes/scikit-learn/meta.yaml index 96cba40b..88e2c8ad 100644 --- a/recipes/scikit-learn/meta.yaml +++ b/recipes/scikit-learn/meta.yaml @@ -7,7 +7,7 @@ extract_packages: - sklearn build: - number: 1 + number: 2 backend-args: # scikit-learn borrows BLAS via scipy.linalg.cython_blas (no external BLAS # link, no Fortran); OpenMP is optional (required:false → single-threaded). @@ -33,4 +33,3 @@ requirements: patches: - relax-scipy-build-cap.patch - disable-openmp-non-android.patch - - ios-guard-multiprocessing-import.patch diff --git a/recipes/scikit-learn/patches/ios-guard-multiprocessing-import.patch b/recipes/scikit-learn/patches/ios-guard-multiprocessing-import.patch deleted file mode 100644 index bd7cae82..00000000 --- a/recipes/scikit-learn/patches/ios-guard-multiprocessing-import.patch +++ /dev/null @@ -1,26 +0,0 @@ -# scikit-learn 1.9's sklearn.callback._transport unconditionally imports -# multiprocessing.connection at module load (for cross-process progress bars), -# pulling in the _multiprocessing C-extension. CPython disables _multiprocessing -# (and _posixsubprocess) on iOS — the app sandbox forbids spawning child processes, -# so process-based parallelism is impossible there — so importing any sklearn -# submodule crashes on iOS with ModuleNotFoundError. The transport is only reached -# under multiprocessing parallelism (never on single-process mobile); guard the -# import so scikit-learn stays importable. No-op on Android (ships _multiprocessing). ---- a/sklearn/callback/_transport.py -+++ b/sklearn/callback/_transport.py -@@ -18,7 +18,14 @@ - - import os - import weakref --from multiprocessing.connection import Client, Connection, Listener -+try: -+ from multiprocessing.connection import Client, Connection, Listener -+except ImportError: -+ # flet's iOS Python ships no _multiprocessing C-extension. This cross-process -+ # progress-bar transport is then unusable, but it is only reached with -+ # multiprocessing parallelism (never on single-process mobile), so fall back -+ # to placeholders and keep scikit-learn importable. -+ Client = Connection = Listener = None - from threading import Thread - from typing import Callable, NamedTuple - From 20654f9f84981ed8610b711dcec7d7968f597e5f Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 12:45:20 +0200 Subject: [PATCH 31/49] =?UTF-8?q?recipe:=20llama-cpp-python=20=E2=80=94=20?= =?UTF-8?q?resolve=20iOS=20.fwork=20markers=20in=20the=20ctypes=20loader?= =?UTF-8?q?=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On iOS serious-python relocates each bundled dylib (libggml-base/libggml-cpu/ libggml/libllama) into a code-signed .framework and leaves a lib.fwork TEXT marker whose content is the app-relative framework-binary path. The loader was trying to `ctypes.CDLL("lib.fwork")` — i.e. dlopen the text marker — which fails, so the ggml chain never preloaded and libllama's sibling @rpath refs went unresolved (app hung/crashed at import; empty console.log, 900s timeout). Now the loader RESOLVES the .fwork marker to the real framework binary (as CPython's AppleFrameworkLoader does) before CDLL, and preloads the chain in dependency order with RTLD_GLOBAL. serious-python leaves the dylibs' install-ids unchanged (@rpath/lib.dylib), so each preloaded framework satisfies the next lib's sibling reference by install-id (the pyarrow shim pattern). Build 2 -> 3. NOTE: if the app still crashes at LAUNCH (before Python), the frameworks are being linked with the un-rewritten sibling @rpath refs — that's the serious-python #223 gap and needs an SP-side install-name rewrite, not a recipe fix. --- recipes/llama-cpp-python/meta.yaml | 2 +- recipes/llama-cpp-python/patches/mobile.patch | 185 +++++++++++------- 2 files changed, 120 insertions(+), 67 deletions(-) diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 28fee473..58ff6a46 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -3,7 +3,7 @@ package: version: "0.3.32" build: - number: 2 + number: 3 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/llama-cpp-python/patches/mobile.patch b/recipes/llama-cpp-python/patches/mobile.patch index 7f793469..3c44366b 100644 --- a/recipes/llama-cpp-python/patches/mobile.patch +++ b/recipes/llama-cpp-python/patches/mobile.patch @@ -8,8 +8,17 @@ It does five things: llama) so the wheel carries single unversioned files instead of a lib*.dylib -> .0 -> .0.15.3 symlink triplet that forge dereferences into three copies (and three colliding frameworks on iOS). -4. Teaches the ctypes loader to find the bundled lib under its iOS framework - name (lib.fwork) and on the "ios" platform. +4. Teaches the ctypes loader to load the bundled libs on iOS, where serious-python + relocates each dylib into a code-signed .framework and leaves a + lib.fwork TEXT marker (its content is the app-relative framework-binary + path). The loader now RESOLVES that marker to the real framework binary (as + CPython's AppleFrameworkLoader does — join the marker text above the wheel dir) + before ctypes.CDLL, instead of trying to dlopen the text marker itself; and it + preloads the ggml chain in dependency order (ggml-base -> ggml-cpu -> ggml -> + llama) with RTLD_GLOBAL. serious-python leaves the dylibs' install-ids + unchanged (@rpath/lib.dylib), so a preloaded framework satisfies the next + lib's sibling @rpath reference by install-id. Also handles sys.platform == "ios" + (flet's 3.13+). 5. Android/Flet 0.86: site-packages ships as a zip and serious-python relocates the bundled libs (libllama/libggml*) into the APK's jniLibs, so the loader's Path.exists() probes against the in-zip base_path all miss. Add a bare-soname @@ -49,7 +58,7 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py -@@ -25,30 +25,33 @@ +@@ -25,35 +25,54 @@ _EMSCRIPTEN_SIDE_MODULE_SUFFIX = ".cpython-00-wasm32-emscripten.so" # Load the library def load_shared_library(lib_base_name: str, base_path: pathlib.Path): """Platform independent shared library loader""" @@ -77,45 +86,68 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py - base_path / f"lib{lib_base_name}.dll", - ] - else: +- raise RuntimeError("Unsupported platform") ++ ++ def _resolve_fwork(marker: pathlib.Path): ++ # serious-python (iOS) relocates each bundled dylib into a code-signed ++ # .framework and leaves a lib.fwork TEXT marker whose content ++ # is the app-relative path to the framework binary. ctypes.CDLL needs the ++ # binary, not the marker; resolve it exactly as CPython's ++ # AppleFrameworkLoader does (join the marker text above the wheel dir). ++ try: ++ rel = marker.read_text().strip() ++ except OSError: ++ return None ++ d = marker.parent ++ for _ in range(16): ++ d = d.parent ++ fb = d / rel ++ if fb.exists(): ++ return fb ++ return None + + def _candidate_paths(name: str) -> List[pathlib.Path]: -+ # Ordered filename candidates for lib `name` inside base_path. ++ # Ordered EXISTING library paths for lib `name` under base_path, with any ++ # iOS .fwork marker resolved to its real framework binary. + if sys.platform == "emscripten": -+ # A CPython-style tag that Pyodide skips during package auto-load. -+ return [base_path / f"lib{name}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}"] -+ if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): -+ return [base_path / f"lib{name}.so"] -+ if sys.platform == "darwin": -+ # mobile-forge: on iOS, flet/serious-python repackages each bundled -+ # .dylib into a code-signed framework and renames the in-wheel file -+ # to lib.fwork (ctypes.CDLL can dlopen it directly). Try that -+ # first, then the plain desktop-macOS names. -+ return [ -+ base_path / f"lib{name}.fwork", -+ base_path / f"lib{name}.so", -+ base_path / f"lib{name}.dylib", -+ ] -+ if sys.platform == "ios": -+ # flet's iOS Python (3.13+) reports sys.platform == "ios". -+ return [ -+ base_path / f"lib{name}.fwork", -+ base_path / f"lib{name}.dylib", -+ base_path / f"lib{name}.so", -+ ] -+ if sys.platform == "win32": -+ return [base_path / f"{name}.dll", base_path / f"lib{name}.dll"] - raise RuntimeError("Unsupported platform") ++ names = [f"lib{name}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}"] ++ elif sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): ++ names = [f"lib{name}.so"] ++ elif sys.platform == "darwin" or sys.platform == "ios": ++ # iOS (sys.platform == "ios" on flet's 3.13+, else "darwin"): try the ++ # framework marker first, then the plain desktop-macOS names. ++ names = [f"lib{name}.fwork", f"lib{name}.dylib", f"lib{name}.so"] ++ elif sys.platform == "win32": ++ names = [f"{name}.dll", f"lib{name}.dll"] ++ else: ++ raise RuntimeError("Unsupported platform") ++ out: List[pathlib.Path] = [] ++ for n in names: ++ p = base_path / n ++ if n.endswith(".fwork"): ++ if p.exists(): ++ fb = _resolve_fwork(p) ++ if fb is not None: ++ out.append(fb) ++ else: ++ out.append(p) ++ return out cdll_args = dict() # type: ignore -@@ -67,9 +70,14 @@ + +- # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32": + os.add_dll_directory(str(base_path)) + os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"] +@@ -67,9 +86,14 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path): os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) cdll_args["winmode"] = ctypes.RTLD_GLOBAL + else: + # mobile-forge: load globally so each library's dependencies (preloaded -+ # just below) satisfy its DT_NEEDED / @rpath entries by soname. The -+ # bundled libs carry no RUNPATH, so on iOS/Android the platform linker -+ # will not find sibling libs on its own. ++ # just below) satisfy its @rpath / DT_NEEDED entries. The bundled libs ++ # carry no usable RUNPATH on iOS/Android; dyld / the Android linker match ++ # an already-loaded image by its install-id / soname instead. + cdll_args["mode"] = ctypes.RTLD_GLOBAL if sys.platform == "emscripten": @@ -123,7 +155,7 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py lib_dir = str(base_path) ld_library_path = os.environ.get("LD_LIBRARY_PATH", "") if lib_dir not in ld_library_path.split(os.pathsep): -@@ -79,30 +87,54 @@ +@@ -79,43 +103,56 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path): else f"{lib_dir}{os.pathsep}{ld_library_path}" ) @@ -135,10 +167,23 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py - dependency_path = ( - base_path / f"lib{dependency}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}" - ) -+ # Preload dependencies (in dependency order) so the main library resolves -+ # them from the already-loaded set: ggml-base before ggml-cpu / ggml, and -+ # all of them before llama / mtmd. Best-effort — the main load reports the -+ # real error if something is genuinely missing. +- if dependency_path.exists(): +- try: +- ctypes.CDLL(str(dependency_path), **cdll_args) # type: ignore +- except Exception as e: +- raise RuntimeError( +- f"Failed to load shared library '{dependency_path}': {e}" +- ) +- +- # Try to load the shared library, handling potential errors +- for lib_path in lib_paths: +- if lib_path.exists(): ++ # Preload dependencies in dependency order so the main library resolves them ++ # from the already-loaded set: ggml-base before ggml-cpu/ggml, all before ++ # llama/mtmd. On iOS the libs are framework-relocated with sibling @rpath refs ++ # that dyld cannot resolve on its own, but each preloaded framework's ++ # install-id (@rpath/lib.dylib, which serious-python leaves unchanged for ++ # dylibs) matches the ref, so RTLD_GLOBAL preloading satisfies them. + dependencies = { + "llama": ("ggml-base", "ggml-cpu", "ggml"), + "mtmd": ("ggml-base", "ggml-cpu", "ggml", "llama"), @@ -147,50 +192,58 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py + for dependency in dependencies.get(lib_base_name, ()): + loaded = False + for dependency_path in _candidate_paths(dependency): - if dependency_path.exists(): - try: - ctypes.CDLL(str(dependency_path), **cdll_args) # type: ignore -- except Exception as e: -- raise RuntimeError( -- f"Failed to load shared library '{dependency_path}': {e}" -- ) -+ except Exception: -+ pass + try: +- return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore +- except Exception as e: +- raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") ++ ctypes.CDLL(str(dependency_path), **cdll_args) # type: ignore + loaded = True + break ++ except Exception: ++ continue + if not loaded: + # Android: the bundled lib is not a file on disk (Flet 0.86 ships -+ # site-packages as a zip and serious-python relocated it into the -+ # APK's jniLibs); load it by bare soname so the platform linker -+ # resolves it from there. Best-effort, like the on-disk branch. -+ for dependency_path in _candidate_paths(dependency): ++ # site-packages as a zip and serious-python relocated it into the APK's ++ # jniLibs); load it by bare soname so the linker resolves it there. ++ for _n in (f"lib{dependency}.so", f"lib{dependency}.dylib"): + try: -+ ctypes.CDLL(dependency_path.name, **cdll_args) # type: ignore ++ ctypes.CDLL(_n, **cdll_args) # type: ignore + break + except OSError: + continue - - # Try to load the shared library, handling potential errors -- for lib_path in lib_paths: -+ for lib_path in _candidate_paths(lib_base_name): - if lib_path.exists(): - try: - return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore - except Exception as e: - raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") - -+ # Android: the library was relocated into the APK's jniLibs and is not a file -+ # on disk (Flet 0.86 ships site-packages as a zip); load it by bare soname so -+ # the platform linker resolves it from jniLibs. ++ ++ # Load the main library. + for lib_path in _candidate_paths(lib_base_name): + try: -+ return ctypes.CDLL(lib_path.name, **cdll_args) # type: ignore ++ return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore ++ except Exception as e: ++ raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") ++ ++ # Android: the library was relocated into the APK's jniLibs (not a file on ++ # disk); load it by bare soname. ++ for _n in (f"lib{lib_base_name}.so", f"lib{lib_base_name}.dylib"): ++ try: ++ return ctypes.CDLL(_n, **cdll_args) # type: ignore + except OSError: + continue -+ + raise FileNotFoundError( f"Shared library with base name '{lib_base_name}' not found" ) + +- +-# ctypes sane type hint helpers +-# +-# - Generic Pointer and Array types +-# - PointerOrRef type with a type hinted byref function +-# +-# NOTE: Only use these for static type checking not for runtime checks +-# no good will come of that +- + if TYPE_CHECKING: + CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore + + diff --git a/vendor/llama.cpp/ggml/src/CMakeLists.txt b/vendor/llama.cpp/ggml/src/CMakeLists.txt --- a/vendor/llama.cpp/ggml/src/CMakeLists.txt +++ b/vendor/llama.cpp/ggml/src/CMakeLists.txt From 845821d5da785cbbdb553eb668e984baccd8c2dd Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 14:03:23 +0200 Subject: [PATCH 32/49] =?UTF-8?q?skills(forge-error-catalogue):=20sp=20#22?= =?UTF-8?q?3=20iOS=20launch-crash=20entry=20(interdependent=20dylibs)=20?= =?UTF-8?q?=E2=80=94=20reconcile=20framework=20install-names;=20+=20local-?= =?UTF-8?q?test-unreleased-sp=20recipe=20(local=20flet-cli,=20--python-ver?= =?UTF-8?q?sion=203.12,=20SERIOUS=5FPYTHON=5FAPP)=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/forge-error-catalogue/SKILL.md | 7 ++- .../references/failure-catalogue.md | 52 +++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/SKILL.md b/.claude/skills/forge-error-catalogue/SKILL.md index 7a772f84..85e3c93f 100644 --- a/.claude/skills/forge-error-catalogue/SKILL.md +++ b/.claude/skills/forge-error-catalogue/SKILL.md @@ -101,7 +101,12 @@ instead of re-deriving it. `libssl.so.3`/`libcrypto`/`libsqlite` not found, import-name errors, old version loaded, **lazy_loader "non-existent stub" (serious_python strips `*.pyi`)**, **hidden runtime deps** (keras→scipy; device-emulating venv method), - **insightface `root=` PermissionError**. + **insightface `root=` PermissionError**, and **iOS app crashes at launch with a + 0-byte `console.log` → `dyld: Library not loaded: @rpath/lib.dylib` for a chain + of interdependent bundled dylibs (pyarrow, llama)** → **serious_python #223**: + reconcile framework install-ids + `@rpath` deps to the dotted-framework paths + (`reconcile_framework_install_names` in darwin scripts; needs an sp release for CI; + local sp fix cc28d13 verified pyarrow 4/4 on-sim). - **Recipe-tester app failures** — host-build (`pg_config` etc.), pypi.flet.dev index precedence, "no matching distribution" (incl. **ios-simulator also resolving the iphoneos wheel**), **sdist-only pure-python dep → pip backtrack** diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 3246fbee..cc304505 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -232,9 +232,10 @@ CPython's sysconfig sets `LDSHARED='...-dynamiclib -F . -framework Python'`, whi ones — **opencv (`cv2`), ncnn, coolprop, faiss (`_swigfaiss`)**, plus latent sherpa_onnx/onnx. The flet-lib dependency is a **red herring** (cv2/ncnn link only system `Accelerate`/`libc++`). Discriminate with `otool -hv ` (`BUNDLE` vs -`DYLIB`). (Note: **llama-cpp-python's** iOS failure is a *different* bug — ctypes -interdependent SHARED dylibs, sp #223 — its libs are already dylibs, so this is a -no-op there.) +`DYLIB`). (Note: **llama-cpp-python / pyarrow** iOS failure is a *different* bug — +interdependent SHARED dylibs, sp #223, which crashes at **launch** not build; see +the dedicated entry just below. Their libs are already dylibs, so this MH_BUNDLE +conversion is a no-op there.) **Fix:** none per-recipe — **forge `fix_wheel` (build.py, iOS branch) converts each `MH_BUNDLE` `.so` to `MH_DYLIB`** in place: inject an `LC_ID_DYLIB` load command @@ -254,6 +255,51 @@ Verified: ncnn (real forge build → `MH_BUNDLE -> MH_DYLIB`, codesign ok) + cv2 --- +### iOS: app builds fine but **crashes at launch**, 0-byte `console.log` — `dyld: Library not loaded: @rpath/lib.dylib, Referenced from: .debug.dylib` (interdependent bundled dylibs, serious_python #223) + +**Symptom:** the `.app` builds and installs, but on launch `console.log` is **0 +bytes** (Flet rebinds stdout→console.log only after Python starts; pytest prints +"test session starts" *before* importing the recipe — so 0 bytes = crash BEFORE +Python). `simctl launch --console-pty` shows `dyld[…]: Library not loaded: +@rpath/lib.dylib … Referenced from: …/recipe-tester.debug.dylib … tried +'…/Frameworks/lib.dylib' (no such file)`. Recipes with a **chain of +interdependent native libs**: pyarrow (`libarrow`←`libarrow_compute`← +`libarrow_python`, plus every `pyarrow.*` C-ext), llama-cpp-python +(`libggml-base`←`libggml-cpu`←`libggml`←`libllama`). + +**Cause:** serious_python framework-izes each site-package `.so`/`.dylib` into a +framework named by its **dotted relative path** (`opt/lib/libarrow.dylib` → +`opt.lib.libarrow.framework/opt.lib.libarrow`) and makes each a `Package.swift` +`binaryTarget` the plugin **links at launch**. But `create_xcframework_from_dylibs` +left every Mach-O **install-id** and interdependent **`@rpath` ref** at the +ORIGINAL bare name (`@rpath/libarrow.dylib`) — it ran `install_name_tool -id` only +for `ext=so`, never for `.dylib`, and never rewrote deps. So a bare +`@rpath/libarrow.dylib` resolves to `Frameworks/libarrow.dylib` (doesn't exist; the +file is inside `…framework/…`) → dyld can't link at launch. The recipe's ctypes +`.fwork` shim is **moot** here — it runs IN Python, after the crash. (Distinct from +the MH_BUNDLE build error above, and from the forge MH_BUNDLE→MH_DYLIB fix which is +necessary-but-not-sufficient.) + +**Fix (serious_python, not per-recipe):** `reconcile_framework_install_names()` in +`serious_python_darwin/darwin/xcframework_utils.sh`, called from +`sync_site_packages.sh` after the `for _sp_ext in so dylib` conversion loop — sets +every framework binary's own install-id to `@rpath/.framework/` and rewrites +every dep matching a sibling's old-id to that framework path, then ad-hoc re-signs; +excludes the python/stdlib xcframeworks. Local sp branch `fix/soref-package-init` +commit **cc28d13** (UNPUSHED). **CI stays red for these recipes' iOS jobs until sp +is released** (published sp lacks it — same release-gating as apsw's SP `_SorefFinder` +fix). **Verified locally:** pyarrow 24.0.0 iOS-sim now launches, `4 passed`, `EXIT 0` +(was this exact dyld crash). Two bash gotchas bit while implementing it — the +`otool -D | head -1` SIGPIPE race (buffer to a var first) and zsh-vs-bash array +indexing in the *test harness* (real sp runs under bash). To verify an unreleased sp +change locally without a release: pubspec `dependency_overrides` path-dep the local +sp packages, build with the **local flet-cli** (`uv run --project .../flet/sdk/python +flet build ios-simulator --python-version 3.12`) not `uvx` (else +`SERIOUS_PYTHON_APP … must be set`), stage the recipe's CI wheels as `-9999` via +`PIP_FIND_LINKS`. Full recipe in the [[serious-python-223-framework-relocation]] memory. + +--- + ### iOS: a flet-lib*'s native lib is silently absent from the app bundle **Symptom:** `flet build ios-simulator` succeeds, but at runtime the consumer From b408c8ae7789d1ac67b6a3b0640d7377f352b805 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 14:34:10 +0200 Subject: [PATCH 33/49] ci: use python-build 29242819654 (iOS _posixshmem) + serious_python fix branch in recipe-tester template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-wheels-version.yml: bump the python_build_run_id fallback 29193662574 -> 29242819654 (flet-dev/python-build run re-enabling _posixshmem for iOS multiprocessing/resource_tracker — unblocks scikit-learn iOS and any multiprocessing user). - recipe-tester/pyproject.toml.tpl: add a dependency_overrides block pulling serious_python from flet-dev/serious-python@fix/soref-package-init so recipe tests validate against the unreleased iOS fixes (#223 framework relocation + _SorefFinder) before they ship. TEMPORARY — remove once sp is released. Lets recipes be validated end-to-end with all pending fixes prior to PR. [skip ci] --- .github/workflows/build-wheels-version.yml | 2 +- tests/recipe-tester/pyproject.toml.tpl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index 672e469c..282d4f09 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -273,7 +273,7 @@ jobs: FORGE_PACKAGES: ${{ matrix.forge_packages }} PREBUILD_RECIPES: ${{ inputs.prebuild_recipes }} PLATFORM: ${{ matrix.platform }} - PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id != '' && inputs.python_build_run_id || '29193662574' }} + PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id != '' && inputs.python_build_run_id || '29242819654' }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PIP_FIND_LINKS: ${{ github.workspace }}/dist UV_PYTHON: ${{ inputs.python_version }} diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 2e233dd1..c64d7c0a 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -41,3 +41,17 @@ path = "." # (empty `[]` — the default — is a no-op). [tool.flet.android] extract_packages = [__EXTRACT_PACKAGES__] + +# TEMPORARY: pull serious_python from the branch carrying the unreleased iOS fixes +# so recipe tests validate against them BEFORE they ship in a published sp release: +# - #223 interdependent-dylib framework relocation (reconcile_framework_install_names) +# — unblocks pyarrow/llama iOS launch (dyld "Library not loaded @rpath/lib*.dylib") +# - _SorefFinder package-__init__ native resolution (apsw et al.) +# flet merges this into the generated Flutter pubspec's dependency_overrides +# (recursive merge, so the nested `git = { … }` table passes straight through). +# REMOVE this block once serious_python is released with these fixes. +[tool.flet.flutter.pubspec.dependency_overrides] +serious_python = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python" } } +serious_python_darwin = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_darwin" } } +serious_python_android = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_android" } } +serious_python_platform_interface = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_platform_interface" } } From bfef1c0c39896a09170e2e6fed44a1992ac4f410 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 14:51:14 +0200 Subject: [PATCH 34/49] skills(forge-ci): pre-PR validation against unreleased serious_python (git override in recipe-tester template) + python-build (run-id fallback bump); python_build_run_id + empty mobile_test_pythons dispatch rows [skip ci] --- .claude/skills/forge-ci/SKILL.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.claude/skills/forge-ci/SKILL.md b/.claude/skills/forge-ci/SKILL.md index e9038180..50a61ad6 100644 --- a/.claude/skills/forge-ci/SKILL.md +++ b/.claude/skills/forge-ci/SKILL.md @@ -204,8 +204,31 @@ the log. | `packages` | `"name:"` entries, comma-separated; `:` suffix means default version. `ALL` expands to every recipe | | `prebuild_recipes` | comma-separated, **ordered**, built per-job before packages | | `python_versions` | defaults to all three; narrow for a quick re-run (e.g. `3.12.13`) | -| `mobile_test_pythons` | default `3.12` — leave it; never `ALL` on this fork | +| `mobile_test_pythons` | default `3.12` — leave it; never `ALL` on this fork. Pass `""` to build wheels WITHOUT the on-device test (e.g. when the test can't pass yet because the fix lives in unreleased serious_python — you'll test locally) | | `archs` | default `android,iOS` | +| `python_build_run_id` | a `flet-dev/python-build` Actions run-id whose artifacts to use instead of the pinned release; empty → the hardcoded FALLBACK in `build-wheels-version.yml` (grep `PYTHON_BUILD_RUN_ID: ${{ … || '' }}`). Bump that fallback to ship an unreleased python-build fix to every job | + +## Testing recipes against UNRELEASED serious_python / python-build (pre-PR validation) + +When a recipe's green depends on an sp/python-build fix that isn't in a published +release yet (e.g. sp #223 iOS framework relocation, python-build iOS `_posixshmem`), +wire CI + local to the fix so recipes validate BEFORE the PR: +- **python-build fix** → bump the `PYTHON_BUILD_RUN_ID` fallback constant in + `build-wheels-version.yml` to the python-build CI run that has it (verify that run + is `success` and has the `python-darwin-` / `python-android-` artifacts). +- **serious_python fix** → add a `[tool.flet.flutter.pubspec.dependency_overrides]` + block to `tests/recipe-tester/pyproject.toml.tpl` with **git** deps for + serious_python + `_darwin` + `_android` + `_platform_interface`, each + `{ git = { url = "…/serious-python.git", ref = "", path = "src/" } }`. + flet merges [tool.flet.flutter.pubspec] via a recursive merge, so the nested git + table passes through to the Flutter pubspec; dart pub clones the branch and runs its + darwin scripts. Mark it TEMPORARY — remove once sp is released. (Local old-Xcode + builds also need `device_info_plus`/`connectivity_plus` pins, but keep those OUT of + the committed template — add post-staging; CI's Xcode doesn't need them.) +- The on-device iOS/APK test in CI still uses PUBLISHED sp, so it can't pass until the + release — dispatch wheel builds with `mobile_test_pythons=""` and verify the recipe + LOCALLY (git or path sp override + local flet-cli + `--python-version 3.12`; see the + local-recipe-testing skill). CI goes green only after the sp/python-build release. ## Watching a run without babysitting From c974366b2cb6c98048c085e6872af5ef13b97749 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 15:15:44 +0200 Subject: [PATCH 35/49] recipes: bump ncnn/coolprop/faiss-cpu build numbers for forge iOS MH_BUNDLE->MH_DYLIB reship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These 3 CMake recipes are fixed by forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB conversion (06f4f71, this branch), not by any recipe-file change. On push to main the detect job only builds recipes whose recipes/** paths changed, so without a bump they would neither rebuild (unchanged) nor reship (a same-tag rebuild is a 409 duplicate-skip on gemfury) — the fixed wheels would never reach users. Comment-only bump mirrors pyarrow's. ncnn 1->2, coolprop 10->11, faiss-cpu 1->2. [skip ci] --- recipes/coolprop/meta.yaml | 6 +++++- recipes/faiss-cpu/meta.yaml | 6 +++++- recipes/ncnn/meta.yaml | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/recipes/coolprop/meta.yaml b/recipes/coolprop/meta.yaml index 77eb8e37..95b98259 100644 --- a/recipes/coolprop/meta.yaml +++ b/recipes/coolprop/meta.yaml @@ -14,7 +14,11 @@ patches: - mkdir-cython-output.patch build: - number: 10 + # Rebuilt at build 11 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB + # conversion (CoolProp's CMake extension is an MH_BUNDLE Xcode refused to + # link into the app's frameworks). The forge fix lands with this branch, so a + # bump is required for the fixed wheel to rebuild + publish on merge. + number: 11 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/faiss-cpu/meta.yaml b/recipes/faiss-cpu/meta.yaml index 60b7a339..729dc3a0 100644 --- a/recipes/faiss-cpu/meta.yaml +++ b/recipes/faiss-cpu/meta.yaml @@ -8,7 +8,11 @@ source: url: https://github.com/facebookresearch/faiss/archive/refs/tags/v{{ version }}.tar.gz build: - number: 1 + # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB + # conversion (faiss's _swigfaiss CMake extension is an MH_BUNDLE Xcode + # refused to link into the app's frameworks). The forge fix lands with this + # branch, so a bump is required for the fixed wheel to rebuild + publish on merge. + number: 2 script_env: # scikit-build-core honors the CMAKE_ARGS env var (appended to its cmake call), # same as recipes/duckdb. faiss defines + BLAS + OpenMP are shared across diff --git a/recipes/ncnn/meta.yaml b/recipes/ncnn/meta.yaml index edbb667f..fe4a6beb 100644 --- a/recipes/ncnn/meta.yaml +++ b/recipes/ncnn/meta.yaml @@ -3,7 +3,11 @@ package: version: "1.0.20260526" build: - number: 1 + # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB + # conversion (ncnn's CMake extension is an MH_BUNDLE Xcode refused to link + # into the app's frameworks). The forge fix lands with this branch, so a + # bump is required for the fixed wheel to rebuild + publish on merge. + number: 2 script_env: _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' # setup.py appends EXTRA_CMAKE_ARGS after its own args, so our overrides From 8e5b7ca987f5d986fe3665e7fb5f95800a75f732 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 15:33:16 +0200 Subject: [PATCH 36/49] =?UTF-8?q?skills(forge-ci):=20PUSH-before-dispatch?= =?UTF-8?q?=20trap=20=E2=80=94=20--ref=20checks=20out=20the=20fork's=20rem?= =?UTF-8?q?ote=20HEAD,=20not=20local=20commits;=20verify=20run=20headSha?= =?UTF-8?q?=20=3D=3D=20local=20HEAD=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/forge-ci/SKILL.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.claude/skills/forge-ci/SKILL.md b/.claude/skills/forge-ci/SKILL.md index 50a61ad6..9f401aa5 100644 --- a/.claude/skills/forge-ci/SKILL.md +++ b/.claude/skills/forge-ci/SKILL.md @@ -84,6 +84,20 @@ runs on one branch (double dispatch, a late push, a killed script whose last cancels the rest — check `gh run list --branch ` and make sure the survivor has the inputs you meant. +**PUSH BEFORE YOU DISPATCH — the #1 silent-invalid-run trap.** `gh workflow run +--ref ` checks out the **fork's REMOTE branch HEAD**, never your local +commits. Committing locally and dispatching without pushing runs *stale code*, +and the run looks healthy — it just tests the old tree. This burns a full +build+test cycle and gives a wrong red/green. Symptom seen once: local template +had a serious_python git override but CI resolved `serious_python 4.3.0` from +pub.dev (hosted), so every android leg failed on the old empty-x86_64- +sitepackages bug. ALWAYS: `git push fork ` (commits carry `[skip ci]` so +the push won't auto-run), then **verify the dispatched run's commit**: +`gh run view --json headSha` must equal your local `git rev-parse HEAD` +before you trust ANY result. The fork remote is usually `fork` +(ndonkoHenri/mobile-forge); `origin` is the upstream flet-dev/mobile-forge — +dispatch against the fork. + ## Chains: when and how to use `prebuild_recipes` `prebuild_recipes` is a comma-separated list of recipes each job builds From 59357cdbeeb5bb12cddd5f4536adf853281736ab Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 15:47:20 +0200 Subject: [PATCH 37/49] recipes: normalize jq/llama-cpp-python/opencv-python to a single build bump (3->2) Each was bumped twice within this branch (once for the Android fix, once for iOS), but nothing publishes until merge and the build number only needs to end up > the last-published (main = build 1). Collapse to a single 1->2 bump. Harmless either way; just tidier. [skip ci] --- recipes/jq/meta.yaml | 2 +- recipes/llama-cpp-python/meta.yaml | 2 +- recipes/opencv-python/meta.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes/jq/meta.yaml b/recipes/jq/meta.yaml index 01fc5d93..5514ef1d 100644 --- a/recipes/jq/meta.yaml +++ b/recipes/jq/meta.yaml @@ -11,7 +11,7 @@ requirements: - flet-libjq 1.7.1 build: - number: 3 + number: 2 script_env: JQPY_USE_SYSTEM_LIBS: 1 diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 58ff6a46..28fee473 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -3,7 +3,7 @@ package: version: "0.3.32" build: - number: 3 + number: 2 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/opencv-python/meta.yaml b/recipes/opencv-python/meta.yaml index 8d91fe43..8e5fff96 100644 --- a/recipes/opencv-python/meta.yaml +++ b/recipes/opencv-python/meta.yaml @@ -9,7 +9,7 @@ extract_packages: - cv2 build: - number: 3 + number: 2 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- From 17a76c0deda75b2651877e7c6fa3c8d52da72949 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 16:09:42 +0200 Subject: [PATCH 38/49] ci: build recipe-tester APK for x86_64 only + parameterize serious_python override 1. Android app build: pass `--arch x86_64` to `flet build apk` (the on-device test runs on an x86_64 emulator). Fixes recipes with excluded_arches (pyarrow drops armeabi-v7a) whose all-ABI app build failed pip resolution ('No matching distribution'), and cuts build time. 2. New `serious_python_ref` workflow input (build-wheels.yml + build-wheels-version.yml, default 'fix/soref-package-init'). stage_recipe.sh now appends the serious_python git dependency_overrides from $SERIOUS_PYTHON_REF instead of the template hardcoding them; empty ref -> published serious_python. The pyproject template no longer hardcodes the branch. Pass -f serious_python_ref='' to test against released sp. [skip ci] --- .github/workflows/build-wheels-version.yml | 17 ++++++++++++++++- .github/workflows/build-wheels.yml | 15 +++++++++++++++ tests/recipe-tester/pyproject.toml.tpl | 17 ++++------------- tests/recipe-tester/stage_recipe.sh | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index 282d4f09..3c58f099 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -48,6 +48,14 @@ on: required: false type: string default: "" + serious_python_ref: + description: | + Git ref of flet-dev/serious-python to pin the recipe-tester's + serious_python to for on-device tests (unreleased sp fix validation). + Empty = published serious_python: + required: false + type: string + default: "fix/soref-package-init" secrets: GEMFURY_TOKEN: required: false @@ -455,6 +463,7 @@ jobs: PKG_NAME: ${{ steps.detect-tests.outputs.pkg_name }} PKG_VERSION: ${{ steps.detect-tests.outputs.pkg_version }} INPUT_PYTHON_VERSION: ${{ inputs.python_version }} + SERIOUS_PYTHON_REF: ${{ inputs.serious_python_ref }} run: | set -euxo pipefail @@ -501,8 +510,13 @@ jobs: fi cd tests/recipe-tester + # The Android on-device test runs on an x86_64 emulator, so build the + # APK for x86_64 ONLY: it's the sole ABI the test needs, it cuts build + # time, and it sidesteps recipes with excluded_arches (e.g. pyarrow + # drops armeabi-v7a) whose all-ABI app build would otherwise fail pip + # resolution ("No matching distribution") for a missing-wheel ABI. PIP_FIND_LINKS="$GITHUB_WORKSPACE/dist-test" \ - uvx --prerelease allow --with 'flet-cli>=0.86.0.dev0' --with 'flet>=0.86.0.dev0' flet build apk -vv --yes --python-version "$(echo "$INPUT_PYTHON_VERSION" | cut -d. -f1-2)" + uvx --prerelease allow --with 'flet-cli>=0.86.0.dev0' --with 'flet>=0.86.0.dev0' flet build apk --arch x86_64 -vv --yes --python-version "$(echo "$INPUT_PYTHON_VERSION" | cut -d. -f1-2)" - name: Test on Android emulator (API 28, x86_64) if: matrix.platform == 'android' && steps.detect-tests.outputs.has_tests == 'true' @@ -525,6 +539,7 @@ jobs: PKG_NAME: ${{ steps.detect-tests.outputs.pkg_name }} PKG_VERSION: ${{ steps.detect-tests.outputs.pkg_version }} INPUT_PYTHON_VERSION: ${{ inputs.python_version }} + SERIOUS_PYTHON_REF: ${{ inputs.serious_python_ref }} run: | set -euxo pipefail diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 4f26898c..371f529e 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -39,6 +39,14 @@ on: python-build support tree, instead of the pinned date-keyed release tarball: required: false default: "" + serious_python_ref: + description: | + Git ref (branch/tag/SHA) of flet-dev/serious-python to pin the + recipe-tester's serious_python (+ platform impls) to, so on-device tests + validate against an UNRELEASED sp fix. Empty = use the published + serious_python. Replaces the old hardcoded override in the pyproject template: + required: false + default: "fix/soref-package-init" # Reusable-workflow entry point for cross-repo callers. workflow_call: @@ -67,6 +75,10 @@ on: type: string required: false default: "" + serious_python_ref: + type: string + required: false + default: "fix/soref-package-init" secrets: GEMFURY_TOKEN: required: false @@ -181,5 +193,8 @@ jobs: prebuild_recipes: ${{ inputs.prebuild_recipes || '' }} mobile_test_pythons: ${{ inputs.mobile_test_pythons || '3.12' }} python_build_run_id: ${{ inputs.python_build_run_id || '' }} + # No `|| default` here: an explicitly-passed empty string must survive so a + # caller can disable the override and test against the published sp. + serious_python_ref: ${{ inputs.serious_python_ref }} secrets: GEMFURY_TOKEN: ${{ secrets.GEMFURY_TOKEN }} diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index c64d7c0a..a6bee5fa 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -42,16 +42,7 @@ path = "." [tool.flet.android] extract_packages = [__EXTRACT_PACKAGES__] -# TEMPORARY: pull serious_python from the branch carrying the unreleased iOS fixes -# so recipe tests validate against them BEFORE they ship in a published sp release: -# - #223 interdependent-dylib framework relocation (reconcile_framework_install_names) -# — unblocks pyarrow/llama iOS launch (dyld "Library not loaded @rpath/lib*.dylib") -# - _SorefFinder package-__init__ native resolution (apsw et al.) -# flet merges this into the generated Flutter pubspec's dependency_overrides -# (recursive merge, so the nested `git = { … }` table passes straight through). -# REMOVE this block once serious_python is released with these fixes. -[tool.flet.flutter.pubspec.dependency_overrides] -serious_python = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python" } } -serious_python_darwin = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_darwin" } } -serious_python_android = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_android" } } -serious_python_platform_interface = { git = { url = "https://github.com/flet-dev/serious-python.git", ref = "fix/soref-package-init", path = "src/serious_python_platform_interface" } } +# NOTE: the serious_python dependency_overrides (git pin for validating unreleased +# sp fixes) are NO LONGER hardcoded here. stage_recipe.sh appends them when the +# `SERIOUS_PYTHON_REF` env var is set (driven by the CI `serious_python_ref` input, +# or set manually for local builds); empty -> the published serious_python is used. diff --git a/tests/recipe-tester/stage_recipe.sh b/tests/recipe-tester/stage_recipe.sh index 8e02a317..c294e9d1 100755 --- a/tests/recipe-tester/stage_recipe.sh +++ b/tests/recipe-tester/stage_recipe.sh @@ -134,6 +134,24 @@ while IFS= read -r tpl_line || [ -n "$tpl_line" ]; do fi done < "$TPL" +# When SERIOUS_PYTHON_REF is set (from the CI `serious_python_ref` input, or a +# local env), pin serious_python + its platform implementations to that ref of +# flet-dev/serious-python so the recipe-tester on-device test builds against an +# UNRELEASED sp fix (#223 framework relocation, _SorefFinder, ...). Appended here +# instead of hardcoded in the template so the ref is a single CI knob; empty -> +# the published serious_python is used. SERIOUS_PYTHON_URL overrides the repo URL. +if [ -n "${SERIOUS_PYTHON_REF:-}" ]; then + SP_URL="${SERIOUS_PYTHON_URL:-https://github.com/flet-dev/serious-python.git}" + { + echo "" + echo "[tool.flet.flutter.pubspec.dependency_overrides]" + for _sp in serious_python serious_python_darwin serious_python_android serious_python_platform_interface; do + echo "$_sp = { git = { url = \"$SP_URL\", ref = \"$SERIOUS_PYTHON_REF\", path = \"src/$_sp\" } }" + done + } >> "$OUT" + echo " serious_python pinned: $SP_URL@$SERIOUS_PYTHON_REF (dependency_overrides)" +fi + echo "Staged recipe '$RECIPE' (dep: $DEP)" echo " recipe_tests/:" ls -1 "$TEST_DIR" | sed 's/^/ /' From aaa268b4995198b799a5c4161f53cc86438f3dcf Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 16:19:10 +0200 Subject: [PATCH 39/49] =?UTF-8?q?recipe:=20llama-cpp-python=20=E2=80=94=20?= =?UTF-8?q?extract=5Fpackages=20[llama=5Fcpp]=20to=20fix=20Android=20dlope?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Flet 0.86 Android, site-packages ship in sitepackages.zip. llama's native libs live in llama_cpp/lib/ (CMAKE_INSTALL_LIBDIR), which is neither an opt/ tree (so copyOpt never relocates them to jniLibs) nor a set of PyInit extensions (so the .soref relocation skips them) — they stay INSIDE the zip, where _ctypes_extensions.py's ctypes.CDLL(base_path / libllama.so) fails with 'dlopen ... library not found'. The CI failure confirmed the lib was still at sitepackages.zip/llama_cpp/lib/libllama.so (not jniLibs), proving flet does not relocate these ctypes libs. Extracting llama_cpp to disk makes base_path a real filesystem dir so the preloaded ggml chain + libllama load. App-level config, so the wheel is unchanged (build stays 2); Android-only (iOS uses .fwork, already green). [skip ci] --- recipes/llama-cpp-python/meta.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 28fee473..5ae52e25 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -1,3 +1,14 @@ +# On Flet 0.86 Android, site-packages ship inside sitepackages.zip (zipimport). +# llama's native libs live in llama_cpp/lib/ (CMAKE_INSTALL_LIBDIR below) — NOT an +# opt/ tree, so copyOpt never relocates them to jniLibs, and they are ctypes libs +# (not PyInit extensions) so the .soref relocation skips them too. They stay inside +# the zip, where _ctypes_extensions.py's `ctypes.CDLL(base_path / libllama.so)` +# fails ("dlopen ... library not found" — you can't dlopen a .so from a zip). +# Extract llama_cpp to disk so base_path is a real filesystem dir and the ggml/llama +# chain loads. (Android-only config; iOS framework-izes these via .fwork, unaffected.) +extract_packages: + - llama_cpp + package: name: llama-cpp-python version: "0.3.32" From 0ed3ae7b61d14dbbdb838fa7fdfd7c6071c98242 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 16:20:20 +0200 Subject: [PATCH 40/49] skills(forge-error-catalogue): ctypes lib trapped in sitepackages.zip (Android) -> extract_packages; distinct from PyInit ABI-tag; llama-cpp-python case [skip ci] --- .../references/failure-catalogue.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index cc304505..65004172 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1077,6 +1077,31 @@ python-magic entry below. --- +### `RuntimeError: Failed to load shared library '.../sitepackages.zip//lib/lib.so'` / `dlopen failed: library "..." not found` (Android, at import — a **ctypes** lib trapped in the zip) + +**Cause:** the package bundles a **ctypes-loaded** native `.so` in its own subdir +(e.g. `llama_cpp/lib/libllama.so` + the `libggml*` chain, from +`-DCMAKE_INSTALL_LIBDIR=llama_cpp/lib`) and loads it by a `__file__`-relative path. +It is NOT relocated to `jniLibs`: it's not in an `opt/` tree (so `copyOpt` skips it) +and it's not a PyInit extension (so the `.soref` ABI-tag relocation skips it too — +`llvm-nm -D` shows no `PyInit_*`). Under 0.86 it stays *inside* `sitepackages.zip`, and +`ctypes.CDLL("/…/lib.so")` fails — you cannot dlopen from a zip. Tell it apart +from the untagged-extension case below: that's an **importable PyInit** `.so` (fix = +ABI-tag); this is a **dependency lib** the loader opens by path (fix = put it on disk). +The failing path literally contains `sitepackages.zip/` (proof the lib is still zipped, +not in `jniLibs`). + +**Fix:** `extract_packages: []` (same field as the data-file entry above) — extracting +the package to disk makes the lib a real file, so `CDLL(base_path/lib.so)` and the +loader's RTLD_GLOBAL preload of the dependency chain resolve. **No wheel change** (build +number unchanged — it's app-level config). iOS is unaffected (these libs are `.fwork` +framework-ized, resolved by the recipe's ctypes shim). Verified: llama-cpp-python. NB do +NOT "fix" it by moving the libs to `opt/lib/` in the recipe — that routes them through +`copyOpt`→jniLibs (a bare-soname load) instead, changing the loader contract; keep them +in the package's own `lib/` and just extract. + +--- + ### `ModuleNotFoundError: No module named '.'` / `cannot import name '<_ext>' … (most likely due to a circular import)` for a NATIVE submodule (Android) **Cause:** the extension `.so` ships **untagged** — `ncnn/ncnn.so`, From d1bf8c6d52a0d197b555809201407e68a810bcde Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 16:35:04 +0200 Subject: [PATCH 41/49] =?UTF-8?q?Revert=20"recipe:=20llama-cpp-python=20?= =?UTF-8?q?=E2=80=94=20extract=5Fpackages=20[llama=5Fcpp]=20to=20fix=20And?= =?UTF-8?q?roid=20dlopen"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit aaa268b4995198b799a5c4161f53cc86438f3dcf. --- recipes/llama-cpp-python/meta.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 5ae52e25..28fee473 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -1,14 +1,3 @@ -# On Flet 0.86 Android, site-packages ship inside sitepackages.zip (zipimport). -# llama's native libs live in llama_cpp/lib/ (CMAKE_INSTALL_LIBDIR below) — NOT an -# opt/ tree, so copyOpt never relocates them to jniLibs, and they are ctypes libs -# (not PyInit extensions) so the .soref relocation skips them too. They stay inside -# the zip, where _ctypes_extensions.py's `ctypes.CDLL(base_path / libllama.so)` -# fails ("dlopen ... library not found" — you can't dlopen a .so from a zip). -# Extract llama_cpp to disk so base_path is a real filesystem dir and the ggml/llama -# chain loads. (Android-only config; iOS framework-izes these via .fwork, unaffected.) -extract_packages: - - llama_cpp - package: name: llama-cpp-python version: "0.3.32" From 0153ea67e4888b48d9ca754ff04e6c523d81413a Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Mon, 13 Jul 2026 16:48:06 +0200 Subject: [PATCH 42/49] =?UTF-8?q?recipe:=20llama-cpp-python=20=E2=80=94=20?= =?UTF-8?q?loader=20falls=20through=20to=20jniLibs=20bare-soname=20on=20An?= =?UTF-8?q?droid=20(real=20fix)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build-3 loader raised RuntimeError on the FIRST (disk/zip) candidate's miss, never reaching its own bare-soname jniLibs fallback right below. Under Flet 0.86 Android the .so IS relocated into jniLibs (verified: unzip -l APK shows lib/arm64-v8a/libllama.so + libggml*), so the disk path always misses and only the soname load works. Change the main-load except to fall through (pass) instead of raising. Build 2->3 (wheel change). extract_packages was a RED HERRING (reverted): local on-device test showed it moved the .pyc to disk but the .so still went to jniLibs, so the raise still fired. Verified on-device (Android arm64, test_native_lib_callable PASS) with the loader fall-through. Skill entry corrected. [skip ci] --- .../references/failure-catalogue.md | 46 ++++++++++--------- recipes/llama-cpp-python/meta.yaml | 5 +- recipes/llama-cpp-python/patches/mobile.patch | 4 +- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 65004172..232417d3 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1077,28 +1077,30 @@ python-magic entry below. --- -### `RuntimeError: Failed to load shared library '.../sitepackages.zip//lib/lib.so'` / `dlopen failed: library "..." not found` (Android, at import — a **ctypes** lib trapped in the zip) - -**Cause:** the package bundles a **ctypes-loaded** native `.so` in its own subdir -(e.g. `llama_cpp/lib/libllama.so` + the `libggml*` chain, from -`-DCMAKE_INSTALL_LIBDIR=llama_cpp/lib`) and loads it by a `__file__`-relative path. -It is NOT relocated to `jniLibs`: it's not in an `opt/` tree (so `copyOpt` skips it) -and it's not a PyInit extension (so the `.soref` ABI-tag relocation skips it too — -`llvm-nm -D` shows no `PyInit_*`). Under 0.86 it stays *inside* `sitepackages.zip`, and -`ctypes.CDLL("/…/lib.so")` fails — you cannot dlopen from a zip. Tell it apart -from the untagged-extension case below: that's an **importable PyInit** `.so` (fix = -ABI-tag); this is a **dependency lib** the loader opens by path (fix = put it on disk). -The failing path literally contains `sitepackages.zip/` (proof the lib is still zipped, -not in `jniLibs`). - -**Fix:** `extract_packages: []` (same field as the data-file entry above) — extracting -the package to disk makes the lib a real file, so `CDLL(base_path/lib.so)` and the -loader's RTLD_GLOBAL preload of the dependency chain resolve. **No wheel change** (build -number unchanged — it's app-level config). iOS is unaffected (these libs are `.fwork` -framework-ized, resolved by the recipe's ctypes shim). Verified: llama-cpp-python. NB do -NOT "fix" it by moving the libs to `opt/lib/` in the recipe — that routes them through -`copyOpt`→jniLibs (a bare-soname load) instead, changing the loader contract; keep them -in the package's own `lib/` and just extract. +### `RuntimeError: Failed to load shared library '...//lib/lib.so'` / `dlopen failed: library "..." not found` (Android, at import — a **ctypes** loader that raises before its own jniLibs fallback) + +**Cause — NOT what the path suggests.** The failing path (`…/sitepackages.zip//lib/ +lib.so` or `…/extract//lib/lib.so`) is the path the loader *tried*, NOT where +the `.so` is. Flet 0.86 **does relocate these ctypes libs into `jniLibs//`** (verified: +`unzip -l APK` shows `lib/arm64-v8a/libllama.so`, `libggml*.so`), so they ARE loadable — +but only by **bare soname**. The bug is in the recipe's own ctypes loader: its main-load +loop does `for lib_path in _candidate_paths(...): try: CDLL(str(lib_path)) except Exception: +raise RuntimeError(...)` — it **raises on the FIRST (disk/zip) candidate's miss and never +reaches the bare-soname jniLibs fallback that sits right below it**. (Regression: build-1 +guarded with `if lib_path.exists()` so it fell through; the iOS `.fwork` rework dropped the +guard and added the raise.) llama-cpp-python (`libggml*`←`libllama`, `-DCMAKE_INSTALL_LIBDIR= +llama_cpp/lib`) hit exactly this. + +**Fix — the loader, not packaging.** Change the main-load `except` to fall through (`pass`) +instead of raising, so the loop ends and the existing `for _n in (f"lib{name}.so", …): +CDLL(_n)` bare-soname fallback runs and the platform linker resolves it from jniLibs. Bump +the recipe build number (it's a wheel change). iOS is unaffected (the `.fwork` candidate is +tried first and succeeds). **`extract_packages` is a RED HERRING here** — I tried it first +and it FAILED: it moved the package's `.pyc` to disk but the `.so` still went to jniLibs, so +the raise-on-first-candidate still fired (path just changed from `sitepackages.zip/…` to +`extract/…`). Verified on-device (Android arm64, `test_native_lib_callable` PASS) after the +loader fall-through. Only reach for `extract_packages` when the missing thing is a **data +file** read by `__file__` (the entry above), not a native lib. --- diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 28fee473..faa3e1c2 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -3,7 +3,10 @@ package: version: "0.3.32" build: - number: 2 + # 3: mobile.patch loader now falls through to the bare-soname jniLibs load on + # Android instead of raising on the first (disk) candidate — under Flet 0.86 + # the .so is relocated into jniLibs, so the disk/zip path always misses. + number: 3 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/llama-cpp-python/patches/mobile.patch b/recipes/llama-cpp-python/patches/mobile.patch index 3c44366b..b5df63ad 100644 --- a/recipes/llama-cpp-python/patches/mobile.patch +++ b/recipes/llama-cpp-python/patches/mobile.patch @@ -216,8 +216,8 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py + for lib_path in _candidate_paths(lib_base_name): + try: + return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore -+ except Exception as e: -+ raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") ++ except Exception: ++ pass # not on disk (Flet 0.86 Android relocates .so to jniLibs) -> fall through to the soname load below + + # Android: the library was relocated into the APK's jniLibs (not a file on + # disk); load it by bare soname. From a81bf90b5445be64947aa4e2585d0b0381b89f53 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 00:28:33 +0200 Subject: [PATCH 43/49] =?UTF-8?q?recipe:=20llama-cpp-python=20=E2=80=94=20?= =?UTF-8?q?accept=20sys.platform=20=3D=3D=20'android'=20in=20the=20loader?= =?UTF-8?q?=20(py3.13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.13+ reports sys.platform == 'android' (PEP 738) where 3.12 reported 'linux'. The mobile.patch loader's _candidate_paths only matched linux/freebsd/ darwin/ios/win32, so on py3.13 Android it hit the else branch and raised 'Unsupported platform' at import — before the bare-soname jniLibs fallback (added in build 3) could run. Add 'android' to the linux branch so it returns lib.so, which misses on the Flet 0.86 zip and falls through to the jniLibs load, exactly as on py3.12. Build 3 -> 4. (General py3.12->3.13 tell: any loader gating on sys.platform.startswith('linux') breaks on 3.13 Android.) --- recipes/llama-cpp-python/meta.yaml | 5 ++++- recipes/llama-cpp-python/patches/mobile.patch | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index faa3e1c2..5ffb6690 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -6,7 +6,10 @@ build: # 3: mobile.patch loader now falls through to the bare-soname jniLibs load on # Android instead of raising on the first (disk) candidate — under Flet 0.86 # the .so is relocated into jniLibs, so the disk/zip path always misses. - number: 3 + # 4: _candidate_paths also accepts sys.platform == "android" — Python 3.13+ + # reports "android" (PEP 738) where 3.12 reported "linux", so on py3.13 the + # loader raised "Unsupported platform" before the jniLibs fallback could run. + number: 4 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/llama-cpp-python/patches/mobile.patch b/recipes/llama-cpp-python/patches/mobile.patch index b5df63ad..907b8ef3 100644 --- a/recipes/llama-cpp-python/patches/mobile.patch +++ b/recipes/llama-cpp-python/patches/mobile.patch @@ -111,8 +111,8 @@ diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py + # iOS .fwork marker resolved to its real framework binary. + if sys.platform == "emscripten": + names = [f"lib{name}{_EMSCRIPTEN_SIDE_MODULE_SUFFIX}"] -+ elif sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): -+ names = [f"lib{name}.so"] ++ elif sys.platform.startswith("linux") or sys.platform.startswith("freebsd") or sys.platform == "android": ++ names = [f"lib{name}.so"] # Python 3.13+ reports "android" (PEP 738), 3.12 reports "linux" + elif sys.platform == "darwin" or sys.platform == "ios": + # iOS (sys.platform == "ios" on flet's 3.13+, else "darwin"): try the + # framework marker first, then the plain desktop-macOS names. From 56d1e78f350ffaa0fc0a7d172093dde12595d3f7 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 00:30:50 +0200 Subject: [PATCH 44/49] =?UTF-8?q?skills(forge-error-catalogue):=20py3.13?= =?UTF-8?q?=20sys.platform=3D=3D'android'=20loader=20tell=20(PEP=20738)=20?= =?UTF-8?q?=E2=80=94=20llama=20case=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../references/failure-catalogue.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 232417d3..4facb69b 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -1104,6 +1104,26 @@ file** read by `__file__` (the entry above), not a native lib. --- +### `RuntimeError: Unsupported platform` (or a loader taking the wrong branch) on Python **3.13+** Android only — `sys.platform == "android"` + +**Cause:** Python **3.13+ reports `sys.platform == "android"`** on Android (PEP 738); **3.12 +reports `"linux"`**. So any recipe loader (or upstream code) that gates on +`sys.platform.startswith("linux")` or an explicit platform list (`== "linux"`, `in ("linux", +"darwin", …)`) silently takes the wrong branch — or hits an `else: raise` — on a **py3.13 +Android** build, while the *same recipe passes on py3.12*. llama-cpp-python hit this: its ctypes +`_candidate_paths` matched `linux/freebsd/darwin/ios/win32/else-raise` and raised `RuntimeError: +Unsupported platform` at import on 3.13, before its jniLibs bare-soname fallback could run. + +**Fix:** add `or sys.platform == "android"` wherever `"linux"` is special-cased (Android is a +Linux kernel, so the linux path is almost always what you want), and bump the build. This is a +**general py3.12→3.13 tell** — grep a recipe's patches/loaders for `sys.platform` before trusting +a 3.13 build. Distinct from the 3.13/3.14 Android **x86_64 `SIGSYS`/seccomp `open()`** crash, +which is a *native* abort from python-build's mimalloc (fixed in the `20260712` snapshot), not a +Python-level branch — that one shows up as a hard crash with no traceback, this one as a clean +`RuntimeError`/`ModuleNotFoundError` with a full pytest traceback. + +--- + ### `ModuleNotFoundError: No module named '.'` / `cannot import name '<_ext>' … (most likely due to a circular import)` for a NATIVE submodule (Android) **Cause:** the extension `.so` ships **untagged** — `ncnn/ncnn.so`, From 385430f33ab82f65446b2d59ce81577fbbb8b7ee Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 01:01:20 +0200 Subject: [PATCH 45/49] forge: fix_wheel drops foreign-arch .so + tolerates symbol-versioned PyInit Two Android fix_wheel gaps surfaced by pymongo and onnxruntime on py3.13: - pymongo: a plain-setuptools sdist builds extensions in-place into a source tree forge reuses across ABI slices, so bdist_wheel globbed the prior slice's arch-tagged .so into the next slice's wheel (x86_64 wheel carried the arm64 _cbson/_cmessage byte-for-byte). serious_python strips the arch off the SOABI tag when writing .soref -> two arches collide on one .soref -> Gradle 'duplicate entry'. Fix: drop any .so whose ABI-tag triplet != this target's platform_triplet (a per-arch wheel must be arch-pure). - onnxruntime: its pybind module exports the init symbol versioned via a linker version script -> 'llvm-nm -D' prints PyInit_..._state@@VERS_1.0, so the exact 'PyInit_' membership test missed it, the .so shipped bare on BOTH arches, no .soref, ModuleNotFoundError on device. Fix: match the base name tolerating a trailing @version suffix. Both are generic fix_wheel changes; no recipe edits. Catalogue updated. [skip ci] --- .../references/failure-catalogue.md | 32 +++++++++++++-- src/forge/build.py | 39 ++++++++++++++++++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index 4facb69b..ffb3b8e2 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -687,6 +687,22 @@ by CI round 1 poisoning the x86_64 wheel with arm64 code), inherited by tflite-runtime. Same root-cause class as a recipe caching a per-arch artifact under a global guard (ckzg's vendored libblst). +**Plain setuptools variant (can't re-key its build dir) — forge auto-drops it:** +a stock sdist with **no shim** builds extensions in-place with `build_ext +--inplace`, writing `NAME.cpython--.so` into the reused source +tree; the *next* ABI slice's `bdist_wheel` then globs the **prior slice's** +arch-tagged `.so` alongside its own (pymongo: the x86_64 wheel carried the arm64 +`bson/_cbson` + `pymongo/_cmessage` **byte-for-byte** — identical size + mtime). +Distinct downstream symptom from the readelf-arch-mismatch above: serious_python +strips the arch off the SOABI tag when writing `.soref`, so two arches collide on +one `.soref` → **Gradle `duplicate entry: .soref`**, not a runtime +import error. **Fix: none per-recipe — forge `fix_wheel` (build.py, Android +branch) drops any `.so` whose ABI-tag triplet ≠ this target's +`platform_triplet`** (a per-arch wheel must be arch-pure). Generic; covers any +in-place-building setuptools recipe. Tell: `unzip -l` shows two `cpython-…-` ++ `cpython-…-` copies of the same module in one wheel. pymongo 4.17 +(fixed build.py 2026-07-14). + --- ### Android wheel ships a dead `lib-jni.so` (force-enabled component that can never work under flet) @@ -1133,9 +1149,8 @@ ABI-tagged names; a bare `NAME.so` is treated as a plain dependency lib, gets no `.soref`, and the import finder can't resolve it. CMake / SWIG / Cython / nanobind builds routinely emit untagged extensions (they can't derive the target SOABI when cross-compiling; setuptools/maturin tag theirs, which is why numpy/pandas/pyarrow are -fine). The x86_64 leg can break while arm64 is fine if the build tags -nondeterministically per arch (onnxruntime). The misleading "circular import" -message is just Python's wording when a `from . import _ext` can't find the `.so`. +fine). The misleading "circular import" message is just Python's wording when a +`from . import _ext` can't find the `.so`. **Fix:** none per-recipe — **forge `fix_wheel` (build.py, Android branch) ABI-tags any bare `.so` exporting `PyInit_`** (a genuine extension; `llvm-nm -D` @@ -1143,7 +1158,16 @@ discriminates it from a dependency lib, which is left untouched) to `.cpython-3X.so`, so serious_python writes its `.soref`. Just rebuild; the fix is generic and covers future CMake/SWIG/Cython recipes. If you hit this, confirm via `unzip -l` that the wheel ships a bare `.so` exporting `PyInit_*`. Verified: -ncnn, faiss-cpu, coolprop; onnxruntime x86_64. +ncnn, faiss-cpu, coolprop, onnxruntime. + +**Subtlety — symbol-versioned `PyInit`:** a build applying a **linker version +script** exports the init symbol *versioned* — `llvm-nm -D` prints +`PyInit_onnxruntime_pybind11_state@@VERS_1.0`, not the bare name. forge's detector +must match the base name *tolerating a `@version` suffix* (`tok == pyinit or +tok.startswith(pyinit + "@")`); an exact-equality check silently misses it, the .so +ships bare on **both** arches (not just x86_64), and you get `ModuleNotFoundError` on +device even though `unzip -l` shows the .so present. Tell: `nm -D | grep PyInit` +shows a `@@`/`@` suffix. onnxruntime 1.27 (fixed build.py 2026-07-14). --- diff --git a/src/forge/build.py b/src/forge/build.py index 86e28a74..7dce972e 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -874,6 +874,34 @@ def fix_wheel(self, wheel_dir: Path): if self.cross_venv.sdk == "android": env = self.compile_env() + # Drop foreign-arch extension modules that leaked into this wheel. + # setuptools' in-place `build_ext` writes NAME.cpython--.so + # into the (reused) unpacked-sdist source tree; when forge builds each + # ABI from that same tree, a prior slice's arch-tagged .so lingers and + # bdist_wheel globs it into the next slice's wheel (pymongo: the x86_64 + # wheel carried the arm64 `_cbson`/`_cmessage` byte-for-byte). Downstream + # serious_python strips the arch off the SOABI tag when writing `.soref` + # markers, so two arches collide on one `.soref` and Gradle fails + # with "duplicate entry". A per-arch wheel must be arch-pure — keep only + # this target's platform triplet. + keep_triplet = self.cross_venv.platform_triplet + foreign_triplets = "|".join( + re.escape(triplet) + for triplet in self.cross_venv.ANDROID_PLATFORM_TRIPLET.values() + if triplet != keep_triplet + ) + foreign_ext_re = re.compile( + rf"\.cpython-\d+-(?:{foreign_triplets})\.so$" + ) + for so in wheel_dir.glob("**/*.so"): + if foreign_ext_re.search(so.name): + log( + self.log_file, + f"[{self.cross_venv}] Dropping foreign-arch extension " + f"{so.name}", + ) + so.unlink() + # ABI-tag bare CPython extension modules so serious_python's Android # packaging recognizes them. That packaging only relocates a native # module into jniLibs and writes the `.soref` marker its on-device @@ -899,7 +927,16 @@ def fix_wheel(self, wheel_dir: Path): ) except (subprocess.CalledProcessError, OSError): continue # not an analyzable ELF; leave it alone - if f"PyInit_{module}" in symbols.split(): + # Match `PyInit_`, tolerating a symbol-version suffix. + # A build applying a linker version script exports the init + # symbol versioned (onnxruntime: `PyInit_..._state@@VERS_1.0`), + # which an exact-equality check would miss — leaving the module + # bare, unrecognized by serious_python, and unimportable on-device. + pyinit = f"PyInit_{module}" + if any( + tok == pyinit or tok.startswith(pyinit + "@") + for tok in symbols.split() + ): tagged = so.with_name(module + ext_suffix) log( self.log_file, From 2ece499569ce89293671b6443574b18417487ea2 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 02:17:48 +0200 Subject: [PATCH 46/49] updates --- .claude/skills/local-recipe-testing/SKILL.md | 26 ++++++++++++++++++++ .github/workflows/build-wheels-version.yml | 9 ++----- .github/workflows/build-wheels.yml | 4 +-- recipes/astropy/meta.yaml | 8 +++--- recipes/coolprop/meta.yaml | 4 --- recipes/faiss-cpu/meta.yaml | 4 --- recipes/llama-cpp-python/meta.yaml | 8 +----- recipes/matplotlib/meta.yaml | 10 ++++---- recipes/ncnn/meta.yaml | 4 --- recipes/opencv-python/meta.yaml | 11 ++++----- recipes/pyarrow/meta.yaml | 3 --- recipes/scikit-learn/meta.yaml | 8 +++--- recipes/spacy/meta.yaml | 12 ++++----- recipes/thinc/meta.yaml | 8 +++--- src/forge/build.py | 17 +++++++------ tests/recipe-tester/pyproject.toml.tpl | 5 ---- 16 files changed, 68 insertions(+), 73 deletions(-) diff --git a/.claude/skills/local-recipe-testing/SKILL.md b/.claude/skills/local-recipe-testing/SKILL.md index 76cccfe8..e65f9384 100644 --- a/.claude/skills/local-recipe-testing/SKILL.md +++ b/.claude/skills/local-recipe-testing/SKILL.md @@ -173,6 +173,32 @@ The recipe-tester proves the wheel loads and its own tests pass. For recipes who Note for consumers that are pure-python + sdist-only (insightface): they need no recipe and no wheel — the verify-app's pyproject declares `[tool.flet] source_packages = [""]` instead (see `new-mobile-recipe` § "When NOT to use"). +## Testing an OFFICIAL PyPI mobile wheel (not a forge wheel) + +Upstream packages now publish cibuildwheel-built iOS/Android wheels to PyPI (cp313+ +only). They are live pip candidates in every 0.86 build, **but while a forge recipe +exists on pypi.flet.dev it deterministically shadows them** — pip's sort at equal +versions is tag-priority (forge `android_24` > official `android_21`) then build tag +(forge `-1-` > none). To force the official wheel on-device without touching the index: +retag a downloaded copy into a find-links dir with a HIGH build tag — and on Android +also lift the platform tag past forge's — + +```bash +uvx --from wheel wheel tags --build 99 --platform-tag android_24_arm64_v8a .whl # android +uvx --from wheel wheel tags --build 99 .whl # iOS: tags already equal +``` + +then build the verify-app with `PIP_FIND_LINKS=` and `--python-version 3.13` +(official wheels are cp313+; pin `flet>=0.86.0.dev0` in the app deps or pip resolves the +stable 0.85 runtime against the 0.86 template — SERIOUS_PYTHON_APP contract mismatch). +Retagging rewrites only dist-info; **hash-verify the payload end-to-end**: the staged +`build/site-packages//.../*.so` must equal the wheel's, and the APK's relocated +`lib//lib.so` equals it after `llvm-strip --strip-debug --strip-unneeded` +(gradle strips jniLibs; byte-identical is the pass criterion). Precedent: lru-dict 1.4.1 +(EXIT 0 android + iOS-sim) and pyzmq 27.1.0 (EXIT 0 android, auditwheel-vendored +`pyzmq.libs/` resolved via the jniLibs basename flatten) — full analysis in +`playground/cibuildwheel-flet-compat.md`. + ## Triage when the in-app test fails Read `console.log` first; if the app died without writing a result, pull logcat: diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index 3c58f099..ff063b29 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -55,7 +55,7 @@ on: Empty = published serious_python: required: false type: string - default: "fix/soref-package-init" + default: "" secrets: GEMFURY_TOKEN: required: false @@ -281,7 +281,7 @@ jobs: FORGE_PACKAGES: ${{ matrix.forge_packages }} PREBUILD_RECIPES: ${{ inputs.prebuild_recipes }} PLATFORM: ${{ matrix.platform }} - PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id != '' && inputs.python_build_run_id || '29242819654' }} + PYTHON_BUILD_RUN_ID: ${{ inputs.python_build_run_id }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PIP_FIND_LINKS: ${{ github.workspace }}/dist UV_PYTHON: ${{ inputs.python_version }} @@ -510,11 +510,6 @@ jobs: fi cd tests/recipe-tester - # The Android on-device test runs on an x86_64 emulator, so build the - # APK for x86_64 ONLY: it's the sole ABI the test needs, it cuts build - # time, and it sidesteps recipes with excluded_arches (e.g. pyarrow - # drops armeabi-v7a) whose all-ABI app build would otherwise fail pip - # resolution ("No matching distribution") for a missing-wheel ABI. PIP_FIND_LINKS="$GITHUB_WORKSPACE/dist-test" \ uvx --prerelease allow --with 'flet-cli>=0.86.0.dev0' --with 'flet>=0.86.0.dev0' flet build apk --arch x86_64 -vv --yes --python-version "$(echo "$INPUT_PYTHON_VERSION" | cut -d. -f1-2)" diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 371f529e..d0f05194 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -46,7 +46,7 @@ on: validate against an UNRELEASED sp fix. Empty = use the published serious_python. Replaces the old hardcoded override in the pyproject template: required: false - default: "fix/soref-package-init" + default: "" # Reusable-workflow entry point for cross-repo callers. workflow_call: @@ -78,7 +78,7 @@ on: serious_python_ref: type: string required: false - default: "fix/soref-package-init" + default: "" secrets: GEMFURY_TOKEN: required: false diff --git a/recipes/astropy/meta.yaml b/recipes/astropy/meta.yaml index fe514dbb..97be81bc 100644 --- a/recipes/astropy/meta.yaml +++ b/recipes/astropy/meta.yaml @@ -2,10 +2,6 @@ package: name: astropy version: "8.0.0" -# astropy reads astropy/CITATION via a real __file__ path at import. -extract_packages: - - astropy - build: number: 1 @@ -17,3 +13,7 @@ requirements: patches: - android-cython-cpow.patch # {% endif %} + +# astropy reads astropy/CITATION via a real __file__ path at import. +extract_packages: + - astropy diff --git a/recipes/coolprop/meta.yaml b/recipes/coolprop/meta.yaml index 95b98259..ccae1eb9 100644 --- a/recipes/coolprop/meta.yaml +++ b/recipes/coolprop/meta.yaml @@ -14,10 +14,6 @@ patches: - mkdir-cython-output.patch build: - # Rebuilt at build 11 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB - # conversion (CoolProp's CMake extension is an MH_BUNDLE Xcode refused to - # link into the app's frameworks). The forge fix lands with this branch, so a - # bump is required for the fixed wheel to rebuild + publish on merge. number: 11 script_env: # {% if sdk == 'android' %} diff --git a/recipes/faiss-cpu/meta.yaml b/recipes/faiss-cpu/meta.yaml index 729dc3a0..34f4c55c 100644 --- a/recipes/faiss-cpu/meta.yaml +++ b/recipes/faiss-cpu/meta.yaml @@ -8,10 +8,6 @@ source: url: https://github.com/facebookresearch/faiss/archive/refs/tags/v{{ version }}.tar.gz build: - # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB - # conversion (faiss's _swigfaiss CMake extension is an MH_BUNDLE Xcode - # refused to link into the app's frameworks). The forge fix lands with this - # branch, so a bump is required for the fixed wheel to rebuild + publish on merge. number: 2 script_env: # scikit-build-core honors the CMAKE_ARGS env var (appended to its cmake call), diff --git a/recipes/llama-cpp-python/meta.yaml b/recipes/llama-cpp-python/meta.yaml index 5ffb6690..28fee473 100644 --- a/recipes/llama-cpp-python/meta.yaml +++ b/recipes/llama-cpp-python/meta.yaml @@ -3,13 +3,7 @@ package: version: "0.3.32" build: - # 3: mobile.patch loader now falls through to the bare-soname jniLibs load on - # Android instead of raising on the first (disk) candidate — under Flet 0.86 - # the .so is relocated into jniLibs, so the disk/zip path always misses. - # 4: _candidate_paths also accepts sys.platform == "android" — Python 3.13+ - # reports "android" (PEP 738) where 3.12 reported "linux", so on py3.13 the - # loader raised "Unsupported platform" before the jniLibs fallback could run. - number: 4 + number: 2 script_env: # {% if sdk == 'android' %} CMAKE_ARGS: >- diff --git a/recipes/matplotlib/meta.yaml b/recipes/matplotlib/meta.yaml index 49e9f1bb..4811b506 100644 --- a/recipes/matplotlib/meta.yaml +++ b/recipes/matplotlib/meta.yaml @@ -2,11 +2,6 @@ package: name: matplotlib version: "3.10.9" -# matplotlib reads mpl-data/matplotlibrc (+ fonts/styles) via a real __file__ -# path at import; extract it so those reads resolve outside Flet 0.86's zip. -extract_packages: - - matplotlib - requirements: build: - ninja @@ -29,5 +24,10 @@ build: - -Csetup-args=--cross-file - -Csetup-args={MESON_CROSS_FILE} +# matplotlib reads mpl-data/matplotlibrc (+ fonts/styles) via a real __file__ +# path at import; extract it so those reads resolve outside Flet 0.86's zip. +extract_packages: + - matplotlib + # potential issues: # https://github.com/godotengine/godot/pull/101036/files \ No newline at end of file diff --git a/recipes/ncnn/meta.yaml b/recipes/ncnn/meta.yaml index fe4a6beb..9b7748bd 100644 --- a/recipes/ncnn/meta.yaml +++ b/recipes/ncnn/meta.yaml @@ -3,10 +3,6 @@ package: version: "1.0.20260526" build: - # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB - # conversion (ncnn's CMake extension is an MH_BUNDLE Xcode refused to link - # into the app's frameworks). The forge fix lands with this branch, so a - # bump is required for the fixed wheel to rebuild + publish on merge. number: 2 script_env: _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' diff --git a/recipes/opencv-python/meta.yaml b/recipes/opencv-python/meta.yaml index 8e5fff96..a45ebd63 100644 --- a/recipes/opencv-python/meta.yaml +++ b/recipes/opencv-python/meta.yaml @@ -2,12 +2,6 @@ package: name: opencv-python version: "5.0.0.93" -# Ship cv2 as a real extracted directory (not inside sitepackages.zip) so the -# loader's extra-submodule scan can os.listdir() the package; see mobile.patch -# note 3 for the Flet 0.86 native-relocation loader fix this pairs with. -extract_packages: - - cv2 - build: number: 2 script_env: @@ -68,3 +62,8 @@ requirements: patches: - mobile.patch + +# So the loader's extra-submodule scan can os.listdir() the package; see mobile.patch +# note 3 for the Flet 0.86 native-relocation loader fix this pairs with. +extract_packages: + - cv2 \ No newline at end of file diff --git a/recipes/pyarrow/meta.yaml b/recipes/pyarrow/meta.yaml index 241d646c..bb444feb 100644 --- a/recipes/pyarrow/meta.yaml +++ b/recipes/pyarrow/meta.yaml @@ -15,9 +15,6 @@ package: # x86_64 -> x86 and arm64/arm64-v8a -> aarch64 covers every shipped slice. # {% set arrow_cpu = 'x86' if 'x86' in arch else 'aarch64' %} build: - # Rebuilt at build 2 to pick up forge fix_wheel's iOS MH_BUNDLE->MH_DYLIB - # conversion (pyarrow's _json/_pyarrow_cpp_tests/lib extensions are CMake - # MODULE bundles that Xcode refused to link into the app's frameworks). number: 2 script_env: # Belt-and-suspenders: hard-lock every OPTIONAL component OFF regardless of diff --git a/recipes/scikit-learn/meta.yaml b/recipes/scikit-learn/meta.yaml index 88e2c8ad..366c0065 100644 --- a/recipes/scikit-learn/meta.yaml +++ b/recipes/scikit-learn/meta.yaml @@ -2,10 +2,6 @@ package: name: scikit-learn version: "1.9.0" -# scikit-learn reads sklearn/utils/_repr_html/estimator.css via __file__ at import. -extract_packages: - - sklearn - build: number: 2 backend-args: @@ -33,3 +29,7 @@ requirements: patches: - relax-scipy-build-cap.patch - disable-openmp-non-android.patch + +# scikit-learn reads sklearn/utils/_repr_html/estimator.css via __file__ at import. +extract_packages: + - sklearn diff --git a/recipes/spacy/meta.yaml b/recipes/spacy/meta.yaml index 619c693b..28b39b1b 100644 --- a/recipes/spacy/meta.yaml +++ b/recipes/spacy/meta.yaml @@ -2,12 +2,6 @@ package: name: spacy version: "3.8.13" -# spacy imports thinc (-> thinc/backends/_custom_kernels.cu) and reads its own -# lang data via __file__; extract both. -extract_packages: - - spacy - - thinc - build: number: 1 @@ -22,3 +16,9 @@ requirements: # load time, which the device runtime doesn't provide unless we bundle it. - flet-libcpp-shared >=27.2.12479018 # {% endif %} + +# spacy imports thinc (-> thinc/backends/_custom_kernels.cu) and reads its own +# lang data via __file__; extract both. +extract_packages: + - spacy + - thinc diff --git a/recipes/thinc/meta.yaml b/recipes/thinc/meta.yaml index a320a784..adee897c 100644 --- a/recipes/thinc/meta.yaml +++ b/recipes/thinc/meta.yaml @@ -2,10 +2,6 @@ package: name: thinc version: "8.3.13" -# thinc reads thinc/backends/_custom_kernels.cu via __file__ at import. -extract_packages: - - thinc - build: number: 1 @@ -24,3 +20,7 @@ requirements: # load time, which the device runtime doesn't provide unless we bundle it. - flet-libcpp-shared >=27.2.12479018 # {% endif %} + +# thinc reads thinc/backends/_custom_kernels.cu via __file__ at import. +extract_packages: + - thinc diff --git a/src/forge/build.py b/src/forge/build.py index 7dce972e..4eec7731 100644 --- a/src/forge/build.py +++ b/src/forge/build.py @@ -794,13 +794,13 @@ def _check_elf_alignment(self, so_path: Path): log(self.log_file, f"[{self.cross_venv}] {so_path.name}: 16KB alignment OK") def _bundle_to_dylib(self, so_path: Path) -> bool: - """Convert a thin ``MH_BUNDLE`` Mach-O extension to ``MH_DYLIB`` in place. + """Convert a thin `MH_BUNDLE` Mach-O extension to `MH_DYLIB` in place. - Injects an ``LC_ID_DYLIB`` load command into the header's free padding and + Injects an `LC_ID_DYLIB` load command into the header's free padding and flips the filetype byte. Returns True if a conversion was made, False if - the file is not a thin little-endian ``MH_BUNDLE`` (already a dylib, a fat + the file is not a thin little-endian `MH_BUNDLE` (already a dylib, a fat binary, or not Mach-O — all left untouched). The caller must re-sign - (``codesign --force --sign -``) afterwards, since the edit invalidates the + (`codesign --force --sign -`) afterwards, since the edit invalidates the linker's ad-hoc signature. """ MH_MAGIC_64 = 0xFEEDFACF @@ -825,7 +825,10 @@ def _bundle_to_dylib(self, so_path: Path) -> bool: off = 32 for _ in range(ncmds): cmd, cmdsize = struct.unpack_from(" the published serious_python is used. From 312e5d2dc656f26a48c3c5fc2dbabfe29be27d06 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 11:50:03 +0200 Subject: [PATCH 47/49] =?UTF-8?q?recipe:=20matplotlib=20=E2=80=94=20fix=20?= =?UTF-8?q?iOS=20launch=20SIGSEGV=20(ft2font=20static-init=20GIL)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit matplotlib's ft2font extension crashed at iOS app LAUNCH with SIGSEGV, before Py_Initialize() — 0-byte console.log, faulting in dyld runInitializers -> _GLOBAL__sub_I_ft2font_wrapper.cpp -> pybind11::gil_scoped_acquire -> PyGILState_Ensure. Root cause: matplotlib's vendored pybind11 enum helper (src/_enums.h) declares each enum via P11X_DECLARE_ENUM, which expands to a namespace-scope global whose initializer acquires the GIL + calls pybind11::cast — i.e. touches the Python C-API from a C++ static initializer. Harmless on desktop/Android (ext dlopen'd at import, interpreter up), fatal on iOS: serious_python links each ext as a framework whose static initializers dyld runs at app launch, before the interpreter exists. Fix (mobile.patch on _enums.h): record each enum spec as plain C++ data at static-init (no GIL, no pybind11::cast) and defer all Python work to bind_enums(), which already runs inside PYBIND11_MODULE (at import). Build 1 -> 2 (also republishes the iOS wheel with forge's MH_BUNDLE->MH_DYLIB conversion; the published build-1 iOS wheel was a stale MH_BUNDLE). Verified on-device: patched wheel on the iOS simulator launches, test_png PASSED, EXIT 0; disassembly confirms the rebuilt static initializer calls only plain C++. Catalogue updated (forge-error-catalogue: iOS launch SIGSEGV in runInitializers). [skip ci] --- .../references/failure-catalogue.md | 49 +++++++++++ recipes/matplotlib/meta.yaml | 6 +- recipes/matplotlib/patches/mobile.patch | 84 +++++++++++++++++++ 3 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 recipes/matplotlib/patches/mobile.patch diff --git a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md index ffb3b8e2..9ed4263c 100644 --- a/.claude/skills/forge-error-catalogue/references/failure-catalogue.md +++ b/.claude/skills/forge-error-catalogue/references/failure-catalogue.md @@ -300,6 +300,55 @@ flet build ios-simulator --python-version 3.12`) not `uvx` (else --- +### iOS: app builds + launches but **SIGSEGV at launch** in `dyld … runInitializers`, 0-byte `console.log` — an extension runs the Python C-API in a C++ **static initializer** + +**Symptom:** same 0-byte `console.log` / crash-before-Python signature as the #223 +entry above (app builds + installs + launches, then dies; the on-device job just +times out waiting for the EXIT sentinel). But the difference is in the crash report: +pull `~/Library/Logs/DiagnosticReports/recipe-tester-*.ips` (a fresh one is written +on every sim launch) — `termination … SIGSEGV`, `EXC_BAD_ACCESS … KERN_INVALID_ADDRESS +at 0x68`, faulting thread bottomed in `dyld_sim … findAndRunAllInitializers → +runInitializersBottomUp` → `. _GLOBAL__sub_I_.cpp` → +`pybind11::gil_scoped_acquire` → `get_internals` → `Python PyGILState_Ensure`. +Seen: matplotlib 3.10 (ft2font, via its vendored `p11x` enum helper). + +**Cause:** serious_python **links** each frameworkized ext into the app, so dyld runs +that ext's C++ **static initializers at app LAUNCH — before `Py_Initialize()`**. Any +global / namespace-scope object whose constructor touches the Python C-API (acquires +the GIL, calls `pybind11::cast`, imports a module) dereferences the not-yet-created +interpreter (`_PyRuntime` still zero → deref at small offset 0x68) → segfault before +any Python runs (hence the 0-byte console.log). It's fine on desktop/Android because +there the ext is `dlopen`'d **at import** (interpreter already up). matplotlib's +`src/_enums.h` `P11X_DECLARE_ENUM` macro is the concrete case: it expands to a +namespace-scope global lambda that runs `py::gil_scoped_acquire gil; p11x::enums[…] = +pybind11::cast(…)` at static-init (ft2font declares Kerning/LoadFlags/FaceFlags/ +StyleFlags this way). **Distinct** from #223 (a *missing-dylib* link error — dyld +`Library not loaded`) and from the MH_BUNDLE→MH_DYLIB conversion (which is correct +here: verify with `codesign -v` OK, `otool -l` shows `LC_BUILD_VERSION platform 7` +IOSSIMULATOR, and `nm -m` shows all undefined syms **two-level bound**, not flat +`dynamically looked up`). The wheel is statically flawless yet crashes — the fault is +purely *when* its initializer runs. + +**Tell:** a well-formed iOS ext (valid sig, right platform, two-level symbols) that +crashes at launch with 0-byte console.log. `nm | grep _GLOBAL__sub_I` confirms a +static initializer exists; inspect the source for Python-API calls at namespace scope. + +**Fix (recipe patch):** move the Python work OUT of the static initializer into the +`PYBIND11_MODULE` body / `PyInit` (runs at import, interpreter up). For matplotlib: +patch `_enums.h` so `P11X_DECLARE_ENUM`'s static-init lambda stores the enum spec as +**plain C++ data** (no GIL, no `pybind11::cast`) and `bind_enums()` — already called +inside `PYBIND11_MODULE` — does the cast + `enum.*` class construction there. General +rule: under serious_python's link-at-launch iOS model an extension **must not call the +Python C-API from a C++ static initializer**. **VERIFIED:** matplotlib 3.10.9 build 2 +(`recipes/matplotlib/patches/mobile.patch` on `_enums.h`) — the patched wheel on the +iOS simulator now launches, `test_png PASSED`, `EXIT 0` (was a launch SIGSEGV); +disassembly confirms the rebuilt `_GLOBAL__sub_I_ft2font_wrapper.cpp` calls only plain +C++ (vector/unordered_map emplace), no `gil_scoped_acquire`/`get_internals`. +opencv/scipy/onnxruntime pybind modules don't hit this — their Python access is all +inside `PyInit`. + +--- + ### iOS: a flet-lib*'s native lib is silently absent from the app bundle **Symptom:** `flet build ios-simulator` succeeds, but at runtime the consumer diff --git a/recipes/matplotlib/meta.yaml b/recipes/matplotlib/meta.yaml index 4811b506..a1e9cab4 100644 --- a/recipes/matplotlib/meta.yaml +++ b/recipes/matplotlib/meta.yaml @@ -15,7 +15,7 @@ requirements: # {% endif %} build: - number: 1 + number: 2 # {% if sdk == 'android' and arch in ['armeabi-v7a', 'x86'] %} script_env: CPPFLAGS: -Wno-c++11-narrowing @@ -29,5 +29,5 @@ build: extract_packages: - matplotlib -# potential issues: -# https://github.com/godotengine/godot/pull/101036/files \ No newline at end of file +patches: + - mobile.patch \ No newline at end of file diff --git a/recipes/matplotlib/patches/mobile.patch b/recipes/matplotlib/patches/mobile.patch new file mode 100644 index 00000000..59226b53 --- /dev/null +++ b/recipes/matplotlib/patches/mobile.patch @@ -0,0 +1,84 @@ +matplotlib ft2font crashes at launch on iOS (SIGSEGV before Py_Initialize). + +matplotlib's vendored pybind11 enum helper (src/_enums.h) declares each enum +(Kerning/LoadFlags/FaceFlags/StyleFlags in ft2font_wrapper.cpp) via the +P11X_DECLARE_ENUM macro, which expands to a namespace-scope global whose +initializer runs `py::gil_scoped_acquire` + `pybind11::cast` — i.e. it touches +the Python C-API from a C++ STATIC INITIALIZER. + +On desktop/Android the extension is dlopen'd at import (interpreter already up), +so that is harmless. But serious_python links each extension into the iOS app as +a framework, and dyld runs its C++ static initializers at APP LAUNCH — before +Py_Initialize(). The GIL acquire then dereferences the not-yet-created +interpreter (_PyRuntime still zero) and the app SIGSEGVs before any Python runs +(0-byte console.log; crash report bottoms in dyld runInitializers -> +_GLOBAL__sub_I_ft2font_wrapper.cpp -> gil_scoped_acquire -> PyGILState_Ensure). + +Fix: record each enum spec as plain C++ data at static-init (no GIL, no +pybind11::cast) and defer ALL Python work to bind_enums(), which already runs +inside PYBIND11_MODULE (at import, interpreter live). Same observable behaviour +on every platform; only WHEN the Python objects are built changes. + +--- a/src/_enums.h ++++ b/src/_enums.h +@@ -32,18 +32,35 @@ + namespace { + namespace py = pybind11; + +- // Holder is (py_base_cls, [(name, value), ...]) before module init; +- // converted to the Python class object after init. ++ // Enum specs collected at static-init as PLAIN C++ DATA — no Python C-API ++ // is touched here. This matters on iOS: serious_python links each extension ++ // as a framework whose C++ static initializers dyld runs at APP LAUNCH, ++ // before Py_Initialize(). Upstream acquired the GIL and called ++ // pybind11::cast in the static initializer (see P11X_DECLARE_ENUM below), ++ // dereferencing the not-yet-created interpreter -> SIGSEGV before any Python ++ // ran. We record the raw (base_cls, [(name, value), ...]) here and defer ALL ++ // Python work to bind_enums(), which runs from PYBIND11_MODULE (at import, ++ // interpreter live). Desktop/Android are unaffected (same end state). ++ using enum_spec = ++ std::pair>>; ++ auto enum_specs = std::unordered_map{}; ++ ++ // Python enum CLASS objects, built by bind_enums() at import; read by the ++ // type_caster below at call time. + auto enums = std::unordered_map{}; + + auto bind_enums(py::module mod) -> void + { +- for (auto& [py_name, spec]: enums) { +- auto const& [py_base_cls, pairs] = +- spec.cast>(); +- mod.attr(py::cast(py_name)) = spec = ++ for (auto& [py_name, spec]: enum_specs) { ++ auto const& [py_base_cls, pairs] = spec; ++ auto py_pairs = py::list{}; ++ for (auto const& [k, v]: pairs) { ++ py_pairs.append(py::make_tuple(k, v)); ++ } ++ enums[py_name] = + py::module::import("enum").attr(py_base_cls.c_str())( +- py_name, pairs, py::arg("module") = mod.attr("__name__")); ++ py_name, py_pairs, py::arg("module") = mod.attr("__name__")); ++ mod.attr(py::cast(py_name)) = enums[py_name]; + } + } + } +@@ -56,13 +73,13 @@ + namespace { \ + [[maybe_unused]] auto const P11X_CAT(enum_placeholder_, __COUNTER__) = \ + [](auto args) { \ +- py::gil_scoped_acquire gil; \ + using int_t = std::underlying_type_t; \ +- auto pairs = std::vector>{}; \ ++ auto pairs = std::vector>{}; \ + for (auto& [k, v]: args) { \ +- pairs.emplace_back(k, int_t(v)); \ ++ pairs.emplace_back(k, static_cast(int_t(v))); \ + } \ +- p11x::enums[py_name] = pybind11::cast(std::pair{py_base_cls, pairs}); \ ++ p11x::enum_specs[py_name] = \ ++ std::make_pair(std::string{py_base_cls}, pairs); \ + return 0; \ + } (std::vector{std::pair __VA_ARGS__}); \ + } \ From fbef5db631b003b82aff30d7d5b897242359372e Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 15:22:02 +0200 Subject: [PATCH 48/49] ci: add publish opt-in to deploy dispatches + bump pymongo/onnxruntime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-wheels(.yml/-version.yml): new `publish` input (boolean, default false). The publish step now fires when a caller opts in (publish: true — e.g. a deploy workflow_dispatch with a pinned python_build_run_id) OR on the existing automatic path (push to main with an empty python_build_run_id). Lets us deploy wheels via dispatch with a specific python-build run + sp ref. - pymongo 1->2, onnxruntime 1->2: their build-1 wheels are already published, so a rebuild would 409-skip; bump so forge fix_wheel's foreign-arch drop (pymongo) and symbol-versioned-PyInit ABI-tag (onnxruntime) actually reach pypi.flet.dev. [skip ci] --- .github/workflows/build-wheels-version.yml | 16 +++++++++++++++- .github/workflows/build-wheels.yml | 19 ++++++++++++++----- recipes/onnxruntime/meta.yaml | 2 +- recipes/pymongo/meta.yaml | 2 +- setup.sh | 2 +- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-wheels-version.yml b/.github/workflows/build-wheels-version.yml index ff063b29..0b86d69b 100644 --- a/.github/workflows/build-wheels-version.yml +++ b/.github/workflows/build-wheels-version.yml @@ -56,6 +56,12 @@ on: required: false type: string default: "" + publish: + description: | + Publish the built wheels to pypi.flet.dev at the end of the run. + required: false + type: boolean + default: false secrets: GEMFURY_TOKEN: required: false @@ -107,6 +113,12 @@ on: python-build support tree, instead of the pinned date-keyed release tarball: required: false default: "" + publish: + description: | + Publish the built wheels to pypi.flet.dev at the end of the run. + type: boolean + required: false + default: false env: FORGE_NDK_VERSION: r27d # used by forge for wheel cross-compile @@ -597,7 +609,9 @@ jobs: retention-days: 90 - name: Publish wheels - if: ${{ success() && hashFiles('dist/*.whl') != '' && github.event_name == 'push' && github.ref == 'refs/heads/main' && inputs.python_build_run_id == '' }} + # Publish when a caller explicitly opts in (`publish: true`), + # OR on the default automatic path (push to main). + if: ${{ success() && hashFiles('dist/*.whl') != '' && (inputs.publish || (github.event_name == 'push' && github.ref == 'refs/heads/main' && inputs.python_build_run_id == '')) }} shell: bash env: GEMFURY_TOKEN: ${{ secrets.GEMFURY_TOKEN }} diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index d0f05194..3ab92ea6 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -47,6 +47,12 @@ on: serious_python. Replaces the old hardcoded override in the pyproject template: required: false default: "" + publish: + description: | + Publish the built wheels to pypi.flet.dev at the end of the run. + type: boolean + required: false + default: false # Reusable-workflow entry point for cross-repo callers. workflow_call: @@ -79,16 +85,18 @@ on: type: string required: false default: "" + publish: + type: boolean + required: false + default: false secrets: GEMFURY_TOKEN: required: false # Cancel in-flight runs when a newer event arrives for the same logical branch. -# TEMPORARILY DISABLED: we fire several targeted workflow_dispatch runs on the same -# branch in parallel (one per prebuild set) and must not let them cancel each other. -# concurrency: -# group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} -# cancel-in-progress: true +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true env: DEFAULT_PYTHONS: "3.12,3.13,3.14" @@ -196,5 +204,6 @@ jobs: # No `|| default` here: an explicitly-passed empty string must survive so a # caller can disable the override and test against the published sp. serious_python_ref: ${{ inputs.serious_python_ref }} + publish: ${{ inputs.publish || false }} secrets: GEMFURY_TOKEN: ${{ secrets.GEMFURY_TOKEN }} diff --git a/recipes/onnxruntime/meta.yaml b/recipes/onnxruntime/meta.yaml index a8a9d847..7279d65a 100644 --- a/recipes/onnxruntime/meta.yaml +++ b/recipes/onnxruntime/meta.yaml @@ -11,7 +11,7 @@ package: - armeabi-v7a build: - number: 1 + number: 2 script_env: _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' # Consumed by the patch-added PEP 517 shim (_forge/forge_ort_backend.py), diff --git a/recipes/pymongo/meta.yaml b/recipes/pymongo/meta.yaml index 02b04b64..52049bc6 100644 --- a/recipes/pymongo/meta.yaml +++ b/recipes/pymongo/meta.yaml @@ -3,4 +3,4 @@ package: version: "4.17.0" build: - number: 1 + number: 2 diff --git a/setup.sh b/setup.sh index 7da11d8c..701b0e1a 100755 --- a/setup.sh +++ b/setup.sh @@ -34,7 +34,7 @@ if ! command -v uv &> /dev/null; then fi # Pinned flet-dev/python-build release to consume (date-keyed YYYYMMDD, PBS-style). -PYTHON_BUILD_RELEASE="${PYTHON_BUILD_RELEASE:-20260712}" +PYTHON_BUILD_RELEASE="${PYTHON_BUILD_RELEASE:-20260714}" # Resolve a full X.Y.Z from a bare X.Y minor using the pinned release's # manifest.json (downloaded + cached under downloads/). Echoes the full version; From cdb87c054170d84fad9fa46df246ba4da2f4bab1 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 14 Jul 2026 18:19:25 +0200 Subject: [PATCH 49/49] update [skip ci] --- .claude/skills/forge-ci/SKILL.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.claude/skills/forge-ci/SKILL.md b/.claude/skills/forge-ci/SKILL.md index 9f401aa5..50a61ad6 100644 --- a/.claude/skills/forge-ci/SKILL.md +++ b/.claude/skills/forge-ci/SKILL.md @@ -84,20 +84,6 @@ runs on one branch (double dispatch, a late push, a killed script whose last cancels the rest — check `gh run list --branch ` and make sure the survivor has the inputs you meant. -**PUSH BEFORE YOU DISPATCH — the #1 silent-invalid-run trap.** `gh workflow run ---ref ` checks out the **fork's REMOTE branch HEAD**, never your local -commits. Committing locally and dispatching without pushing runs *stale code*, -and the run looks healthy — it just tests the old tree. This burns a full -build+test cycle and gives a wrong red/green. Symptom seen once: local template -had a serious_python git override but CI resolved `serious_python 4.3.0` from -pub.dev (hosted), so every android leg failed on the old empty-x86_64- -sitepackages bug. ALWAYS: `git push fork ` (commits carry `[skip ci]` so -the push won't auto-run), then **verify the dispatched run's commit**: -`gh run view --json headSha` must equal your local `git rev-parse HEAD` -before you trust ANY result. The fork remote is usually `fork` -(ndonkoHenri/mobile-forge); `origin` is the upstream flet-dev/mobile-forge — -dispatch against the fork. - ## Chains: when and how to use `prebuild_recipes` `prebuild_recipes` is a comma-separated list of recipes each job builds