diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d9b0bd5..1a87dd13 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,48 +19,6 @@ permissions: pull-requests: write jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - release: name: Build and release needs: linux @@ -174,12 +132,6 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Build wheels uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4be7148e..6e787f3c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,48 +17,6 @@ env: WASI_SDK_VERSION: 34 jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - test: name: Test needs: linux @@ -104,12 +62,6 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Lint shell: bash run: bash .github/workflows/lint.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e2975f3..172aad92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,16 +34,16 @@ First, make sure you've got all the submodules cloned. git submodule update --init --recursive ``` -Next, install WASI-SDK 33 to `/opt/wasi-sdk` (alternatively, you can specify a +Next, install WASI-SDK 34 to `/opt/wasi-sdk` (alternatively, you can specify a different location and reference it later using the `WASI_SDK_PATH` environment variable). Replace `arm64-linux` with `x86_64-linux`, `arm64-macos`, `x86_64-macos`, `arm64-windows`, or `x86_64-windows` below depending on your architecure and OS, if necessary. ```shell -curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-arm64-linux.tar.gz -tar xf wasi-sdk-33.0-arm64-linux.tar.gz -sudo mv wasi-sdk-33.0-arm64-linux /opt/wasi-sdk +curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-34/wasi-sdk-34.0-arm64-linux.tar.gz +tar xf wasi-sdk-34.0-arm64-linux.tar.gz +sudo mv wasi-sdk-34.0-arm64-linux /opt/wasi-sdk export WASI_SDK_PATH=/opt/wasi-sdk ``` diff --git a/Cargo.lock b/Cargo.lock index c8842db8..1b61fdf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,9 +161,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auditable-serde" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7bf8143dfc3c0258df908843e169b5cc5fcf76c7718bd66135ef4a9cd558c5" +checksum = "d026218ae25ba5c72834245412dd1338f6d270d2c5109ee03a4badec288d4056" dependencies = [ "semver", "serde", @@ -443,15 +443,14 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "component-init-transform" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905e0d94b9dce0ad90bf44b5dedaab3f1a8275964d5012fec8e91656ad91d81e" +source = "git+https://github.com/dicej/component-init?rev=d8bad1bf#d8bad1bfd93390f502a0e2315491ce0b65961499" dependencies = [ "anyhow", "async-trait", "futures", - "wasm-encoder 0.244.0", - "wasm-metadata 0.244.0", - "wasmparser 0.244.0", + "wasm-encoder 0.257.1", + "wasm-metadata 0.257.1", + "wasmparser 0.257.1", ] [[package]] @@ -1132,7 +1131,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "foldhash 0.1.5", - "serde", ] [[package]] @@ -2426,9 +2424,9 @@ dependencies = [ [[package]] name = "spdx" -version = "0.10.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +checksum = "081670c233dfbed55690cc0cd38424e0e24ac1b2673d0b408b3f7b684738dfa9" dependencies = [ "smallvec", ] @@ -3024,6 +3022,16 @@ dependencies = [ "wasmparser 0.254.0", ] +[[package]] +name = "wasm-encoder" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8ad9f0a39050867bda22e6486c316e2e52d20a42154d5bc433934bf738d085" +dependencies = [ + "leb128fmt", + "wasmparser 0.257.1", +] + [[package]] name = "wasm-encoder" version = "0.258.0" @@ -3041,14 +3049,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "auditable-serde", - "flate2", "indexmap", - "serde", - "serde_derive", - "serde_json", - "spdx", - "url", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] @@ -3065,6 +3066,25 @@ dependencies = [ "wasmparser 0.254.0", ] +[[package]] +name = "wasm-metadata" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57460ad9bce753e8a80d47a445cb87c8724ea57f463d9c906a947c41032ce1ac" +dependencies = [ + "anyhow", + "auditable-serde", + "flate2", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "url", + "wasm-encoder 0.257.1", + "wasmparser 0.257.1", +] + [[package]] name = "wasm-metadata" version = "0.258.0" @@ -3087,7 +3107,6 @@ dependencies = [ "hashbrown 0.15.5", "indexmap", "semver", - "serde", ] [[package]] @@ -3103,6 +3122,19 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92fc335fb6d48f46bda1d8b26b69e28320c15ac3272208333833d6e217e2b4a" +dependencies = [ + "bitflags", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + [[package]] name = "wasmparser" version = "0.258.0" diff --git a/Cargo.toml b/Cargo.toml index 7804fb50..f382b326 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,8 @@ pyo3 = { version = "0.28.3", features = [ wasmtime = "48.0.0" wasmtime-wasi = { version = "48.0.0", features = [ "p3" ] } once_cell = "1.20.2" -component-init-transform = "0.2.0" +# TODO: publish a release to crates.io and use it: +component-init-transform = { git = "https://github.com/dicej/component-init", rev = "d8bad1bf" } async-trait = "0.1.83" futures = "0.3.31" tokio = { version = "1.41.0", features = [ diff --git a/build.rs b/build.rs index 5c2e65c6..158cbe3e 100644 --- a/build.rs +++ b/build.rs @@ -38,8 +38,13 @@ const CLANG_EXECUTABLE: &str = "clang"; // TODO: switch to upstream release per // https://github.com/bytecodealliance/componentize-py/issues/215 -const CPYTHON_TARBALL_URL: &str = "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-30"; -const CPYTHON_TARBALL_BASE_DIR: &str = "dicej-cpython-0e13686"; +const CPYTHON_SOURCE_TARBALL_URL: &str = + "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-34"; +const CPYTHON_SOURCE_TARBALL_BASE_DIR: &str = "dicej-cpython-1cf1514"; + +const CPYTHON_BINARY_TARBALL_URL: &str = "https://github.com/dicej/cpython/releases/download/v3.14.0-wasi-sdk-34/cpython-wasi-v3.14.0-wasi-sdk-34.tar.zst"; + +static TARGETS: &[&str] = &["wasip2", "wasip3"]; fn main() -> Result<()> { println!("cargo:rerun-if-changed=build.rs"); @@ -61,13 +66,10 @@ fn main() -> Result<()> { } fn stubs_for_clippy(out_dir: &Path) -> Result<()> { - println!( - "cargo:warning=using stubbed runtime, core library, and adapter for static analysis purposes..." - ); + println!("cargo:warning=using dummy embedded files for static analysis purposes..."); - let files = [ - "libcomponentize_py_runtime_sync.so.zst", - "libcomponentize_py_runtime_async.so.zst", + let libraries = [ + "libcomponentize_py_runtime.so.zst", "libpython3.14.so.zst", "libc.so.zst", "libwasi-emulated-mman.so.zst", @@ -76,10 +78,24 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { "libwasi-emulated-signal.so.zst", "libc++.so.zst", "libc++abi.so.zst", - "wasi_snapshot_preview1.reactor.wasm.zst", ]; - for file in files { + for file in libraries { + for target in TARGETS { + let target_dir = out_dir.join(target); + fs::create_dir_all(&target_dir)?; + + let path = target_dir.join(file); + + if !path.exists() { + Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?.do_finish()?; + } + } + } + + let adapters = ["wasi_snapshot_preview1.reactor.wasm.zst"]; + + for file in adapters { let path = out_dir.join(file); if !path.exists() { @@ -87,20 +103,16 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { } } - let path = out_dir.join("python-lib.tar.zst"); - - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; - } + let tarballs = ["python-lib.tar.zst", "bundled.tar.zst"]; - let path = out_dir.join("bundled.tar.zst"); + for file in tarballs { + let path = out_dir.join(file); - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; + if !path.exists() { + Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) + .into_inner()? + .do_finish()?; + } } Ok(()) @@ -154,7 +166,7 @@ fn find_wasi_sdk(out_dir: &Path) -> Result { url, wasi_sdk.display() ); - fetch_extract(&url, out_dir)?; + fetch_extract_gzip(&url, out_dir)?; Ok(wasi_sdk) } @@ -164,60 +176,56 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> { let wasi_sdk = find_wasi_sdk(out_dir)?; - maybe_make_cpython(&repo_dir, &wasi_sdk)?; + for target in TARGETS { + maybe_make_cpython(target, &repo_dir, &wasi_sdk)?; - let cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + let cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); - make_pyo3_config(&repo_dir)?; + let library_dir = out_dir.join(target); + fs::create_dir_all(&library_dir)?; - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - false, - "libcomponentize_py_runtime_sync.so", - )?; - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - true, - "libcomponentize_py_runtime_async.so", - )?; - - let libraries = [ - "libc.so", - "libwasi-emulated-mman.so", - "libwasi-emulated-process-clocks.so", - "libwasi-emulated-getpid.so", - "libwasi-emulated-signal.so", - ]; + make_pyo3_config(target, &repo_dir)?; - for library in libraries { - compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2"), - library, - out_dir, - true, + make_runtime( + target, + &repo_dir, + &library_dir, + &wasi_sdk, + &cpython_wasi_dir, )?; - } - let libraries = ["libc++.so", "libc++abi.so"]; + let libraries = [ + "libc.so", + "libwasi-emulated-mman.so", + "libwasi-emulated-process-clocks.so", + "libwasi-emulated-getpid.so", + "libwasi-emulated-signal.so", + ]; + + for library in libraries { + compress( + &wasi_sdk.join(format!("share/wasi-sysroot/lib/wasm32-{target}")), + library, + &library_dir, + true, + )?; + } - for library in libraries { - compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2/noeh"), - library, - out_dir, - true, - )?; - } + let libraries = ["libc++.so", "libc++abi.so"]; - compress(&cpython_wasi_dir, "libpython3.14.so", out_dir, true)?; + for library in libraries { + compress( + &wasi_sdk.join(format!("share/wasi-sysroot/lib/wasm32-{target}/noeh")), + library, + &library_dir, + true, + )?; + } - let path = repo_dir.join("cpython/builddir/wasi/install/lib/python3.14"); + compress(&cpython_wasi_dir, "libpython3.14.so", &library_dir, true)?; + } + + let path = repo_dir.join("cpython/builddir/wasip2/install/lib/python3.14"); if path.exists() { let mut builder = Builder::new(Encoder::new( @@ -305,18 +313,29 @@ fn add(builder: &mut Builder, root: &Path, path: &Path) -> Result<() Ok(()) } -fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { +fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { let cpython_dir = repo_dir.join("cpython"); if !cpython_dir.exists() { - let url = &env::var("CPYTHON_TARBALL_URL").unwrap_or_else(|_| CPYTHON_TARBALL_URL.into()); - let base_dir = &env::var_os("CPYTHON_TARBALL_BASE_DIR") - .unwrap_or_else(|| CPYTHON_TARBALL_BASE_DIR.into()); - println!("cargo:warning=downloading CPython source code from {url}"); - fetch_extract(url, repo_dir)?; - fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { + let url = &env::var("CPYTHON_SOURCE_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_SOURCE_TARBALL_URL.into()); + let base_dir = &env::var_os("CPYTHON_SOURCE_TARBALL_BASE_DIR") + .unwrap_or_else(|| CPYTHON_SOURCE_TARBALL_BASE_DIR.into()); + println!("cargo:warning=downloading CPython source code from {url}"); + fetch_extract_gzip(url, repo_dir)?; + fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + } else { + let url = &env::var("CPYTHON_BINARY_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); + println!("cargo:warning=downloading CPython build from {url}"); + let dir = cpython_dir.join("builddir"); + fs::create_dir_all(&dir)?; + fetch_extract_zstd(url, &dir)?; + return Ok(()); + } } - let cpython_wasi_dir = cpython_dir.join("builddir/wasi"); + let cpython_wasi_dir = cpython_dir.join("builddir").join(target); if !cpython_wasi_dir.join("libpython3.14.so").exists() { fs::create_dir_all(&cpython_wasi_dir)?; if !cpython_wasi_dir.join("libpython3.14.a").exists() { @@ -338,9 +357,9 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { } let lib_install_dir = cpython_wasi_dir.join("deps"); - build_zlib(wasi_sdk, &lib_install_dir)?; + build_zlib(target, wasi_sdk, &lib_install_dir)?; - build_sqlite(wasi_sdk, &lib_install_dir)?; + build_sqlite(target, wasi_sdk, &lib_install_dir)?; let config_guess = run(Command::new("../../config.guess").current_dir(&cpython_wasi_dir))?; @@ -358,12 +377,12 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { ) .env( "CFLAGS", - format!("--target=wasm32-wasip2 -fPIC -I{dir}/deps/include"), + format!("--target=wasm32-{target} -fPIC -I{dir}/deps/include"), ) .env("WASI_SDK_PATH", wasi_sdk) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 -L{dir}/deps/lib"), + format!("--target=wasm32-{target} -L{dir}/deps/lib"), ) .current_dir(&cpython_wasi_dir) .args([ @@ -388,7 +407,7 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { // Link libpython3.14.so - now includes libsqlite3.a run(Command::new(wasi_sdk.join("bin/clang")) - .arg("--target=wasm32-wasip2") + .arg(format!("--target=wasm32-{target}")) .arg("-shared") .arg("-o") .arg(cpython_wasi_dir.join("libpython3.14.so")) @@ -475,9 +494,9 @@ fn run(command: &mut Command) -> Result> { } } -fn make_pyo3_config(repo_dir: &Path) -> Result<()> { +fn make_pyo3_config(target: &str, repo_dir: &Path) -> Result<()> { let out_dir = env::var("OUT_DIR")?; - let mut cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + let mut cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); let mut cygpath = Command::new("cygpath"); cygpath.arg("-w").arg(&cpython_wasi_dir); if let Ok(output) = cygpath.output() { @@ -497,7 +516,10 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { "lib_dir={}", cpython_wasi_dir.to_str().unwrap() )?; - fs::write(Path::new(&out_dir).join("pyo3-config.txt"), pyo3_config)?; + fs::write( + Path::new(&out_dir).join(target).join("pyo3-config.txt"), + pyo3_config, + )?; println!("cargo:rerun-if-changed=pyo3-config.txt"); @@ -505,37 +527,26 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { } fn make_runtime( + target: &str, repo_dir: &Path, out_dir: &Path, wasi_sdk: &Path, cpython_wasi_dir: &Path, - async_: bool, - name: &str, ) -> Result<()> { let mut cmd = Command::new("cargo"); cmd.current_dir(repo_dir.join("runtime")) .arg("build") - .arg("--target=wasm32-wasip2"); + .arg(format!("--target=wasm32-{target}")); if !DEBUG_RUNTIME { cmd.arg("--release"); } - if async_ { + if target == "wasip3" { cmd.arg("--features=async"); } - for (key, _) in env::vars_os() { - if key - .to_str() - .map(|key| key.starts_with("RUST") || key.starts_with("CARGO")) - .unwrap_or(false) - { - cmd.env_remove(&key); - } - } - - let target = if async_ { "async" } else { "sync" }; + cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); let clang = wasi_sdk.join(format!("bin/{CLANG_EXECUTABLE}")); cmd.env( @@ -543,6 +554,7 @@ fn make_runtime( format!( "--cfg pyo3_disable_reference_pool \ -Clink-args=-Wl,--skip-wit-component \ + -Clink-args=-Wl,--export-if-defined=__wasm_library_tls_info \ -Clink-args=-shared \ -Clink-args=-L{} \ -Clink-args=-lpython3.14 \ @@ -550,7 +562,10 @@ fn make_runtime( cpython_wasi_dir.to_str().unwrap() ), ) - .env("CARGO_TARGET_WASM32_WASIP2_LINKER", clang) + .env( + format!("CARGO_TARGET_WASM32_{}_LINKER", target.to_uppercase()), + clang, + ) .env("CARGO_TARGET_DIR", out_dir.join(target)) .env("PYO3_CONFIG_FILE", out_dir.join("pyo3-config.txt")); @@ -562,10 +577,11 @@ fn make_runtime( let build = if DEBUG_RUNTIME { "debug" } else { "release" }; let path = out_dir.join(target).join(format!( - "wasm32-wasip2/{build}/componentize_py_runtime.wasm" + "wasm32-{target}/{build}/componentize_py_runtime.wasm" )); if path.exists() { + let name = "libcomponentize_py_runtime.so"; fs::copy(&path, out_dir.join(name))?; compress(out_dir, name, out_dir, false)?; } else { @@ -575,7 +591,7 @@ fn make_runtime( Ok(()) } -fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { +fn fetch_extract_gzip(url: &str, out_dir: &Path) -> Result<()> { let response = reqwest::blocking::get(url)?; let decoder = flate2::read::GzDecoder::new(response); let mut archive = tar::Archive::new(decoder); @@ -583,7 +599,15 @@ fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { Ok(()) } -fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { +fn fetch_extract_zstd(url: &str, out_dir: &Path) -> Result<()> { + let response = reqwest::blocking::get(url)?; + let decoder = zstd::Decoder::new(response)?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(out_dir)?; + Ok(()) +} + +fn add_compile_envs(target: &str, wasi_sdk: &Path, command: &mut Command) { let sysroot = wasi_sdk.join("share/wasi-sysroot"); let sysroot = sysroot.to_string_lossy(); command @@ -592,17 +616,17 @@ fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env( "CFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -I{sysroot}/include/wasm32-wasip2 -D_WASI_EMULATED_SIGNAL -fPIC"), + format!("--target=wasm32-{target} --sysroot={sysroot} -I{sysroot}/include/wasm32-{target} -D_WASI_EMULATED_SIGNAL -fPIC"), ) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") + format!("--target=wasm32-{target} --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") ); } -fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_zlib(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); - fetch_extract( + fetch_extract_gzip( "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz", &out_dir, )?; @@ -613,7 +637,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", install_dir.display()))?; let mut configure = Command::new("./configure"); - add_compile_envs(wasi_sdk, &mut configure); + add_compile_envs(target, wasi_sdk, &mut configure); configure .current_dir(&src_dir) .arg("--static") @@ -631,7 +655,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", clang_dir.display()))?; let mut make = Command::new("make"); - add_compile_envs(wasi_sdk, &mut make); + add_compile_envs(target, wasi_sdk, &mut make); make.current_dir(src_dir) .arg(format!("AR={ar_dir}")) .arg("ARFLAGS=rcs") @@ -650,7 +674,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { /// - SQLITE_OMIT_WAL: WAL requires mmap which isn't available in WASI preview1 /// - SQLITE_OMIT_LOAD_EXTENSION: No dlopen in WASI /// - SQLITE_THREADSAFE=0: Single-threaded for WASM -fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_sqlite(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); // Check if already built @@ -661,12 +685,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { println!("cargo:warning=Building SQLite {SQLITE_VERSION} for WASI..."); - // Download SQLite amalgamation - let url = format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); - fetch_extract(&url, &out_dir)?; - let src_dir = out_dir.join(format!("sqlite-autoconf-{SQLITE_VERSION}")); + if !src_dir.exists() { + // Download SQLite amalgamation + let url = + format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); + fetch_extract_gzip(&url, &out_dir)?; + } + + let build_dir = src_dir.join(target); + fs::create_dir_all(&build_dir)?; + // Ensure install directories exist fs::create_dir_all(install_dir.join("lib"))?; fs::create_dir_all(install_dir.join("include"))?; @@ -687,9 +717,9 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Note: Don't set SQLITE_THREADSAFE here - let --disable-threadsafe handle it // to avoid macro redefinition warnings let sqlite_cflags = format!( - "--target=wasm32-wasip2 \ + "--target=wasm32-{target} \ --sysroot={sysroot_str} \ - -I{sysroot_str}/include/wasm32-wasip2 \ + -I{sysroot_str}/include/wasm32-{target} \ -D_WASI_EMULATED_SIGNAL \ -D_WASI_EMULATED_PROCESS_CLOCKS \ -fPIC \ @@ -702,18 +732,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { ); // Configure SQLite - let mut configure = Command::new("./configure"); + let mut configure = Command::new("../configure"); configure - .current_dir(&src_dir) + .current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env("CFLAGS", &sqlite_cflags) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot_str} -L{sysroot_str}/lib",), + format!("--target=wasm32-{target} --sysroot={sysroot_str} -L{sysroot_str}/lib",), ) - .arg("--host=wasm32-wasip2") + .arg(format!("--host=wasm32-{target}")) .arg(format!("--prefix={install_dir_str}")) .arg("--disable-shared") .arg("--enable-static") @@ -725,7 +755,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Build only the static library (not the shell, which fails to link on WASI) let mut make = Command::new("make"); - make.current_dir(&src_dir) + make.current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) @@ -738,7 +768,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Manual install since we didn't build everything // Copy the library fs::copy( - src_dir.join("libsqlite3.a"), + build_dir.join("libsqlite3.a"), install_dir.join("lib/libsqlite3.a"), )?; // Copy the headers diff --git a/bundled/componentize_py_async_support/__init__.py b/bundled/componentize_py_async_support/__init__.py index d4025667..6095b6f9 100644 --- a/bundled/componentize_py_async_support/__init__.py +++ b/bundled/componentize_py_async_support/__init__.py @@ -10,6 +10,7 @@ import componentize_py_runtime import subprocess +from threading import local from os import PathLike from socket import AddressFamily, AddressInfo, SocketKind, socket from concurrent.futures import Executor @@ -67,7 +68,7 @@ class _Status: async def _noop() -> None: pass - + class _Loop(asyncio.AbstractEventLoop): def __init__(self) -> None: self.running: bool = False @@ -80,13 +81,13 @@ def poll(self, future_state: _FutureState) -> None: for handle in handles: if not handle._cancelled: handle._run() - + if self.exception is not None: raise self.exception if len(handles) == 0 and len(future_state.handles) == 0: return - + def get_debug(self) -> bool: return False @@ -187,7 +188,7 @@ async def getnameinfo(self, sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int = 0) -> tuple[str, str]: raise NotImplementedError - + async def create_connection( self, protocol_factory: Callable[[], _ProtocolT], @@ -207,7 +208,7 @@ async def create_connection( interleave: int | None = None, ) -> tuple[Transport, _ProtocolT]: raise NotImplementedError - + async def create_server( self, protocol_factory: Callable[[], BaseProtocol], @@ -420,12 +421,9 @@ def default_exception_handler(self, context: dict[str, Any]) -> None: def set_debug(self, enabled: bool) -> None: raise NotImplementedError - + _future_state: ContextVar[_FutureState] = ContextVar("_future_state") _loop = _Loop() -asyncio.set_event_loop(_loop) -_loop.running = True -asyncio.events._set_running_loop(_loop) def _set_future_state(future_state: _FutureState) -> None: global _future_state @@ -448,12 +446,17 @@ async def _return_result(export_index: int, borrows: int, coroutine: Any) -> Non assert _future_state.get().pending_count > 0 _future_state.get().pending_count -= 1 +_thread_local = local() + def first_poll(export_index: int, borrows: int, coroutine: Any) -> int: """Internal function called by generated code for exported functions. This is not meant to be called by application code. """ + asyncio.set_event_loop(_loop) + _loop.running = True + asyncio.events._set_running_loop(_loop) context = Context() future_state = _FutureState(None, {}, [], 1) context.run(_set_future_state, future_state) @@ -468,12 +471,12 @@ def _poll(future_state: _FutureState) -> int: if future_state.pending_count == 0: if future_state.waitable_set is not None: componentize_py_runtime.waitable_set_drop(future_state.waitable_set) - + return _CallbackCode.EXIT else: waitable_set = future_state.waitable_set assert waitable_set is not None - componentize_py_runtime.context_set(future_state) + _thread_local.state = future_state return _CallbackCode.WAIT | (waitable_set << 4) def callback(event0: int, event1: int, event2: int) -> int: @@ -482,9 +485,9 @@ def callback(event0: int, event1: int, event2: int) -> int: This is not meant to be called by application code. """ - future_state = componentize_py_runtime.context_get() - componentize_py_runtime.context_set(None) - + future_state = _thread_local.state + _thread_local.state = None + match event0: case _Event.NONE: pass @@ -509,7 +512,7 @@ def callback(event0: int, event1: int, event2: int) -> int: raise NotImplementedError return _poll(future_state) - + async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """Internal function called by generated code for imported functions. @@ -518,7 +521,7 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """ global _loop global _future_state - + if isinstance(result, Ok): return result.value else: @@ -526,11 +529,11 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: waitable, promise = result.value future = _loop.create_future() future_state.futures[waitable] = future - + if future_state.waitable_set is None: future_state.waitable_set = componentize_py_runtime.waitable_set_new() componentize_py_runtime.waitable_join(waitable, future_state.waitable_set) - + return cast(T, componentize_py_runtime.promise_get_result(await future, promise)) async def _wrap_spawned(coroutine: Any) -> None: @@ -566,5 +569,5 @@ def spawn(coroutine: Any) -> None: global _future_state _future_state.get().pending_count += 1 - + asyncio.create_task(_wrap_spawned(coroutine)) diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e2d8576c..7f8469f4 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -13,7 +13,7 @@ anyhow = "1.0.91" once_cell = "1.20.2" pyo3 = { version="0.28.3", features = ["abi3-py312", "num-bigint"] } num-bigint = "0.4.6" -wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc"] } +wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc", "async"] } wit-dylib-ffi = { git = "https://github.com/bytecodealliance/wasm-tools", tag = "v1.258.0", features = ["async-raw"] } [features] diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4eb7c4e3..9d43e27c 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -22,7 +22,7 @@ use { pyo3::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, exceptions::PyAssertionError, - intern, + ffi, intern, types::{ PyAnyMethods, PyBool, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyMappingMethods, PyModule, PyModuleMethods, PyString, @@ -31,9 +31,10 @@ use { }, std::{ alloc::{self, Layout}, + cell::Cell, iter, marker::PhantomData, - mem, slice, str, + mem, ptr, slice, str, sync::Once, }, wit_dylib_ffi::{ @@ -53,6 +54,8 @@ mod bindings { export!(MyExports); } +const CALLBACK_CODE_EXIT: u32 = 0; + static WIT: OnceCell = OnceCell::new(); static STUB_WASI: OnceCell = OnceCell::new(); static EXPORTS: OnceCell> = OnceCell::new(); @@ -1167,7 +1170,7 @@ fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), St struct MyExports; impl Guest for MyExports { - fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { + async fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { let result = do_init(app_name, symbols, stub_wasi); // This tells the WASI Preview 1 component adapter to reset its state. @@ -1201,7 +1204,9 @@ struct MyInterpreter; impl MyInterpreter { fn export_call_(func: ExportFunction, cx: &mut MyCall<'_>, async_: bool) -> u32 { - Python::attach(|py| { + let gil_state = unsafe { ffi::PyGILState_Ensure() }; + + let result = Python::attach(|py| { if !*STUB_WASI.get().unwrap() { static ONCE: Once = Once::new(); ONCE.call_once(|| { @@ -1299,12 +1304,24 @@ impl MyInterpreter { release_borrows(py, mem::take(&mut cx.borrows)); - 0 + CALLBACK_CODE_EXIT } - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state) }; + } else { + THREAD_STATE.with(|v| { + v.set((Some(gil_state), unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } } +std::thread_local!(static THREAD_STATE: Cell<(Option, *mut ffi::PyThreadState)> = const { Cell::new((None, ptr::null_mut())) } ); + impl Interpreter for MyInterpreter { type CallCx<'a> = MyCall<'a>; @@ -1335,7 +1352,13 @@ impl Interpreter for MyInterpreter { fn export_async_callback(event0: u32, event1: u32, event2: u32) -> u32 { #[cfg(feature = "async")] { - Python::attach(|py| { + let gil_state = THREAD_STATE.with(|v| { + let (gil_state, thread_state) = v.get(); + unsafe { ffi::PyEval_RestoreThread(thread_state) }; + gil_state + }); + + let result = Python::attach(|py| { async_::CALLBACK .get() .unwrap() @@ -1343,7 +1366,17 @@ impl Interpreter for MyInterpreter { .unwrap() .extract(py) .unwrap() - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state.unwrap()) }; + } else { + THREAD_STATE.with(|v| { + v.set((gil_state, unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } #[cfg(not(feature = "async"))] { @@ -1427,7 +1460,9 @@ impl Drop for MyCall<'_> { fn drop(&mut self) { for &(ptr, layout) in &self.deferred_deallocations { unsafe { - alloc::dealloc(ptr, layout); + if layout.size() > 0 { + alloc::dealloc(ptr, layout); + } } } } @@ -2081,7 +2116,9 @@ impl Call for MyCall<'_> { .to_owned() .into_any() .unbind(); - alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + if len > 0 { + alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + } value })); true @@ -2209,34 +2246,3 @@ fn exported_resource_to_canon( } wit_dylib_ffi::export!(MyInterpreter); - -// As of this writing, recent Rust `nightly` builds include a version of the -// `libc` crate that expects `wasi-libc` to define the following global -// variables, but `wasi-libc` defines them as preprocessor constants which -// aren't visible at link time, so we need to define them somewhere. Ideally, -// we should fix this upstream, but for now we work around it: - -#[unsafe(no_mangle)] -static _CLOCK_PROCESS_CPUTIME_ID: u8 = 2; -#[unsafe(no_mangle)] -static _CLOCK_THREAD_CPUTIME_ID: u8 = 3; - -// Traditionally, `wit-bindgen` would provide a `cabi_realloc` implementation, -// but recent versions use a weak symbol trick to avoid conflicts when more than -// one `wit-bindgen` version is used, and that trick does not currently play -// nice with how we build this library. So for now, we just define it ourselves -// here: -/// # Safety -/// TODO -#[unsafe(export_name = "cabi_realloc")] -pub unsafe extern "C" fn cabi_realloc( - old_ptr: *mut u8, - old_len: usize, - align: usize, - new_size: usize, -) -> *mut u8 { - assert!(old_ptr.is_null()); - assert!(old_len == 0); - - unsafe { alloc::alloc(Layout::from_size_align(new_size, align).unwrap()) } -} diff --git a/src/lib.rs b/src/lib.rs index 6ccb33df..53e06e0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,7 @@ mod test; mod util; const DEBUG_PYTHON_BINDINGS: bool = false; +const DEBUG_PRE_INIT: bool = false; /// The default name of the Python module containing code generated from the /// specified WIT world. This may be overriden programatically or via the CLI @@ -76,7 +77,14 @@ impl WasiView for Ctx { } } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Target { + Wasip2, + Wasip3, +} + pub struct Library { + target: Target, name: String, module: Vec, dl_openable: bool, @@ -555,13 +563,49 @@ impl ComponentGenerator<'_> { // Extract relevant metadata from the `Resolve` into a `Summary` instance, // which we'll use to generate Wasm- and Python-level bindings. + // Determine whether to use the WASIp2 or WASIp3 target based on whether + // the world uses any async features. + // + // TODO: Allow the user to explicitly specify the target instead of + // inferring it here, e.g. if they want to use WASIp3 despite the target + // world not using any async features. + // + // TODO #2: Creating a temporary `Summary` is a heavyweight way to + // determine whether the world uses async features, especially since we + // will create the real one down below, but otherwise we'd have an + // ordering problem because we need to know the target before we call + // `wit_dylib::create_with_metadata` (which produces the metadata we'll + // need to create the real `Summary`), and we can't call that until we + // know the target. We should be able to extract the code that + // `Summary::try_new` uses to check for async features and use it + // without the rest of the things `Summary::try_new` does. + let need_async = Summary::try_new( + &resolve, + &iter::once((world, Naming::from_full(self.full_names))).collect(), + &import_interface_names, + &export_interface_names, + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + )? + .need_async(); + + let target = if need_async { + Target::Wasip3 + } else { + Target::Wasip2 + }; + let (mut bindings, metadata) = wit_dylib::create_with_metadata( &resolve, world, Some(&mut DylibOpts { - stack_pointer: wit_dylib::StackPointer::Global, interpreter: Some("libcomponentize_py_runtime.so".into()), async_: Default::default(), + stack_pointer: match target { + Target::Wasip2 => wit_dylib::StackPointer::Global, + Target::Wasip3 => wit_dylib::StackPointer::TaskContext, + }, }), ); @@ -627,29 +671,18 @@ impl ComponentGenerator<'_> { &stream_and_future_indexes, )?; - let need_async = summary.need_async(); - - // Now that we know whether to use the sync or async version of - // `libcomponentize_py_runtime.so`, update `libraries` accordingly. + // Now that we know which target to use, update `libraries` accordingly. // - // Note that we have two separate versions because older runtimes don't - // understand the new async ABI, so we only use the async version if it's - // actually needed. + // Note that we must only use libraries which match the target because + // the targets have mutually incompatible ABIs, plus users may wish to + // target runtimes which do not support WASIp3 or async features. let mut libraries = libraries .into_iter() - .filter_map(|library| match (need_async, library.name.as_str()) { - (true, "libcomponentize_py_runtime_sync.so") - | (false, "libcomponentize_py_runtime_async.so") => None, - (true, "libcomponentize_py_runtime_async.so") - | (false, "libcomponentize_py_runtime_sync.so") => Some(Library { - name: "libcomponentize_py_runtime.so".into(), - ..library - }), - _ => Some(library), - }) + .filter(|library| library.target == target) .collect::>(); libraries.push(Library { + target, name: "libcomponentize_py_bindings.so".into(), module: bindings, dl_openable: false, @@ -673,10 +706,14 @@ impl ComponentGenerator<'_> { let stderr = MemoryOutputPipe::new(10000); let mut wasi = WasiCtxBuilder::new(); - wasi.stdin(MemoryInputPipe::new(Bytes::new())) - .stdout(stdout.clone()) - .stderr(stderr.clone()) - .env("PYTHONUNBUFFERED", "1") + if DEBUG_PRE_INIT { + wasi.inherit_stdio(); + } else { + wasi.stdin(MemoryInputPipe::new(Bytes::new())) + .stdout(stdout.clone()) + .stderr(stderr.clone()); + } + wasi.env("PYTHONUNBUFFERED", "1") .env("PYTHONHOME", "/python") .preopened_dir( embedded_python_standard_lib.path(), @@ -849,10 +886,16 @@ impl ComponentGenerator<'_> { let instance = pre.instance_pre.instantiate_async(&mut store).await?; let guest = pre.indices.interface0.load(&mut store, &instance)?; - guest - .call_init(&mut store, &app_name, &symbols, stub_wasi) - .await? - .map_err(|e| anyhow!("{e}"))?; + store + .run_concurrent(async |store| { + guest + .call_init(store, app_name, symbols, stub_wasi) + .await? + .map_err(|e| anyhow!("{e}"))?; + + anyhow::Ok(()) + }) + .await??; Ok(Box::new(MyInvoker { store, instance }) as Box) } @@ -1038,6 +1081,7 @@ fn add_wasi_and_stubs( linker: &mut Linker, ) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; enum Stub<'a> { Function(&'a String, &'a FunctionKind), @@ -1093,10 +1137,10 @@ fn add_wasi_and_stubs( for (interface_name, stubs) in stubs { if let Some(interface_name) = interface_name { // Note that we do _not_ stub interfaces which appear to be part of - // WASIp2 since those should be provided by the - // `wasmtime_wasi::add_to_linker_async` call above, and adding stubs - // to those same interfaces would just cause trouble. - if !is_wasip2_cli(&interface_name) + // WASIp2/p3 since those should be provided by the + // `add_to_linker{_async}` calls above, and adding stubs to those + // same interfaces would just cause trouble. + if !is_wasi_cli(&interface_name) && let Ok(mut instance) = linker.instance(&interface_name) { for stub in stubs { @@ -1178,12 +1222,12 @@ fn add_wasi_and_stubs( Ok(()) } -fn is_wasip2_cli(interface_name: &str) -> bool { +fn is_wasi_cli(interface_name: &str) -> bool { (interface_name.starts_with("wasi:cli/") || interface_name.starts_with("wasi:clocks/") || interface_name.starts_with("wasi:random/") || interface_name.starts_with("wasi:io/") || interface_name.starts_with("wasi:filesystem/") || interface_name.starts_with("wasi:sockets/")) - && interface_name.contains("@0.2.") + && (interface_name.contains("@0.2.") || interface_name.contains("@0.3.")) } diff --git a/src/link.rs b/src/link.rs index 86793864..3425e8ad 100644 --- a/src/link.rs +++ b/src/link.rs @@ -9,6 +9,7 @@ pub fn link_libraries(libraries: &[Library]) -> Result> { linker.use_built_in_libdl(true).encoder().validate(true); for Library { + target: _, name, module, dl_openable, diff --git a/src/prelink.rs b/src/prelink.rs index 3aac57af..a89d6e1b 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -14,7 +14,7 @@ use tar::Archive; use tempfile::TempDir; use zstd::Decoder; -use crate::{ComponentizePyConfig, ConfigContext, Library, RawComponentizePyConfig}; +use crate::{ComponentizePyConfig, ConfigContext, Library, RawComponentizePyConfig, Target}; static NATIVE_EXTENSION_SUFFIXES: &[&str] = &[".cpython-314-wasm32-wasi.so", ".abi3.so"]; @@ -60,82 +60,164 @@ pub fn embedded_helper_utils() -> Result { pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result> { let mut libraries = vec![ Library { - name: "libcomponentize_py_runtime_sync.so".into(), + target: Target::Wasip2, + name: "libcomponentize_py_runtime.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libcomponentize_py_runtime_sync.so.zst" + "/wasip2/libcomponentize_py_runtime.so.zst" ))))?, dl_openable: false, }, Library { - name: "libcomponentize_py_runtime_async.so".into(), + target: Target::Wasip3, + name: "libcomponentize_py_runtime.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libcomponentize_py_runtime_async.so.zst" + "/wasip3/libcomponentize_py_runtime.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libpython3.14.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libpython3.14.so.zst" + "/wasip2/libpython3.14.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, + name: "libpython3.14.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libpython3.14.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, name: "libc.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc.so.zst" + "/wasip3/libc.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libwasi-emulated-mman.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-mman.so.zst" + "/wasip2/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-process-clocks.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, name: "libwasi-emulated-process-clocks.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-process-clocks.so.zst" + "/wasip3/libwasi-emulated-process-clocks.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libwasi-emulated-getpid.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-getpid.so.zst" + "/wasip2/libwasi-emulated-getpid.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-getpid.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-signal.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, name: "libwasi-emulated-signal.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-signal.so.zst" + "/wasip3/libwasi-emulated-signal.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libc++.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc++.so.zst" + "/wasip2/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc++abi.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, name: "libc++abi.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc++abi.so.zst" + "/wasip3/libc++abi.so.zst" ))))?, dl_openable: false, }, @@ -152,6 +234,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result) -> Result) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; componentize_py::test::echoes::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) } diff --git a/src/test/python_source/app.py b/src/test/python_source/app.py index 7eded4e6..e3edfbfd 100644 --- a/src/test/python_source/app.py +++ b/src/test/python_source/app.py @@ -229,7 +229,7 @@ def test_resource_alias(self, things: List[imports.resource_alias1.Thing]) -> Li def add(self, a: imports.resource_floats.Float, b: imports.resource_floats.Float) -> imports.resource_floats.Float: return imports.resource_floats.Float(a.get() + b.get() + 5) - def read_file(self, path: str) -> bytes: + async def read_file(self, path: str) -> bytes: try: with open(file=path, mode="rb") as f: return f.read() diff --git a/src/test/tests.rs b/src/test/tests.rs index 8dd48bd3..06e2c685 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -119,6 +119,7 @@ impl super::Host for Host { fn add_to_linker(linker: &mut Linker) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; Tests::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; foo_sdk::FooWorldUnion::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) @@ -977,14 +978,18 @@ fn filesystem() -> Result<()> { TESTER.test_with_wasi::(wasi, |world, store, runtime| { runtime.block_on(async { - let value = world - .call_read_file(store, filename) - .await? - .map_err(|s| anyhow!("{s}"))?; + store + .run_concurrent(async |store| { + let value = world + .call_read_file(store, filename.to_string()) + .await? + .map_err(|s| anyhow!("{s}"))?; - assert_eq!(&value, message); + assert_eq!(&value, message); - Ok(()) + Ok(()) + }) + .await? }) }) } diff --git a/src/test/wit/tests.wit b/src/test/wit/tests.wit index c645cfac..6e610cee 100644 --- a/src/test/wit/tests.wit +++ b/src/test/wit/tests.wit @@ -213,7 +213,7 @@ world tests { export add: func(a: borrow, b: borrow) -> own; - export read-file: func(path: string) -> result, string>; + export read-file: async func(path: string) -> result, string>; export test-refcounts: func(); diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index b58a191c..9d7889db 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -1,7 +1,7 @@ #![deny(warnings)] use { - anyhow::{Result, anyhow}, + anyhow::{anyhow, Result}, proptest::{ strategy::{Just, Strategy, ValueTree}, test_runner::{Config, TestRng, TestRunner}, @@ -60,10 +60,28 @@ enum Type { }, Tuple(Vec), List(Box), + Map(Box, Box), +} + +fn any_key_type() -> impl Strategy { + (0..11).prop_flat_map(move |index| match index { + 0 => Just(Type::Bool).boxed(), + 1 => Just(Type::U8).boxed(), + 2 => Just(Type::S8).boxed(), + 3 => Just(Type::U16).boxed(), + 4 => Just(Type::S16).boxed(), + 5 => Just(Type::U32).boxed(), + 6 => Just(Type::S32).boxed(), + 7 => Just(Type::U64).boxed(), + 8 => Just(Type::S64).boxed(), + 9 => Just(Type::Char).boxed(), + 10 => Just(Type::String).boxed(), + _ => unreachable!(), + }) } fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy { - (0..21).prop_flat_map(move |index| match index { + (0..22).prop_flat_map(move |index| match index { 0 => Just(Type::Bool).boxed(), 1 => Just(Type::U8).boxed(), 2 => Just(Type::S8).boxed(), @@ -142,6 +160,9 @@ fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy any_type(max_size, next_id.clone()) .prop_map(|ty| Type::List(Box::new(ty))) .boxed(), + 21 => (any_key_type(), any_type(max_size, next_id.clone())) + .prop_map(|(k, v)| Type::Map(Box::new(k), Box::new(v))) + .boxed(), _ => unreachable!(), }) } @@ -271,6 +292,9 @@ fn wit_type_name(wit: &mut String, ty: &Type) -> String { Type::List(ty) => { format!("list<{}>", wit_type_name(wit, ty)) } + Type::Map(k, v) => { + format!("map<{}, {}>", wit_type_name(wit, k), wit_type_name(wit, v)) + } } } @@ -326,6 +350,9 @@ fn rust_type_name(ty: &Type) -> String { Type::List(ty) => { format!("Vec<{}>", rust_type_name(ty)) } + Type::Map(k, v) => { + format!("HashMap<{}, {}>", rust_type_name(k), rust_type_name(v)) + } } } @@ -418,6 +445,12 @@ fn equality(a: &str, b: &str, ty: &Type) -> String { "{a}.len() == {b}.len() && {a}.iter().zip({b}.iter()).all(|(a, b)| {})", equality("a", "b", ty) ), + Type::Map(k, v) => format!( + "{a}.len() == {b}.len() && {a}.iter().collect::>().into_iter()\ + .zip({b}.iter().collect::>().into_iter()).all(|((ak, av), (bk, bv))| {} && {})", + equality("ak", "bk", k), + equality("av", "bv", v) + ), } } @@ -547,6 +580,13 @@ fn strategy(ty: &Type, max_list_size: usize) -> String { strategy(ty, max_list_size / 2) ) } + Type::Map(k, v) => { + format!( + "proptest::collection::hash_map({}, {}, 0..{max_list_size}.max(1))", + strategy(k, max_list_size / 2), + strategy(v, max_list_size / 2) + ) + } } } @@ -814,6 +854,7 @@ use {{ component::{{Instance, InstancePre, Linker, TypedFunc, HasSelf}}, Store, }}, + std::collections::{{HashMap, BTreeMap}}, }}; wasmtime::component::bindgen!({{ @@ -838,6 +879,7 @@ impl super::Host for Host {{ fn add_to_linker(linker: &mut Linker) -> Result<()> {{ wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; {PREFIX}::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) }} diff --git a/wit/init.wit b/wit/init.wit index 2f5de1ae..919ba99e 100644 --- a/wit/init.wit +++ b/wit/init.wit @@ -98,6 +98,6 @@ world init { results: list, } - init: func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; + init: async func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; } }