add async and WASIp3 support

This adds basic support for async exports, async imports, streams, and futures.
See the new `http-p3` example for a practical demonstration.

This does not include support for cancelling subtasks, reads, or writes.  That
shouldn't be difficult to add, but can wait until a later PR.

I've also used this opportunity to update the Rust edition to 2024, update some
deps, and reflow some comments.

Signed-off-by: Joel Dice <[email protected]>

Author: dicej

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.18.0"
+version = "0.19.0"
 edition = "2024"
 exclude = ["cpython"]
 
@@ -14,23 +14,24 @@ clap = { version = "4.5.20", features = ["derive"] }
 tar = "0.4.42"
 tempfile = "3.13.0"
 zstd = "0.13.2"
-# TODO: switch to wasm-tools 1.241.0 when available
-wasm-encoder = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "b1d8ff59" }
-wit-dylib = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "b1d8ff59" }
-wit-parser = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "b1d8ff59" }
-wit-component = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "b1d8ff59" }
-wasmparser = { git = "https://github.com/bytecodealliance/wasm-tools", rev = "b1d8ff59" }
+# TODO: switch to wasm-tools release once https://github.com/bytecodealliance/wasm-tools/pull/2367 has been merged and released
+wasm-encoder = { git = "https://github.com/dicej/wasm-tools", rev = "416c5da7" }
+wit-dylib = { git = "https://github.com/dicej/wasm-tools", rev = "416c5da7" }
+wit-parser = { git = "https://github.com/dicej/wasm-tools", rev = "416c5da7" }
+wit-component = { git = "https://github.com/dicej/wasm-tools", rev = "416c5da7" }
+wasmparser = { git = "https://github.com/dicej/wasm-tools", rev = "416c5da7" }
 indexmap = "2.6.0"
 bincode = "1.3.3"
 heck = "0.5.0"
 pyo3 = { version = "0.26.0", features = [
     "abi3-py39",
     "extension-module",
 ], optional = true }
-wasmtime = { version = "37.0.2", features = [ "component-model-async" ] }
-wasmtime-wasi = "37.0.2"
+# TODO: switch to wasmtime 39.0.x when available
+wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", rev = "efd56f68", features = [ "component-model-async" ] }
+wasmtime-wasi = { git = "https://github.com/bytecodealliance/wasmtime", rev = "efd56f68", features = [ "p3" ] }
 once_cell = "1.20.2"
-component-init-transform = { git = "https://github.com/dicej/component-init", rev = "3b9680fe" }
+component-init-transform = { git = "https://github.com/dicej/component-init", rev = "2d965957" }
 async-trait = "0.1.83"
 futures = "0.3.31"
 tokio = { version = "1.41.0", features = [
@@ -82,3 +83,4 @@ flate2 = "1.1.1"
 
 [workspace]
 members = ["runtime", "test-generator"]
+

Cargo.toml

@@ -52,7 +52,8 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> {
     );
 
     let files = [
-        "libcomponentize_py_runtime.so.zst",
+        "libcomponentize_py_runtime_sync.so.zst",
+        "libcomponentize_py_runtime_async.so.zst",
         "libpython3.14.so.zst",
         "libc.so.zst",
         "libwasi-emulated-mman.so.zst",
@@ -103,62 +104,20 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> {
 
     make_pyo3_config(&repo_dir)?;
 
-    let mut cmd = Command::new("rustup");
-    cmd.current_dir("runtime")
-        .arg("run")
-        .arg("nightly")
-        .arg("cargo")
-        .arg("build")
-        .arg("-Z")
-        .arg("build-std=panic_abort,std")
-        .arg("--release")
-        .arg("--target=wasm32-wasip1");
-
-    for (key, _) in env::vars_os() {
-        if key
-            .to_str()
-            .map(|key| key.starts_with("RUST") || key.starts_with("CARGO"))
-            .unwrap_or(false)
-        {
-            cmd.env_remove(&key);
-        }
-    }
-
-    cmd.env(
-        "RUSTFLAGS",
-        "-C relocation-model=pic --cfg pyo3_disable_reference_pool",
-    )
-    .env("CARGO_TARGET_DIR", out_dir)
-    .env("PYO3_CONFIG_FILE", out_dir.join("pyo3-config.txt"));
-
-    let status = cmd.status()?;
-    assert!(status.success());
-    println!("cargo:rerun-if-changed=runtime");
-
-    let path = out_dir.join("wasm32-wasip1/release/libcomponentize_py_runtime.a");
-
-    if path.exists() {
-        let clang = wasi_sdk.join(format!("bin/{CLANG_EXECUTABLE}"));
-        if clang.exists() {
-            let name = "libcomponentize_py_runtime.so";
-
-            run(Command::new(clang)
-                .arg("-shared")
-                .arg("-o")
-                .arg(out_dir.join(name))
-                .arg("-Wl,--whole-archive")
-                .arg(&path)
-                .arg("-Wl,--no-whole-archive")
-                .arg(format!("-L{}", cpython_wasi_dir.to_str().unwrap()))
-                .arg("-lpython3.14"))?;
-
-            compress(out_dir, name, out_dir, false)?;
-        } else {
-            bail!("no such file: {}", clang.display())
-        }
-    } else {
-        bail!("no such file: {}", path.display())
-    }
+    make_runtime(
+        out_dir,
+        &wasi_sdk,
+        &cpython_wasi_dir,
+        false,
+        "libcomponentize_py_runtime_sync.so",
+    )?;
+    make_runtime(
+        out_dir,
+        &wasi_sdk,
+        &cpython_wasi_dir,
+        true,
+        "libcomponentize_py_runtime_async.so",
+    )?;
 
     let libraries = [
         "libc.so",
@@ -406,6 +365,79 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> {
     Ok(())
 }
 
+fn make_runtime(
+    out_dir: &Path,
+    wasi_sdk: &Path,
+    cpython_wasi_dir: &Path,
+    async_: bool,
+    name: &str,
+) -> Result<()> {
+    let mut cmd = Command::new("rustup");
+    cmd.current_dir("runtime")
+        .arg("run")
+        .arg("nightly")
+        .arg("cargo")
+        .arg("build")
+        .arg("-Z")
+        .arg("build-std=panic_abort,std")
+        .arg("--release")
+        .arg("--target=wasm32-wasip1");
+
+    if async_ {
+        cmd.arg("--features=async");
+    }
+
+    for (key, _) in env::vars_os() {
+        if key
+            .to_str()
+            .map(|key| key.starts_with("RUST") || key.starts_with("CARGO"))
+            .unwrap_or(false)
+        {
+            cmd.env_remove(&key);
+        }
+    }
+
+    let target = if async_ { "async" } else { "sync" };
+
+    cmd.env(
+        "RUSTFLAGS",
+        "-C relocation-model=pic --cfg pyo3_disable_reference_pool",
+    )
+    .env("CARGO_TARGET_DIR", out_dir.join(target))
+    .env("PYO3_CONFIG_FILE", out_dir.join("pyo3-config.txt"));
+
+    let status = cmd.status()?;
+    assert!(status.success());
+    println!("cargo:rerun-if-changed=runtime");
+
+    let path = out_dir
+        .join(target)
+        .join("wasm32-wasip1/release/libcomponentize_py_runtime.a");
+
+    if path.exists() {
+        let clang = wasi_sdk.join(format!("bin/{CLANG_EXECUTABLE}"));
+        if clang.exists() {
+            run(Command::new(clang)
+                .arg("-shared")
+                .arg("-o")
+                .arg(out_dir.join(name))
+                .arg("-Wl,--whole-archive")
+                .arg(&path)
+                .arg("-Wl,--no-whole-archive")
+                .arg(format!("-L{}", cpython_wasi_dir.to_str().unwrap()))
+                .arg("-lpython3.14"))?;
+
+            compress(out_dir, name, out_dir, false)?;
+        } else {
+            bail!("no such file: {}", clang.display())
+        }
+    } else {
+        bail!("no such file: {}", path.display())
+    }
+
+    Ok(())
+}
+
 fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> {
     let response = reqwest::blocking::get(url)?;
     let decoder = flate2::read::GzDecoder::new(response);