wasm: initial support for wasm32-wasi target (#4716)

This adds initial, unstable, support for the wasm32-wasi target. Not all of Tokio's
features are supported yet as WASI's non-blocking APIs are still limited.

Refs: #4827

.github/workflows/ci.yml

@@ -489,9 +489,23 @@ jobs:
       - name: Install cargo-wasi
         run: cargo install cargo-wasi
 
-      # TODO: Expand this when full WASI support lands.
-      #   Currently, this is a bare bones regression test
-      #   for features that work today with wasi.
+      - name: WASI test tokio full
+        run: cargo test -p tokio --target wasm32-wasi --features full
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+      - name: WASI test tokio-util full
+        run: cargo test -p tokio-util --target wasm32-wasi --features full
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+      - name: WASI test tokio-stream
+        run: cargo test -p tokio-stream --target wasm32-wasi --features time,net,io-util,sync
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
 
       - name: test tests-integration --features wasi-rt
         # TODO: this should become: `cargo hack wasi test --each-feature`

tests-integration/tests/macros_main.rs

@@ -1,4 +1,8 @@
-#![cfg(all(feature = "macros", feature = "rt-multi-thread"))]
+#![cfg(all(
+    feature = "macros",
+    feature = "rt-multi-thread",
+    not(target_os = "wasi")
+))]
 
 #[tokio::main]
 async fn basic_main() -> usize {

tests-integration/tests/macros_select.rs

@@ -4,6 +4,7 @@ use futures::channel::oneshot;
 use futures::executor::block_on;
 use std::thread;
 
+#[cfg_attr(target_os = "wasi", ignore = "WASI: std::thread::spawn not supported")]
 #[test]
 fn join_with_select() {
     block_on(async {

tests-integration/tests/process_stdio.rs

@@ -1,5 +1,5 @@
 #![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(target_os = "wasi")))]
 
 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
 use tokio::join;

tokio-stream/Cargo.toml

@@ -38,6 +38,7 @@ parking_lot = "0.12.0"
 tokio-test = { path = "../tokio-test" }
 futures = { version = "0.3", default-features = false }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 proptest = "1"
 
 [package.metadata.docs.rs]

tokio-stream/tests/stream_panic.rs

@@ -1,5 +1,5 @@
 #![warn(rust_2018_idioms)]
-#![cfg(feature = "time")]
+#![cfg(all(feature = "time", not(target_os = "wasi")))] // Wasi does not support panic recovery
 
 use parking_lot::{const_mutex, Mutex};
 use std::error::Error;

tokio-stream/tests/stream_stream_map.rs

@@ -325,6 +325,7 @@ fn one_ready_many_none() {
     }
 }
 
+#[cfg(not(target_os = "wasi"))]
 proptest::proptest! {
     #[test]
     fn fuzz_pending_complete_mix(kinds: Vec<bool>) {

tokio-util/src/lib.rs

@@ -29,6 +29,7 @@ cfg_codec! {
 }
 
 cfg_net! {
+    #[cfg(not(target_arch = "wasm32"))]
     pub mod udp;
     pub mod net;
 }

tokio-util/src/task/mod.rs

@@ -2,7 +2,9 @@
 
 #[cfg(tokio_unstable)]
 mod join_map;
+#[cfg(not(target_os = "wasi"))]
 mod spawn_pinned;
+#[cfg(not(target_os = "wasi"))]
 pub use spawn_pinned::LocalPoolHandle;
 
 #[cfg(tokio_unstable)]

tokio-util/tests/context.rs

@@ -1,4 +1,5 @@
 #![cfg(feature = "rt")]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 #![warn(rust_2018_idioms)]
 
 use tokio::runtime::Builder;

tokio-util/tests/io_sync_bridge.rs

@@ -1,4 +1,5 @@
 #![cfg(feature = "io-util")]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 
 use std::error::Error;
 use std::io::{Cursor, Read, Result as IoResult};

tokio-util/tests/panic.rs

@@ -1,5 +1,5 @@
 #![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support panic recovery
 
 use parking_lot::{const_mutex, Mutex};
 use std::error::Error;

tokio-util/tests/spawn_pinned.rs

@@ -1,4 +1,5 @@
 #![warn(rust_2018_idioms)]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 
 use std::rc::Rc;
 use std::sync::Arc;

tokio-util/tests/time_delay_queue.rs

@@ -778,6 +778,7 @@ async fn compact_change_deadline() {
     assert!(entry.is_none());
 }
 
+#[cfg_attr(target_os = "wasi", ignore = "FIXME: Does not seem to work with WASI")]
 #[tokio::test(start_paused = true)]
 async fn remove_after_compact() {
     let now = Instant::now();
@@ -794,6 +795,7 @@ async fn remove_after_compact() {
     assert!(panic.is_err());
 }
 
+#[cfg_attr(target_os = "wasi", ignore = "FIXME: Does not seem to work with WASI")]
 #[tokio::test(start_paused = true)]
 async fn remove_after_compact_poll() {
     let now = Instant::now();

tokio-util/tests/udp.rs

@@ -1,4 +1,5 @@
 #![warn(rust_2018_idioms)]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support UDP
 
 use tokio::net::UdpSocket;
 use tokio_stream::StreamExt;

tokio/Cargo.toml

@@ -51,7 +51,6 @@ net = [
   "mio/os-poll",
   "mio/os-ext",
   "mio/net",
-  "socket2",
   "winapi/fileapi",
   "winapi/handleapi",
   "winapi/namedpipeapi",
@@ -112,11 +111,13 @@ pin-project-lite = "0.2.0"
 bytes = { version = "1.0.0", optional = true }
 once_cell = { version = "1.5.2", optional = true }
 memchr = { version = "2.2", optional = true }
-mio = { version = "0.8.1", optional = true }
-socket2 = { version = "0.4.4", optional = true, features = [ "all" ] }
+mio = { version = "0.8.4", optional = true }
 num_cpus = { version = "1.8.0", optional = true }
 parking_lot = { version = "0.12.0", optional = true }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+socket2 = { version = "0.4.4", features = [ "all" ] }
+
 # Currently unstable. The API exposed by these features may be broken at any time.
 # Requires `--cfg tokio_unstable` to enable.
 [target.'cfg(tokio_unstable)'.dependencies]
@@ -149,10 +150,12 @@ async-stream = "0.3"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 proptest = "1"
-rand = "0.8.0"
 socket2 = "0.4"
 
-[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dev-dependencies]
+rand = "0.8.0"
+
+[target.'cfg(all(target_arch = "wasm32", not(target_os = "wasi")))'.dev-dependencies]
 wasm-bindgen-test = "0.3.0"
 
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]

tokio/src/coop.rs

@@ -207,7 +207,7 @@ cfg_coop! {
 mod test {
     use super::*;
 
-    #[cfg(target_arch = "wasm32")]
+    #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
     use wasm_bindgen_test::wasm_bindgen_test as test;
 
     fn get() -> Budget {

tokio/src/io/driver/mod.rs

@@ -72,6 +72,8 @@ pub(super) struct Inner {
     io_dispatch: RwLock<IoDispatcher>,
 
     /// Used to wake up the reactor from a call to `turn`.
+    /// Not supported on Wasi due to lack of threading support.
+    #[cfg(not(target_os = "wasi"))]
     waker: mio::Waker,
 
     metrics: IoDriverMetrics,
@@ -115,6 +117,7 @@ impl Driver {
     /// creation.
     pub(crate) fn new() -> io::Result<Driver> {
         let poll = mio::Poll::new()?;
+        #[cfg(not(target_os = "wasi"))]
         let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
         let registry = poll.registry().try_clone()?;
 
@@ -129,6 +132,7 @@ impl Driver {
             inner: Arc::new(Inner {
                 registry,
                 io_dispatch: RwLock::new(IoDispatcher::new(allocator)),
+                #[cfg(not(target_os = "wasi"))]
                 waker,
                 metrics: IoDriverMetrics::default(),
             }),
@@ -164,6 +168,11 @@ impl Driver {
         match self.poll.poll(&mut events, max_wait) {
             Ok(_) => {}
             Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
+            #[cfg(target_os = "wasi")]
+            Err(e) if e.kind() == io::ErrorKind::InvalidInput => {
+                // In case of wasm32_wasi this error happens, when trying to poll without subscriptions
+                // just return from the park, as there would be nothing, which wakes us up.
+            }
             Err(e) => return Err(e),
         }
 
@@ -300,6 +309,7 @@ impl Handle {
     /// blocked in `turn`, then the next call to `turn` will not block and
     /// return immediately.
     fn wakeup(&self) {
+        #[cfg(not(target_os = "wasi"))]
         self.inner.waker.wake().expect("failed to wake I/O driver");
     }
 }

tokio/src/io/mod.rs

@@ -211,9 +211,11 @@ cfg_io_driver_impl! {
         pub use driver::{Interest, Ready};
     }
 
+    #[cfg_attr(target_os = "wasi", allow(unused_imports))]
     mod poll_evented;
 
     #[cfg(not(loom))]
+    #[cfg_attr(target_os = "wasi", allow(unused_imports))]
     pub(crate) use poll_evented::PollEvented;
 }
 

tokio/src/lib.rs

@@ -393,6 +393,20 @@ compile_error! {
     "Tokio requires the platform pointer width to be 32, 64, or 128 bits"
 }
 
+#[cfg(all(
+    not(tokio_unstable),
+    target_arch = "wasm32",
+    any(
+        feature = "fs",
+        feature = "io-std",
+        feature = "net",
+        feature = "process",
+        feature = "rt-multi-thread",
+        feature = "signal"
+    )
+))]
+compile_error!("Only features sync,macros,io-util,rt are supported on wasm.");
+
 // Includes re-exports used by macros.
 //
 // This module is not intended to be part of the public API. In general, any

tokio/src/macros/cfg.rs

@@ -61,6 +61,7 @@ macro_rules! cfg_fs {
     ($($item:item)*) => {
         $(
             #[cfg(feature = "fs")]
+            #[cfg(not(target_os = "wasi"))]
             #[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
             $item
         )*
@@ -247,6 +248,7 @@ macro_rules! cfg_process {
             #[cfg(feature = "process")]
             #[cfg_attr(docsrs, doc(cfg(feature = "process")))]
             #[cfg(not(loom))]
+            #[cfg(not(target_os = "wasi"))]
             $item
         )*
     }
@@ -275,6 +277,7 @@ macro_rules! cfg_signal {
             #[cfg(feature = "signal")]
             #[cfg_attr(docsrs, doc(cfg(feature = "signal")))]
             #[cfg(not(loom))]
+            #[cfg(not(target_os = "wasi"))]
             $item
         )*
     }
@@ -334,7 +337,7 @@ macro_rules! cfg_not_rt {
 macro_rules! cfg_rt_multi_thread {
     ($($item:item)*) => {
         $(
-            #[cfg(feature = "rt-multi-thread")]
+            #[cfg(all(feature = "rt-multi-thread", not(target_os = "wasi")))]
             #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
             $item
         )*
@@ -451,7 +454,8 @@ macro_rules! cfg_has_atomic_u64 {
                     target_arch = "arm",
                     target_arch = "mips",
                     target_arch = "powerpc",
-                    target_arch = "riscv32"
+                    target_arch = "riscv32",
+                    target_arch = "wasm32"
                     )))]
             $item
         )*
@@ -465,9 +469,28 @@ macro_rules! cfg_not_has_atomic_u64 {
                     target_arch = "arm",
                     target_arch = "mips",
                     target_arch = "powerpc",
-                    target_arch = "riscv32"
+                    target_arch = "riscv32",
+                    target_arch = "wasm32"
                     ))]
             $item
         )*
     }
 }
+
+macro_rules! cfg_not_wasi {
+    ($($item:item)*) => {
+        $(
+            #[cfg(not(target_os = "wasi"))]
+            $item
+        )*
+    }
+}
+
+macro_rules! cfg_is_wasm_not_wasi {
+    ($($item:item)*) => {
+        $(
+            #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
+            $item
+        )*
+    }
+}

tokio/src/net/mod.rs

@@ -23,8 +23,10 @@
 //! [`UnixDatagram`]: UnixDatagram
 
 mod addr;
-#[cfg(feature = "net")]
-pub(crate) use addr::to_socket_addrs;
+cfg_not_wasi! {
+    #[cfg(feature = "net")]
+    pub(crate) use addr::to_socket_addrs;
+}
 pub use addr::ToSocketAddrs;
 
 cfg_net! {
@@ -33,11 +35,13 @@ cfg_net! {
 
     pub mod tcp;
     pub use tcp::listener::TcpListener;
-    pub use tcp::socket::TcpSocket;
     pub use tcp::stream::TcpStream;
+    cfg_not_wasi! {
+        pub use tcp::socket::TcpSocket;
 
-    mod udp;
-    pub use udp::UdpSocket;
+        mod udp;
+        pub use udp::UdpSocket;
+    }
 }
 
 cfg_net_unix! {