diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96ac286e974..7da18f2b314 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,3 +339,35 @@ jobs: with: file: coverage.lcov name: ${{ matrix.os }} + + emscripten: + name: emscripten + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.11.0-beta.1 + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-emscripten + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: pip install nox + - uses: actions/cache@v3 + id: cache + with: + path: | + .nox/emscripten + key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} + - uses: Swatinem/rust-cache@v1 + with: + key: cargo-emscripten-wasm32 + - name: Build + if: steps.cache.outputs.cache-hit != 'true' + run: nox -s build_emscripten + - name: Test + run: nox -s test_emscripten diff --git a/emscripten/Makefile b/emscripten/Makefile new file mode 100644 index 00000000000..eec64876298 --- /dev/null +++ b/emscripten/Makefile @@ -0,0 +1,88 @@ +CURDIR=$(abspath .) + +# These three are passed in from nox. +BUILDROOT ?= $(CURDIR)/builddir +PYMAJORMINORMICRO ?= 3.11.0 +PYPRERELEASE ?= b1 # I'm not sure how to split 3.11.0b1 in Make. + +EMSCRIPTEN_VERSION=3.1.13 + +export EMSDKDIR = $(BUILDROOT)/emsdk + +PLATFORM=wasm32_emscripten +SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM) + +# BASH_ENV tells bash to source emsdk_env.sh on startup. +export BASH_ENV := $(CURDIR)/env.sh +# Use bash to run each command so that env.sh will be used. +SHELL := /bin/bash + + +# Set version variables. +version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%)) +PYMAJOR=$(word 1,$(version_tuple)) +PYMINOR=$(word 2,$(version_tuple)) +PYMICRO=$(word 3,$(version_tuple)) +PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE) +PYMAJORMINOR=$(PYMAJOR).$(PYMINOR) + + +PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz +PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz +PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION) + +PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib + +all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a + +$(BUILDROOT)/.exists: + mkdir -p $(BUILDROOT) + touch $@ + + +# Install emscripten +$(EMSDKDIR): $(CURDIR)/emscripten_patches/* $(BUILDROOT)/.exists + git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR) + $(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION) + cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1 + $(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION) + + +$(PYTHONTARBALL): + [ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads + wget -q -O $@ $(PYTHONURL) + +$(PYTHONBUILD)/.patched: $(PYTHONTARBALL) + [ -d $(PYTHONBUILD) ] || ( \ + mkdir -p $(dir $(PYTHONBUILD));\ + tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \ + ) + touch $@ + +$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(BUILDROOT)/emsdk + cd $(PYTHONBUILD) && \ + CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ + emconfigure ./configure -C \ + --host=wasm32-unknown-emscripten \ + --build=$(shell $(PYTHONBUILD)/config.guess) \ + --with-emscripten-target=browser \ + --enable-wasm-dynamic-linking \ + --with-build-python=python3.11 + +$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile + cd $(PYTHONBUILD) && \ + emmake make -j3 libpython$(PYMAJORMINOR).a + + # Generate sysconfigdata + _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars + cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib + + mkdir -p $(PYTHONLIBDIR) + # Copy libexpat.a, libmpdec.a, and libpython3.11.a + # In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS + find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \; + # Install Python stdlib + cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR) + +clean: + rm -rf $(BUILDROOT) diff --git a/emscripten/emscripten_patches/0001-Add-_gxx_personality_v0-stub-to-library.js.patch b/emscripten/emscripten_patches/0001-Add-_gxx_personality_v0-stub-to-library.js.patch new file mode 100644 index 00000000000..bd0af28a03f --- /dev/null +++ b/emscripten/emscripten_patches/0001-Add-_gxx_personality_v0-stub-to-library.js.patch @@ -0,0 +1,28 @@ +From 4b56f37c3dc9185a235a8314086c4d7a6239b2f8 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Sat, 4 Jun 2022 19:19:47 -0700 +Subject: [PATCH] Add _gxx_personality_v0 stub to library.js + +Mitigation for an incompatibility between Rust and Emscripten: +https://github.com/rust-lang/rust/issues/85821 +https://github.com/emscripten-core/emscripten/issues/17128 +--- + src/library.js | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/library.js b/src/library.js +index e7bb4c38e..7d01744df 100644 +--- a/src/library.js ++++ b/src/library.js +@@ -403,6 +403,8 @@ mergeInto(LibraryManager.library, { + abort('Assertion failed: ' + UTF8ToString(condition) + ', at: ' + [filename ? UTF8ToString(filename) : 'unknown filename', line, func ? UTF8ToString(func) : 'unknown function']); + }, + ++ __gxx_personality_v0: function() {}, ++ + // ========================================================================== + // time.h + // ========================================================================== +-- +2.25.1 + diff --git a/emscripten/env.sh b/emscripten/env.sh new file mode 100644 index 00000000000..87b7b55138e --- /dev/null +++ b/emscripten/env.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Activate emsdk environment. emsdk_env.sh writes a lot to stderr so we suppress +# the output. This also prevents it from complaining when emscripten isn't yet +# installed. +source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true diff --git a/emscripten/pybuilddir.txt b/emscripten/pybuilddir.txt new file mode 100644 index 00000000000..59f2a4a7546 --- /dev/null +++ b/emscripten/pybuilddir.txt @@ -0,0 +1 @@ +build/lib.linux-x86_64-3.11 \ No newline at end of file diff --git a/emscripten/runner.py b/emscripten/runner.py new file mode 100755 index 00000000000..95eaa8d4f36 --- /dev/null +++ b/emscripten/runner.py @@ -0,0 +1,8 @@ +#!/usr/local/bin/python +import pathlib +import sys +import subprocess + +p = pathlib.Path(sys.argv[1]) + +sys.exit(subprocess.call(["node", p.name], cwd=p.parent)) diff --git a/noxfile.py b/noxfile.py index e399a79cf9c..52c8b40ce64 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,5 +1,7 @@ import time from glob import glob +from pathlib import Path +import re import nox @@ -128,3 +130,66 @@ def contributors(session: nox.Session) -> None: for author in authors: print(f"@{author}") + + +class EmscriptenInfo: + def __init__(self): + rootdir = Path(__file__).parent + self.emscripten_dir = rootdir / "emscripten" + self.builddir = rootdir / ".nox/emscripten" + self.builddir.mkdir(exist_ok=True, parents=True) + + self.pyversion = "3.11.0b1" + self.pymajor, self.pyminor, self.pymicro = self.pyversion.split(".") + self.pymicro, self.pydev = re.match( + "([0-9]*)([^0-9].*)?", self.pymicro + ).groups() + if self.pydev is None: + self.pydev = "" + + self.pymajorminor = f"{self.pymajor}.{self.pyminor}" + self.pymajorminormicro = f"{self.pymajorminor}.{self.pymicro}" + + +@nox.session(venv_backend="none") +def build_emscripten(session: nox.Session): + info = EmscriptenInfo() + session.run( + "make", + "-C", + str(info.emscripten_dir), + f"BUILDROOT={info.builddir}", + f"PYMAJORMINORMICRO={info.pymajorminormicro}", + f"PYPRERELEASE={info.pydev}", + external=True, + ) + + +@nox.session(venv_backend="none") +def test_emscripten(session: nox.Session): + info = EmscriptenInfo() + + libdir = info.builddir / f"install/Python-{info.pyversion}/lib" + pythonlibdir = libdir / f"python{info.pymajorminor}" + + target = "wasm32-unknown-emscripten" + + session.env["CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER"] = "python " + str( + info.emscripten_dir / "runner.py" + ) + session.env["RUSTFLAGS"] = " ".join( + [ + f"-L native={libdir}", + "-C link-arg=--preload-file", + f"-C link-arg={pythonlibdir}@/lib/python{info.pymajorminor}", + f"-C link-arg=-lpython{info.pymajorminor}", + "-C link-arg=-lexpat", + "-C link-arg=-lmpdec", + ] + ) + session.env["CARGO_BUILD_TARGET"] = target + session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir + session.run("rustup", "target", "add", target, "--toolchain", "stable") + session.run( + "bash", "-c", f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test" + ) diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 8508e6df338..e6867901b9e 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -6,6 +6,7 @@ use crate::types::PyString; #[cfg(target_endian = "little")] use libc::wchar_t; +#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_datetime_fromtimestamp() { Python::with_gil(|py| { @@ -23,6 +24,7 @@ fn test_datetime_fromtimestamp() { }) } +#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_date_fromtimestamp() { Python::with_gil(|py| { @@ -40,6 +42,7 @@ fn test_date_fromtimestamp() { }) } +#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_utc_timezone() { Python::with_gil(|py| { @@ -183,6 +186,7 @@ fn ucs4() { } #[test] +#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[cfg(not(PyPy))] fn test_get_tzinfo() { crate::Python::with_gil(|py| { diff --git a/src/gil.rs b/src/gil.rs index d5258ed60fc..37e0ed2495d 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -729,6 +729,7 @@ mod tests { } #[test] + #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled fn test_clone_without_gil() { use crate::{Py, PyAny}; use std::{sync::Arc, thread}; @@ -799,6 +800,7 @@ mod tests { } #[test] + #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled fn test_clone_in_other_thread() { use crate::Py; use std::{sync::Arc, thread}; diff --git a/src/marker.rs b/src/marker.rs index 6c27833bee5..e3d48051f8b 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -942,6 +942,7 @@ mod tests { } #[test] + #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled fn test_allow_threads_releases_and_acquires_gil() { Python::with_gil(|py| { let b = std::sync::Arc::new(std::sync::Barrier::new(2)); diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 2d55caca9b8..810f19a4821 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -546,6 +546,7 @@ fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyObject>) -> *mut ffi::PyObject { #[cfg(test)] mod tests { #[test] + #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_new_with_fold() { crate::Python::with_gil(|py| { use crate::types::{PyDateTime, PyTimeAccess}; @@ -560,6 +561,7 @@ mod tests { #[cfg(not(PyPy))] #[test] + #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_get_tzinfo() { crate::Python::with_gil(|py| { use crate::conversion::ToPyObject; diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index ad8c8c07d8d..b3ebc5df4d0 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -259,6 +259,7 @@ fn test_unsendable() -> PyResult<()> { /// If a class is marked as `unsendable`, it panics when accessed by another thread. #[test] +#[cfg_attr(target_arch = "wasm32", ignore)] #[should_panic( expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!" )] @@ -267,6 +268,7 @@ fn panic_unsendable_base() { } #[test] +#[cfg_attr(target_arch = "wasm32", ignore)] #[should_panic( expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!" )] diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index f85c925a673..26d9f66685c 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -1,6 +1,7 @@ #![cfg(feature = "macros")] #[rustversion::stable] +#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm #[test] fn test_compile_errors() { // stable - require all tests to pass @@ -8,6 +9,7 @@ fn test_compile_errors() { } #[cfg(not(feature = "nightly"))] +#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled #[rustversion::nightly] #[test] fn test_compile_errors() { @@ -17,6 +19,7 @@ fn test_compile_errors() { } #[cfg(feature = "nightly")] +#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm #[rustversion::nightly] #[test] fn test_compile_errors() { diff --git a/tests/test_dict_iter.rs b/tests/test_dict_iter.rs index 1a79ca92f81..51f54167aec 100644 --- a/tests/test_dict_iter.rs +++ b/tests/test_dict_iter.rs @@ -2,6 +2,7 @@ use pyo3::prelude::*; use pyo3::types::IntoPyDict; #[test] +#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails. fn iter_dict_nosegv() { let gil = Python::acquire_gil(); let py = gil.python(); diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index da42f1b50a0..9483f6ae6d5 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -17,6 +17,7 @@ fn fail_to_open_file() -> PyResult<()> { } #[test] +#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails. #[cfg(not(target_os = "windows"))] fn test_filenotfounderror() { let gil = Python::acquire_gil(); diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 68881d6fb89..635412ca41a 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -698,6 +698,7 @@ impl OnceFuture { } #[test] +#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop) fn test_await() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -747,6 +748,7 @@ impl AsyncIterator { } #[test] +#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop) fn test_anext_aiter() { let gil = Python::acquire_gil(); let py = gil.python();