Cryptography v36.0.2 w/ Rust (#2378)

pyodide · Jun 9, 2022 · dbbbbb6 · dbbbbb6
1 parent b880ec2
commit dbbbbb6
Show file tree

Hide file tree

Showing 22 changed files with 195 additions and 159 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -129,6 +129,11 @@ jobs:
             - -pkg2-v20220303-{{ checksum "Makefile.envs" }}
             - -pkg2-v20220303
 
+      - run:
+          name: install rust
+          command: |
+            make rust
+
       - run:
           name: build packages
           no_output_timeout: 60m

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -101,5 +101,5 @@ repos:
     rev: "v2.1.0"
     hooks:
       - id: codespell
-        args: ["-L", "te,slowy,aray,ba,nd,classs,feld"]
+        args: ["-L", "te,slowy,aray,ba,nd,classs,crate,feld"]
         exclude: ^benchmark/benchmarks/pystone_benchmarks/pystone\.py$
diff --git a/Makefile b/Makefile
@@ -239,6 +239,12 @@ emsdk/emsdk/.complete:
 	date +"[%F %T] done building emsdk."
 
 
+rust:
+	wget https://sh.rustup.rs -O /rustup.sh
+	sh /rustup.sh -y
+	source $(HOME)/.cargo/env && rustup target add wasm32-unknown-emscripten --toolchain stable
+
+
 FORCE:
 
 

diff --git a/Makefile.envs b/Makefile.envs
@@ -146,5 +146,18 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \
 
 export STDLIB_MODULE_CFLAGS= $(SIDE_MODULE_CFLAGS) -I Include/ -I .
 
+# For RUST
+export CARGO_HOME ?= $(HOME)/.cargo
+export CARGO_BUILD_TARGET=wasm32-unknown-emscripten
+export CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=emcc
+export PYO3_CONFIG_FILE=$(PYODIDE_ROOT)/tools/pyo3_config.ini
+
+# idealy we could automatically include all SIDE_MODULE_LDFLAGS here
+export RUSTFLAGS= \
+	-C relocation-model=pic \
+	-C target-feature=+mutable-globals \
+	-C link-arg=-sSIDE_MODULE=1 \
+	-C link-arg=-sWASM_BIGINT
+
 .output_vars:
 	set
diff --git a/cpython/adjust_sysconfig.py b/cpython/adjust_sysconfig.py
@@ -23,7 +23,7 @@ def adjust_sysconfig(config_vars: dict[str, str]):
         MAINCC="cc",
         LDSHARED="cc",
         LINKCC="cc",
-        BLDSHARED="cc",
+        BLDSHARED="emcc -sSIDE_MODULE=1",  # setuptools-rust looks at this
         CXX="c++",
         LDCXXSHARED="c++",
     )

diff --git a/docs/development/building-from-sources.md b/docs/development/building-from-sources.md
@@ -151,9 +151,18 @@ meta-package. Other supported meta-packages are,
   "pytest". This option is non exhaustive and is mainly intended to make build
   faster while testing a diverse set of scientific packages.
 - "\*" builds all packages
+- You can exclude a package by prefixing it with "!".
 
 micropip and distutils are always automatically included.
 
+The cryptography package is a Rust extension. If you want to build it, you will
+need Rust >= 1.41, you need the
+[CARGO_HOME](https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-reads)
+environment variable set appropriately, and you need the
+`wasm32-unknown-emscripten` toolchain installed. If you run `make rust`, Pyodide
+will install this stuff automatically. If you want to build every package except
+for cryptography, you can set `PYODIDE_PACKAGES="*,!cryptography"`.
+
 ## Environment variables
 
 The following environment variables additionally impact the build:

diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md
@@ -328,3 +328,10 @@ imports are synchronous so it is impossible to load `.so` files lazily.
 
    meta-yaml.md
 ```
+
+### Rust/PyO3 Packages
+
+We currently build Cryptography which is a Rust extension built with PyO3 and
+setuptools-rust. It should be reasonably easy to build other Rust extensions.
+Currently it is necessary to run `source $CARGO_HOME/env` in the build script,
+but other than that there may be no other issues if you are lucky.
diff --git a/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch b/emsdk/patches/0001-Throw-away-errors-in-minify_wasm_js.patch
@@ -1,7 +1,7 @@
 From e83a295f8e1b8c48d4748d1811a4b22840f25e14 Mon Sep 17 00:00:00 2001
 From: Hood <hood@mit.edu>
 Date: Thu, 24 Jun 2021 04:08:02 -0700
-Subject: [PATCH 1/6] Throw away errors in minify_wasm_js
+Subject: [PATCH 1/8] Throw away errors in minify_wasm_js
 
 ---
  emcc.py | 13 ++++++++-----

diff --git a/emsdk/patches/0002-Fix-dup.patch b/emsdk/patches/0002-Fix-dup.patch
@@ -1,7 +1,7 @@
 From 40956dee436737d9dd40e0b57c6e2ebd26569920 Mon Sep 17 00:00:00 2001
 From: Hood <hood@mit.edu>
 Date: Wed, 8 Sep 2021 17:49:15 -0700
-Subject: [PATCH 2/6] Fix dup
+Subject: [PATCH 2/8] Fix dup
 
 This fixes two problems with the `dup` system calls:
 1. `dup` expects that every file descriptor has a corresponding file (so pipes and (https://github.com/emscripten-core/emscripten/issues/14640)

diff --git a/emsdk/patches/0003-Fix-side-module-exception-handling.patch b/emsdk/patches/0003-Fix-side-module-exception-handling.patch
@@ -1,7 +1,7 @@
 From 73b89ee1b5a57c65824baf91b547be32b69decbd Mon Sep 17 00:00:00 2001
 From: Hood Chatham <roberthoodchatham@gmail.com>
 Date: Tue, 15 Feb 2022 23:27:03 -0500
-Subject: [PATCH 3/6] Fix side module exception handling
+Subject: [PATCH 3/8] Fix side module exception handling
 
 See https://github.com/emscripten-core/emscripten/pull/16309
 

diff --git a/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch b/emsdk/patches/0004-Fix-lookupPath-when-applied-to-a-symlink-loop.patch
@@ -1,7 +1,7 @@
 From 8de43377d0e72a4c5794f8494a06d81a9609090f Mon Sep 17 00:00:00 2001
 From: Hood Chatham <roberthoodchatham@gmail.com>
 Date: Wed, 2 Mar 2022 13:44:14 -0800
-Subject: [PATCH 4/6] Fix lookupPath when applied to a symlink loop
+Subject: [PATCH 4/8] Fix lookupPath when applied to a symlink loop
 
 The following code leads to an infinite loop in lookupPath:
 

diff --git a/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch b/emsdk/patches/0005-Indicate-Emscripten-version-in-uname.patch
@@ -1,7 +1,7 @@
 From 8eed4062c1a0bd6aa7859938240f8761e6e72114 Mon Sep 17 00:00:00 2001
 From: Hood Chatham <roberthoodchatham@gmail.com>
 Date: Wed, 18 May 2022 21:49:17 -0700
-Subject: [PATCH 5/6] Indicate Emscripten version in uname
+Subject: [PATCH 5/8] Indicate Emscripten version in uname
 
 This patch has been upstreamed:
 https://github.com/emscripten-core/emscripten/pull/17026

diff --git a/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch b/emsdk/patches/0006-Add-BigInt64Array-shim-for-Safari-14.patch
@@ -1,7 +1,7 @@
 From 4462c2871933db0adc4b46a24f77b942caff969c Mon Sep 17 00:00:00 2001
 From: Hood Chatham <roberthoodchatham@gmail.com>
 Date: Sun, 29 May 2022 18:17:29 -0700
-Subject: [PATCH 6/6] Add BigInt64Array shim for Safari 14
+Subject: [PATCH 6/8] Add BigInt64Array shim for Safari 14
 
 ---
  src/parseTools.js             |  2 +-

diff --git a/emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch b/emsdk/patches/0007-Disable-whole-archive-when-linking-rust.patch
@@ -0,0 +1,33 @@
+From 9fb8391c7e6b9f36f45a1448e49597f899cce29d Mon Sep 17 00:00:00 2001
+From: Hood Chatham <roberthoodchatham@gmail.com>
+Date: Thu, 6 Jan 2022 09:40:39 -0800
+Subject: [PATCH 7/8] Disable whole-archive when linking rust
+
+Rust .rlib archives contain an extra metadata file called lib.rmeta.
+Emscripten sets `--whole-archive` by default if LINKABLE is set.
+But with `--whole-archive` the linker crashes with an error due to
+the extra files. This disables the problematic `--whole-archive`
+setting when we are linking rust.
+
+See emscripten issue:
+https://github.com/emscripten-core/emscripten/issues/17109
+---
+ tools/building.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/tools/building.py b/tools/building.py
+index 512561c69..d180993a1 100644
+--- a/tools/building.py
++++ b/tools/building.py
+@@ -443,7 +443,7 @@ def link_lld(args, target, external_symbols=None):
+
+   # Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to
+   # include all archive contents.
+-  if settings.LINKABLE:
++  if settings.LINKABLE and not any(arg.endswith(".rlib") for arg in args):
+     args.insert(0, '--whole-archive')
+     args.append('--no-whole-archive')
+
+-- 
+2.25.1
+
diff --git a/emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch b/emsdk/patches/0008-Add-signature-to-emscripten_get_now.patch
@@ -0,0 +1,26 @@
+From b6839bba05c1a5eab390f47309a7bef1b7294eca Mon Sep 17 00:00:00 2001
+From: Hood Chatham <roberthoodchatham@gmail.com>
+Date: Wed, 1 Jun 2022 10:54:08 -0700
+Subject: [PATCH 8/8] Add signature to emscripten_get_now
+
+Upstream PR:
+https://github.com/emscripten-core/emscripten/pull/17123/files
+---
+ src/library.js | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/library.js b/src/library.js
+index ef8a618bb..5b5a9d838 100644
+--- a/src/library.js
++++ b/src/library.js
+@@ -2657,6 +2657,7 @@ LibraryManager.library = {
+   },
+
+   emscripten_get_now__import: true,
++  emscripten_get_now__sig: 'd',
+   emscripten_get_now: ';' +
+ #if ENVIRONMENT_MAY_BE_NODE
+                                "if (ENVIRONMENT_IS_NODE) {\n" +
+-- 
+2.25.1
+
diff --git a/packages/cryptography/meta.yaml b/packages/cryptography/meta.yaml
@@ -1,20 +1,29 @@
 package:
   name: cryptography
-  version: 3.4.8
+  version: 36.0.2
 
 source:
-  url: https://files.pythonhosted.org/packages/cc/98/8a258ab4787e6f835d350639792527d2eb7946ff9fc0caca9c3f4cf5dcfe/cryptography-3.4.8.tar.gz
-  sha256: 94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c
+  url: https://files.pythonhosted.org/packages/10/a7/51953e73828deef2b58ba1604de9167843ee9cd4185d8aaffcb45dd1932d/cryptography-36.0.2.tar.gz
+  sha256: 70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9
+  patches:
+    # TODO: remove this when chrono makes a release
+    - patches/0001-Use-patched-chrono.patch
+    - patches/0002-Add-instant-patch.patch
 build:
+  script: |
+    export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl)
+    export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl)
+    source $CARGO_HOME/env
+
+    # TODO: remove this when chrono makes a release
+    git clone --depth 1 https://github.com/hoodmane/chrono.git --branch pyodide
+    # TODO: remove this when instant makes a release
+    git clone --depth 1 https://github.com/hoodmane/instant.git --branch emscripten-no-leading-underscore
   cflags: |
     -Wno-implicit-function-declaration
     $(OPENSSL_INCLUDE_PATH)
   ldflags: |
     $(OPENSSL_LIBRARY_PATH)
-  script: |
-    export CRYPTOGRAPHY_DONT_BUILD_RUST=1
-    export OPENSSL_INCLUDE_PATH=$(pkg-config --cflags-only-I --dont-define-prefix openssl)
-    export OPENSSL_LIBRARY_PATH=$(pkg-config --libs-only-L --dont-define-prefix openssl)
 requirements:
   run:
     - openssl

diff --git a/packages/cryptography/patches/0001-Use-patched-chrono.patch b/packages/cryptography/patches/0001-Use-patched-chrono.patch
@@ -0,0 +1,26 @@
+From 53ee3cee6aae5333d61ac687677cf2ca39cb831a Mon Sep 17 00:00:00 2001
+From: Hood Chatham <roberthoodchatham@gmail.com>
+Date: Fri, 8 Apr 2022 18:17:51 -0700
+Subject: [PATCH 1/2] Use patched chrono
+
+---
+ src/rust/Cargo.toml | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
+index 617167d0..35bf38ae 100644
+--- a/src/rust/Cargo.toml
++++ b/src/rust/Cargo.toml
+@@ -5,6 +5,9 @@ authors = ["The cryptography developers <cryptography-dev@python.org>"]
+ edition = "2018"
+ publish = false
+
++[patch.crates-io]
++chrono = { path =  "../../chrono" }
++
+ [dependencies]
+ lazy_static = "1"
+ pyo3 = { version = "0.15.1" }
+-- 
+2.25.1
+
diff --git a/packages/cryptography/patches/0002-Add-instant-patch.patch b/packages/cryptography/patches/0002-Add-instant-patch.patch
@@ -0,0 +1,24 @@
+From 196ace04edb63a93000418f91d56a3cec11b8345 Mon Sep 17 00:00:00 2001
+From: Hood Chatham <roberthoodchatham@gmail.com>
+Date: Wed, 1 Jun 2022 10:11:44 -0700
+Subject: [PATCH 2/2] Add instant patch
+
+---
+ src/rust/Cargo.toml | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
+index 35bf38ae..23a64977 100644
+--- a/src/rust/Cargo.toml
++++ b/src/rust/Cargo.toml
+@@ -7,6 +7,7 @@ publish = false
+
+ [patch.crates-io]
+ chrono = { path =  "../../chrono" }
++instant = { path = "../../instant" }
+
+ [dependencies]
+ lazy_static = "1"
+-- 
+2.25.1
+