From dde7f765f45f73910ebd4c02e7af6b322b18e1f2 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 5 Dec 2019 12:12:39 -0800 Subject: [PATCH 1/2] Use `wat` and `wasmprinter` crates from crates.io Don't rely on `wasm2wat` and `wat2wasm` crates if we can avoid it. --- crates/fuzz-utils/Cargo.toml | 6 +- crates/fuzz-utils/src/lib.rs | 31 +++++----- crates/tests-utils/src/lib.rs | 67 +--------------------- crates/tests/Cargo.toml | 10 ++-- crates/tests/tests/function_imports.rs | 3 +- crates/tests/tests/invalid.rs | 2 +- crates/tests/tests/round_trip.rs | 5 +- crates/tests/tests/round_trip/simd.wat | 28 ++++----- crates/tests/tests/valid.rs | 2 +- src/module/functions/local_function/mod.rs | 2 +- src/module/mod.rs | 2 +- 11 files changed, 47 insertions(+), 111 deletions(-) diff --git a/crates/fuzz-utils/Cargo.toml b/crates/fuzz-utils/Cargo.toml index 630eea1b..8e998155 100644 --- a/crates/fuzz-utils/Cargo.toml +++ b/crates/fuzz-utils/Cargo.toml @@ -6,10 +6,12 @@ version = "0.1.0" publish = false [dependencies] -rand = { version = "0.7.0", features = ['small_rng'] } -tempfile = "3.1.0" anyhow = "1.0" env_logger = "0.7.0" +rand = { version = "0.7.0", features = ['small_rng'] } +tempfile = "3.1.0" +wasmparser = "0.44" +wat = "1.0" [dependencies.walrus] path = "../.." diff --git a/crates/fuzz-utils/src/lib.rs b/crates/fuzz-utils/src/lib.rs index baac8175..9452afdc 100755 --- a/crates/fuzz-utils/src/lib.rs +++ b/crates/fuzz-utils/src/lib.rs @@ -10,7 +10,7 @@ use std::fs; use std::marker::PhantomData; use std::path::Path; use std::time; -use walrus_tests_utils::{wasm_interp, wat2wasm}; +use walrus_tests_utils::wasm_interp; /// `Ok(T)` or a `Err(anyhow::Error)` pub type Result = std::result::Result; @@ -95,8 +95,7 @@ where } fn wat2wasm(&self, wat: &str) -> Result> { - fs::write(self.scratch.path(), wat).context("failed to write to scratch file")?; - wat2wasm(self.scratch.path(), &[]) + Ok(wat::parse_str(wat)?) } fn interp(&self, wasm: &[u8]) -> Result { @@ -416,22 +415,18 @@ impl TestCaseGenerator for WasmOptTtf { } }; - if { - // Only generate programs that wat2wasm can handle. - let tmp = tempfile::NamedTempFile::new().unwrap(); - fs::write(tmp.path(), &wat).unwrap(); - wat2wasm(tmp.path(), &[]).is_err() - } { - eprintln!( - "Warning: `wasm-opt -fff` generated wat that wat2wasm cannot handle; \ - skipping. wat =\n{}", - String::from_utf8_lossy(&wat) - ); - last_wat = Some(wat); - continue; + // Only generate programs that wat2wasm can handle. + if let Ok(bytes) = wat::parse_bytes(&wat) { + if wasmparser::validate(&bytes, None).is_ok() { + return String::from_utf8(wat).unwrap(); + } } - - return String::from_utf8(wat).unwrap(); + eprintln!( + "Warning: `wasm-opt -fff` generated wat that wat2wasm cannot handle; \ + skipping. wat =\n{}", + String::from_utf8_lossy(&wat) + ); + last_wat = Some(wat); } } } diff --git a/crates/tests-utils/src/lib.rs b/crates/tests-utils/src/lib.rs index e4b76a67..381879db 100644 --- a/crates/tests-utils/src/lib.rs +++ b/crates/tests-utils/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::{bail, Context}; use std::ffi::OsStr; use std::fs; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::{Command, Stdio}; use std::sync::Once; @@ -22,7 +22,7 @@ pub const FEATURES: &[&str] = &[ fn require_tool(tool: &str, repo: &str) { let diagnostic = format!("Could not spawn {}; do you have {} installed?", tool, repo); - let status = Command::new("wat2wasm") + let status = Command::new(tool) .arg("--help") .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -31,69 +31,8 @@ fn require_tool(tool: &str, repo: &str) { assert!(status.success(), "{}", diagnostic) } -fn require_wat2wasm() { - require_tool("wat2wasm", "https://github.com/WebAssembly/wabt"); -} - -/// Compile the `.wat` file at the given path into a `.wasm`. -pub fn wat2wasm(path: &Path, extra_args: &[&str]) -> Result> { - static CHECK: Once = Once::new(); - CHECK.call_once(require_wat2wasm); - - let file = tempfile::NamedTempFile::new().context("could not create named temp file")?; - - let mut wasm = PathBuf::from(path); - wasm.set_extension("wasm"); - - let mut cmd = Command::new("wat2wasm"); - cmd.arg(path) - .args(FEATURES) - .arg("--debug-names") - .arg("-o") - .arg(file.path()) - .args(extra_args); - println!("running: {:?}", cmd); - let output = cmd.output().context("could not spawn wat2wasm")?; - - if !output.status.success() { - bail!( - "wat2wasm exited with status {:?}\n\nstderr = '''\n{}\n'''", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - } - - let buf = fs::read(file.path())?; - Ok(buf) -} - -fn require_wasm2wat() { - require_tool("wasm2wat", "https://github.com/WebAssembly/wabt"); -} - -/// Disassemble the `.wasm` file at the given path into a `.wat`. -pub fn wasm2wat(path: &Path) -> Result { - static CHECK: Once = Once::new(); - CHECK.call_once(require_wasm2wat); - - let mut cmd = Command::new("wasm2wat"); - cmd.arg(path); - cmd.args(FEATURES); - println!("running: {:?}", cmd); - let output = cmd.output().context("could not run wasm2wat")?; - if !output.status.success() { - bail!( - "wasm2wat exited with status {:?}\n\nstderr = '''\n{}\n'''", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - } - - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - fn require_wasm_interp() { - require_tool("wasm-insterp", "https://github.com/WebAssembly/wabt"); + require_tool("wasm-interp", "https://github.com/WebAssembly/wabt"); } /// Run `wasm-interp` on the given wat file. diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml index 5855ee1f..1b108bf9 100644 --- a/crates/tests/Cargo.toml +++ b/crates/tests/Cargo.toml @@ -10,12 +10,14 @@ walkdir = "2.2.9" [dev-dependencies] anyhow = "1.0" +env_logger = "0.7.0" +serde = { version = "1.0.99", features = ['derive'] } +serde_json = { version = "1.0.40", features = ['preserve_order'] } +tempfile = "3.1.0" walrus = { path = "../.." } walrus-tests-utils = { path = "../tests-utils" } -tempfile = "3.1.0" -serde_json = { version = "1.0.40", features = ['preserve_order'] } -serde = { version = "1.0.99", features = ['derive'] } -env_logger = "0.7.0" +wasmprinter = "0.2" +wat = "1.0" [features] parallel = ['walrus/parallel'] diff --git a/crates/tests/tests/function_imports.rs b/crates/tests/tests/function_imports.rs index e1ab743d..4fa72eff 100644 --- a/crates/tests/tests/function_imports.rs +++ b/crates/tests/tests/function_imports.rs @@ -1,5 +1,4 @@ use std::path::Path; -use walrus_tests_utils::wat2wasm; fn run(wat_path: &Path) -> Result<(), anyhow::Error> { static INIT_LOGS: std::sync::Once = std::sync::Once::new(); @@ -7,7 +6,7 @@ fn run(wat_path: &Path) -> Result<(), anyhow::Error> { env_logger::init(); }); - let wasm = wat2wasm(wat_path, &[])?; + let wasm = wat::parse_file(wat_path)?; let module = walrus::Module::from_buffer(&wasm)?; assert!(module.imports.find("doggo", "husky").is_some()); diff --git a/crates/tests/tests/invalid.rs b/crates/tests/tests/invalid.rs index 5f97c7b8..3bdffcea 100644 --- a/crates/tests/tests/invalid.rs +++ b/crates/tests/tests/invalid.rs @@ -7,7 +7,7 @@ fn run(wat: &Path) -> Result<(), anyhow::Error> { env_logger::init(); }); - let wasm = walrus_tests_utils::wat2wasm(wat, &["--no-check"])?; + let wasm = wat::parse_file(wat)?; // NB: reading the module will do the validation. match walrus::Module::from_buffer(&wasm) { diff --git a/crates/tests/tests/round_trip.rs b/crates/tests/tests/round_trip.rs index 80154d19..7f0f57c6 100644 --- a/crates/tests/tests/round_trip.rs +++ b/crates/tests/tests/round_trip.rs @@ -1,6 +1,5 @@ use std::env; use std::path::Path; -use walrus_tests_utils::{wasm2wat, wat2wasm}; fn run(wat_path: &Path) -> Result<(), anyhow::Error> { static INIT_LOGS: std::sync::Once = std::sync::Once::new(); @@ -8,7 +7,7 @@ fn run(wat_path: &Path) -> Result<(), anyhow::Error> { env_logger::init(); }); - let wasm = wat2wasm(wat_path, &[])?; + let wasm = wat::parse_file(wat_path)?; let mut module = walrus::Module::from_buffer(&wasm)?; if env::var("WALRUS_TESTS_DOT").is_ok() { @@ -19,7 +18,7 @@ fn run(wat_path: &Path) -> Result<(), anyhow::Error> { walrus::passes::gc::run(&mut module); module.emit_wasm_file(&out_wasm_file)?; - let out_wat = wasm2wat(&out_wasm_file)?; + let out_wat = wasmprinter::print_file(&out_wasm_file)?; let checker = walrus_tests::FileCheck::from_file(wat_path); checker.check(&out_wat); Ok(()) diff --git a/crates/tests/tests/round_trip/simd.wat b/crates/tests/tests/round_trip/simd.wat index eecb260f..78cec071 100644 --- a/crates/tests/tests/round_trip/simd.wat +++ b/crates/tests/tests/round_trip/simd.wat @@ -554,27 +554,27 @@ (func $i8x16.replace_lane (type 11) (param v128 i32) (result v128) local.get 0 local.get 1 - i8x16.replace_lane 2 ) + i8x16.replace_lane 2) (func $i16x8.replace_lane (type 11) (param v128 i32) (result v128) local.get 0 local.get 1 - i16x8.replace_lane 2 ) + i16x8.replace_lane 2) (func $i32x4.replace_lane (type 11) (param v128 i32) (result v128) local.get 0 local.get 1 - i32x4.replace_lane 2 ) + i32x4.replace_lane 2) (func $i64x2.replace_lane (type 12) (param v128 i64) (result v128) local.get 0 local.get 1 - i64x2.replace_lane 0 ) + i64x2.replace_lane 0) (func $f32x4.replace_lane (type 13) (param v128 f32) (result v128) local.get 0 local.get 1 - f32x4.replace_lane 2 ) + f32x4.replace_lane 2) (func $f64x2.replace_lane (type 14) (param v128 f64) (result v128) local.get 0 local.get 1 - f64x2.replace_lane 0 ) + f64x2.replace_lane 0) (func $i8x16.eq (type 15) (param v128 v128) (result v128) local.get 0 local.get 1 @@ -915,43 +915,43 @@ i8x16.splat) (func $i8x16.extract_lane_s (type 6) (param v128) (result i32) local.get 0 - i8x16.extract_lane_s 1 ) + i8x16.extract_lane_s 1) (func $i8x16.extract_lane_u (type 6) (param v128) (result i32) local.get 0 - i8x16.extract_lane_u 2 ) + i8x16.extract_lane_u 2) (func $i16x8.splat (type 1) (param i32) (result v128) local.get 0 i16x8.splat) (func $i16x8.extract_lane_s (type 6) (param v128) (result i32) local.get 0 - i16x8.extract_lane_s 1 ) + i16x8.extract_lane_s 1) (func $i16x8.extract_lane_u (type 6) (param v128) (result i32) local.get 0 - i16x8.extract_lane_u 2 ) + i16x8.extract_lane_u 2) (func $i32x4.splat (type 1) (param i32) (result v128) local.get 0 i32x4.splat) (func $i32x4.extract_lane (type 6) (param v128) (result i32) local.get 0 - i32x4.extract_lane 1 ) + i32x4.extract_lane 1) (func $i64x2.splat (type 3) (param i64) (result v128) local.get 0 i64x2.splat) (func $i64x2.extract_lane (type 7) (param v128) (result i64) local.get 0 - i64x2.extract_lane 1 ) + i64x2.extract_lane 1) (func $f32x4.splat (type 4) (param f32) (result v128) local.get 0 f32x4.splat) (func $f32x4.extract_lane (type 8) (param v128) (result f32) local.get 0 - f32x4.extract_lane 1 ) + f32x4.extract_lane 1) (func $f64x2.splat (type 5) (param f64) (result v128) local.get 0 f64x2.splat) (func $f64x2.extract_lane (type 9) (param v128) (result f64) local.get 0 - f64x2.extract_lane 1 ) + f64x2.extract_lane 1) (func $v128.not (type 10) (param v128) (result v128) local.get 0 v128.not) diff --git a/crates/tests/tests/valid.rs b/crates/tests/tests/valid.rs index 7413a22c..fdb68ab0 100644 --- a/crates/tests/tests/valid.rs +++ b/crates/tests/tests/valid.rs @@ -8,7 +8,7 @@ fn run(wat: &Path) -> Result<(), anyhow::Error> { env_logger::init(); }); - let wasm = walrus_tests_utils::wat2wasm(wat, &[])?; + let wasm = wat::parse_file(wat)?; // NB: reading the module will do the validation. let module = walrus::Module::from_buffer(&wasm)?; diff --git a/src/module/functions/local_function/mod.rs b/src/module/functions/local_function/mod.rs index 560b09af..231db9cb 100644 --- a/src/module/functions/local_function/mod.rs +++ b/src/module/functions/local_function/mod.rs @@ -1195,7 +1195,7 @@ fn validate_instruction<'context>( ctx.alloc_instr(RefIsNull {}, loc); ctx.push_operand(Some(I32)); } - Operator::RefFunc { function_index }=> { + Operator::RefFunc { function_index } => { let func = ctx .indices .get_func(function_index) diff --git a/src/module/mod.rs b/src/module/mod.rs index e5caf12f..d4d83084 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -26,7 +26,7 @@ pub use crate::module::data::{ActiveData, ActiveDataLocation, Data, DataId, Data pub use crate::module::elements::{Element, ElementId, ModuleElements}; pub use crate::module::exports::{Export, ExportId, ExportItem, ModuleExports}; pub use crate::module::functions::{Function, FunctionId, ModuleFunctions}; -pub use crate::module::functions::{FunctionKind, LocalFunction, ImportedFunction}; +pub use crate::module::functions::{FunctionKind, ImportedFunction, LocalFunction}; pub use crate::module::globals::{Global, GlobalId, GlobalKind, ModuleGlobals}; pub use crate::module::imports::{Import, ImportId, ImportKind, ModuleImports}; pub use crate::module::locals::ModuleLocals; From a5ee7cd26c73590981fd687b95952213b4959d5a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 5 Dec 2019 12:16:09 -0800 Subject: [PATCH 2/2] Migrate to Github Actions from Travis CI --- .github/workflows/main.yml | 86 ++++++++++++++++++++++++++++++ .travis.yml | 106 ------------------------------------- 2 files changed, 86 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..f5650ef7 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,86 @@ +name: CI +on: [push, pull_request] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable, beta, nightly] + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update ${{ matrix.rust }} && rustup default ${{ matrix.rust }} + - name: Install wabt + run: | + set -e + curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.12/wabt-1.0.12-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/wabt-1.0.12" + - name: Install binaryen + run: | + set -e + curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/binaryen-1.39.1" + - run: cargo build --all + - run: cargo test --all + - run: cargo check --benches + - run: cargo test --features parallel + - run: cargo test --features parallel --manifest-path crates/tests/Cargo.toml + + fuzz_crate: + name: Fuzz Crate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update stable && rustup default stable + - name: Install wabt + run: | + set -e + curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.12/wabt-1.0.12-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/wabt-1.0.12" + - name: Install binaryen + run: | + set -e + curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/binaryen-1.39.1" + - name: Run fuzzer + run: cargo test -p walrus-fuzz-utils > fuzz.log || (tail -n 1000 fuzz.log && exit 1) + env: + # 300 seconds = 5 minutes. + WALRUS_FUZZ_TIMEOUT: 300 + + fuzz: + name: Fuzz + runs-on: ubuntu-latest + strategy: + matrix: + test: [watgen, wasm-opt-ttf, raw] + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update nightly && rustup default nightly + - run: cargo install cargo-fuzz + - name: Install wabt + run: | + set -e + curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.12/wabt-1.0.12-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/wabt-1.0.12" + - name: Install binaryen + run: | + set -e + curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf - + echo "##[add-path]`pwd`/binaryen-1.39.1" + - name: Run fuzzer + run: | + cargo fuzz run ${{ matrix.test }} -- -max_total_time=300 -rss_limit_mb=4096 > fuzz.log 2>&1 || (tail -n 1000 fuzz.log && exit 1) + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update stable && rustup default stable && rustup component add rustfmt + - run: cargo fmt -- --check diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 090b9430..00000000 --- a/.travis.yml +++ /dev/null @@ -1,106 +0,0 @@ -language: rust -sudo: false - -before_install: - - target=x86_64-unknown-linux-musl - - curl -L https://github.com/mozilla/sccache/releases/download/0.2.7/sccache-0.2.7-$target.tar.gz | tar xzf - - - export PATH=$PATH:`pwd`/sccache-0.2.7-$target - - export RUSTC_WRAPPER=sccache - -before_script: - - export RUST_BACKTRACE=1 - -after_script: - - sccache -s - -install: - # Build WABT - - git clone --recursive https://github.com/WebAssembly/wabt.git - - mkdir wabt/build - - cd wabt/build - - cmake .. -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - - make -j 8 - - cd - - - export PATH=$PATH:$(pwd)/wabt/build - - which wasm2wat - - which wat2wasm - # Build binaryen - - git clone --recursive https://github.com/WebAssembly/binaryen.git - - mkdir binaryen/build - - cd binaryen/build - - cmake .. -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - - make -j 8 - - cd - - - export PATH=$PATH:$(pwd)/binaryen/build/bin - - which wasm-opt - -matrix: - include: - # First up: some jobs to run our fuzzers a bit longer than the default when - # we do `cargo test`. - # - # We run these first because they take the longest. - - name: "fuzz-utils (stable)" - rust: stable - env: - # 300 seconds = 5 minutes. - - WALRUS_FUZZ_TIMEOUT=300 - script: - | - # When the fuzzing fails, the logs are too big for Travis, so just - # show the relevant tail portion of the log. - cargo test -p walrus-fuzz-utils > fuzz.log || { - tail -n 1000 fuzz.log && exit 1 - } - - name: "cargo fuzz run watgen (nightly)" - rust: nightly - script: - | - which cargo-fuzz || cargo install cargo-fuzz - # When the fuzzing fails, the logs are too big for Travis, so just - # show the relevant tail portion of the log. - cargo fuzz run watgen -- -max_total_time=300 > fuzz.log 2>&1 || { - tail -n 1000 fuzz.log && exit 1 - } - - name: "cargo fuzz run wasm-opt-ttf (nightly)" - rust: nightly - script: - | - which cargo-fuzz || cargo install cargo-fuzz - # When the fuzzing fails, the logs are too big for Travis, so just - # show the relevant tail portion of the log. - cargo fuzz run wasm-opt-ttf -- -max_total_time=300 > fuzz.log 2>&1 || { - tail -n 1000 fuzz.log && exit 1 - } - - name: "cargo fuzz run raw (nightly)" - rust: nightly - script: - | - which cargo-fuzz || cargo install cargo-fuzz - # When the fuzzing fails, the logs are too big for Travis, so just - # show the relevant tail portion of the log. - cargo fuzz run raw -- -max_total_time=300 -rss_limit_mb=4096 > fuzz.log 2>&1 || { - tail -n 1000 fuzz.log && exit 1 - } - - - name: "test (stable)" - rust: stable - - name: "test (beta)" - rust: beta - - name: "test (nightly)" - rust: nightly - - - name: "check benches" - rust: stable - script: - - cargo check --benches - -script: - - cargo build --all - - cargo test --all - - cargo test --features parallel - - cargo test --features parallel --manifest-path crates/tests/Cargo.toml - -notifications: - email: - on_success: never