Use trampolines for all libcalls in engine-universal and engine-dylib

In both of these engines, the compiled code may be loaded in memory far
from the Wasmer runtime which means that libcalls may not be reachable
through the normal relocation types. Instead a trampoline is needed to
allow reaching any address in the 64-bit address space.

In the case of engine-dylib, this is even worse since the symbols are
not exported by the executable without some special linker flags. The
solution here is to manually patch in the addresses at load time into
a data table of function pointers.

.github/workflows/test-sys.yaml

@@ -166,6 +166,7 @@ jobs:
         shell: bash
       - name: Setup Rust target
         run: |
+          mkdir -p .cargo
           cat << EOF > .cargo/config.toml
           [build]
           target = "${{ matrix.target }}"

lib/compiler-cranelift/src/compiler.rs

@@ -295,7 +295,6 @@ impl Compiler for CraneliftCompiler {
             function_call_trampolines,
             dynamic_function_trampolines,
             dwarf,
-            None,
         ))
     }
 }

lib/compiler-cranelift/src/config.rs

@@ -135,6 +135,12 @@ impl Cranelift {
             flags.enable("is_pic").expect("should be a valid flag");
         }
 
+        // We set up libcall trampolines in engine-dylib and engine-universal.
+        // These trampolines are always reachable through short jumps.
+        flags
+            .enable("use_colocated_libcalls")
+            .expect("should be a valid flag");
+
         // Invert cranelift's default-on verification to instead default off.
         let enable_verifier = if self.enable_verifier {
             "true"

lib/compiler-cranelift/src/func_environ.rs

@@ -1132,9 +1132,7 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro
         Ok(func.import_function(ir::ExtFuncData {
             name,
             signature,
-            // We currently allocate all code segments independently, so nothing
-            // is colocated.
-            colocated: false,
+            colocated: true,
         }))
     }
 

lib/compiler-cranelift/src/translator/translation_utils.rs

@@ -90,6 +90,7 @@ pub fn irreloc_to_relocationkind(reloc: Reloc) -> RelocationKind {
         Reloc::X86CallPCRel4 => RelocationKind::X86CallPCRel4,
         Reloc::X86CallPLTRel4 => RelocationKind::X86CallPLTRel4,
         Reloc::X86GOTPCRel4 => RelocationKind::X86GOTPCRel4,
+        Reloc::Arm64Call => RelocationKind::Arm64Call,
         _ => panic!("The relocation {} is not yet supported.", reloc),
     }
 }

lib/compiler-llvm/src/compiler.rs

@@ -12,10 +12,9 @@ use rayon::iter::ParallelBridge;
 use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
 use std::sync::Arc;
 use wasmer_compiler::{
-    Architecture, Compilation, CompileError, CompileModuleInfo, Compiler, CustomSection,
-    CustomSectionProtection, Dwarf, FunctionBodyData, ModuleMiddleware, ModuleTranslationState,
-    RelocationTarget, SectionBody, SectionIndex, Symbol, SymbolRegistry, Target,
-    TrampolinesSection,
+    Compilation, CompileError, CompileModuleInfo, Compiler, CustomSection, CustomSectionProtection,
+    Dwarf, FunctionBodyData, ModuleMiddleware, ModuleTranslationState, RelocationTarget,
+    SectionBody, SectionIndex, Symbol, SymbolRegistry, Target,
 };
 use wasmer_types::entity::{EntityRef, PrimaryMap};
 use wasmer_types::{FunctionIndex, LocalFunctionIndex, SignatureIndex};
@@ -305,37 +304,6 @@ impl Compiler for LLVMCompiler {
             })
             .collect::<PrimaryMap<LocalFunctionIndex, _>>();
 
-        let trampolines = match target.triple().architecture {
-            Architecture::Aarch64(_) => {
-                let nj = 16;
-                // We create a jump to an absolute 64bits address
-                // using x17 as a scratch register, SystemV declare both x16 and x17 as Intra-Procedural  scratch register
-                // but Apple ask to just not use x16
-                // LDR x17, #8      51 00 00 58
-                // BR x17           20 02 1f d6
-                // JMPADDR          00 00 00 00 00 00 00 00
-                let onejump = [
-                    0x51, 0x00, 0x00, 0x58, 0x20, 0x02, 0x1f, 0xd6, 0, 0, 0, 0, 0, 0, 0, 0,
-                ];
-                let trampolines = Some(TrampolinesSection::new(
-                    SectionIndex::from_u32(module_custom_sections.len() as u32),
-                    nj,
-                    onejump.len(),
-                ));
-                let mut alljmps = vec![];
-                for _ in 0..nj {
-                    alljmps.extend(onejump.iter().copied());
-                }
-                module_custom_sections.push(CustomSection {
-                    protection: CustomSectionProtection::ReadExecute,
-                    bytes: SectionBody::new_with_vec(alljmps),
-                    relocations: vec![],
-                });
-                trampolines
-            }
-            _ => None,
-        };
-
         let dwarf = if !frame_section_bytes.is_empty() {
             let dwarf = Some(Dwarf::new(SectionIndex::from_u32(
                 module_custom_sections.len() as u32,
@@ -400,7 +368,6 @@ impl Compiler for LLVMCompiler {
             function_call_trampolines,
             dynamic_function_trampolines,
             dwarf,
-            trampolines,
         ))
     }
 }

lib/compiler-singlepass/src/compiler.rs

@@ -187,7 +187,6 @@ impl Compiler for SinglepassCompiler {
             function_call_trampolines,
             dynamic_function_trampolines,
             None,
-            None,
         ))
     }
 }

lib/compiler/src/function.rs

@@ -109,33 +109,6 @@ impl Dwarf {
     }
 }
 
-/// Trampolines section used by ARM short jump (26bits)
-#[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
-#[cfg_attr(
-    feature = "enable-rkyv",
-    derive(RkyvSerialize, RkyvDeserialize, Archive)
-)]
-#[derive(Debug, PartialEq, Eq, Clone, MemoryUsage)]
-pub struct TrampolinesSection {
-    /// SectionIndex for the actual Trampolines code
-    pub section_index: SectionIndex,
-    /// Number of jump slots in the section
-    pub slots: usize,
-    /// Slot size
-    pub size: usize,
-}
-
-impl TrampolinesSection {
-    /// Creates a `Trampolines` struct with the indice for its section, and number of slots and size of slot
-    pub fn new(section_index: SectionIndex, slots: usize, size: usize) -> Self {
-        Self {
-            section_index,
-            slots,
-            size,
-        }
-    }
-}
-
 /// The result of compiling a WebAssembly module's functions.
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
 #[derive(Debug, PartialEq, Eq)]
@@ -182,9 +155,6 @@ pub struct Compilation {
 
     /// Section ids corresponding to the Dwarf debug info
     debug: Option<Dwarf>,
-
-    /// Trampolines for the arch that needs it
-    trampolines: Option<TrampolinesSection>,
 }
 
 impl Compilation {
@@ -195,15 +165,13 @@ impl Compilation {
         function_call_trampolines: PrimaryMap<SignatureIndex, FunctionBody>,
         dynamic_function_trampolines: PrimaryMap<FunctionIndex, FunctionBody>,
         debug: Option<Dwarf>,
-        trampolines: Option<TrampolinesSection>,
     ) -> Self {
         Self {
             functions,
             custom_sections,
             function_call_trampolines,
             dynamic_function_trampolines,
             debug,
-            trampolines,
         }
     }
 
@@ -281,11 +249,6 @@ impl Compilation {
     pub fn get_debug(&self) -> Option<Dwarf> {
         self.debug.clone()
     }
-
-    /// Returns the Trampilines info.
-    pub fn get_trampolines(&self) -> Option<TrampolinesSection> {
-        self.trampolines.clone()
-    }
 }
 
 impl<'a> IntoIterator for &'a Compilation {

lib/compiler/src/lib.rs

@@ -74,7 +74,7 @@ pub use crate::error::{
 };
 pub use crate::function::{
     Compilation, CompiledFunction, CompiledFunctionFrameInfo, CustomSections, Dwarf, FunctionBody,
-    Functions, TrampolinesSection,
+    Functions,
 };
 pub use crate::jump_table::{JumpTable, JumpTableOffsets};
 pub use crate::module::CompileModuleInfo;

lib/engine-dylib/Cargo.toml

@@ -26,6 +26,8 @@ which = "4.0"
 rkyv = "0.7.20"
 loupe = "0.1"
 enumset = "1.0"
+enum-iterator = "0.7.0"
+object = { version = "0.27", default-features = false, features = ["write"] }
 
 [features]
 # Enable the `compiler` feature if you want the engine to compile

lib/engine-dylib/src/artifact.rs

@@ -3,6 +3,7 @@
 
 use crate::engine::{DylibEngine, DylibEngineInner};
 use crate::serialize::ModuleMetadata;
+use crate::trampoline::{emit_trampolines, fill_trampoline_table, WASMER_TRAMPOLINES_SYMBOL};
 use enumset::EnumSet;
 use libloading::{Library, Symbol as LibrarySymbol};
 use loupe::MemoryUsage;
@@ -232,8 +233,28 @@ impl DylibArtifact {
             &metadata_binary,
         );
 
+        let mut extra_filepath = None;
         let filepath = match maybe_obj_bytes {
             Some(obj_bytes) => {
+                extra_filepath = {
+                    // Create a separate object file with the trampolines.
+                    let mut obj =
+                        get_object_for_target(&target_triple).map_err(to_compile_error)?;
+                    emit_trampolines(&mut obj, engine.target());
+                    let file = tempfile::Builder::new()
+                        .prefix("wasmer_dylib_")
+                        .suffix(".o")
+                        .tempfile()
+                        .map_err(to_compile_error)?;
+
+                    // Re-open it.
+                    let (mut file, filepath) = file.keep().map_err(to_compile_error)?;
+                    let obj_bytes = obj.write().map_err(to_compile_error)?;
+                    file.write_all(&obj_bytes).map_err(to_compile_error)?;
+                    Some(filepath)
+                };
+
+                // Write the object file generated by the compiler.
                 let obj_bytes = obj_bytes?;
                 let file = tempfile::Builder::new()
                     .prefix("wasmer_dylib_")
@@ -254,6 +275,7 @@ impl DylibArtifact {
                     function_body_inputs,
                 )?;
                 let mut obj = get_object_for_target(&target_triple).map_err(to_compile_error)?;
+                emit_trampolines(&mut obj, engine.target());
                 emit_data(
                     &mut obj,
                     WASMER_METADATA_SYMBOL,
@@ -371,6 +393,7 @@ impl DylibArtifact {
         let linker = engine_inner.linker().executable();
         let output = Command::new(linker)
             .arg(&filepath)
+            .args(&extra_filepath)
             .arg("-o")
             .arg(&output_filepath)
             .args(&target_args)
@@ -385,6 +408,11 @@ impl DylibArtifact {
         if fs::metadata(&filepath).is_ok() {
             fs::remove_file(filepath).map_err(to_compile_error)?;
         }
+        if let Some(filepath) = extra_filepath {
+            if fs::metadata(&filepath).is_ok() {
+                fs::remove_file(filepath).map_err(to_compile_error)?;
+            }
+        }
 
         let output = output?;
 
@@ -621,7 +649,7 @@ impl DylibArtifact {
             DeserializeError::CorruptedBinary(format!("Library loading failed: {}", e))
         })?;
         let shared_path: PathBuf = PathBuf::from(path);
-        let symbol: LibrarySymbol<*mut [u8; MetadataHeader::LEN]> =
+        let metadata_symbol: LibrarySymbol<*mut [u8; MetadataHeader::LEN]> =
             lib.get(WASMER_METADATA_SYMBOL).map_err(|e| {
                 DeserializeError::CorruptedBinary(format!(
                     "The provided object file doesn't seem to be generated by Wasmer: {}",
@@ -630,13 +658,22 @@ impl DylibArtifact {
             })?;
         use std::slice;
 
-        let metadata = &**symbol;
+        let metadata = &**metadata_symbol;
         let metadata_len = MetadataHeader::parse(metadata)?;
         let metadata_slice: &'static [u8] =
             slice::from_raw_parts(metadata.as_ptr().add(MetadataHeader::LEN), metadata_len);
 
         let metadata = ModuleMetadata::deserialize(metadata_slice)?;
 
+        let trampolines_symbol: LibrarySymbol<usize> =
+            lib.get(WASMER_TRAMPOLINES_SYMBOL).map_err(|e| {
+                DeserializeError::CorruptedBinary(format!(
+                    "The provided object file doesn't seem to be generated by Wasmer: {}",
+                    e
+                ))
+            })?;
+        fill_trampoline_table(trampolines_symbol.into_raw().into_raw() as *mut usize);
+
         let mut engine_inner = engine.inner_mut();
 
         Self::from_parts(&mut engine_inner, metadata, shared_path, lib)

lib/engine-dylib/src/lib.rs

@@ -27,6 +27,7 @@ mod artifact;
 mod builder;
 mod engine;
 mod serialize;
+mod trampoline;
 
 pub use crate::artifact::DylibArtifact;
 pub use crate::builder::Dylib;