Merge branch 'main' into 4615-new-publish-flow

lib/cli/src/commands/package/build.rs

@@ -3,6 +3,7 @@ use std::path::PathBuf;
 use anyhow::Context;
 use dialoguer::console::{style, Emoji};
 use indicatif::ProgressBar;
+use sha2::Digest;
 use wasmer_config::package::PackageHash;
 use webc::wasmer_package::Package;
 
@@ -54,9 +55,10 @@ impl PackageBuild {
             )
         };
         let pkg = webc::wasmer_package::Package::from_manifest(manifest_path)?;
-        let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash().ok_or(anyhow::anyhow!(
-            "No webc hash was provided while trying to build"
-        ))?);
+        let data = pkg.serialize()?;
+        let hash = sha2::Sha256::digest(&data).into();
+        let pkg_hash = PackageHash::from_sha256_bytes(hash);
+
         let name = if let Some(manifest_pkg) = manifest.package {
             if let Some(name) = manifest_pkg.name {
                 if let Some(version) = manifest_pkg.version {
@@ -119,8 +121,6 @@ impl PackageBuild {
             );
         }
 
-        let data = pkg.serialize()?;
-
         pb.println(format!(
             "{} {}Writing package...",
             style("[3/3]").bold().dim(),

lib/cli/src/commands/run/mod.rs

@@ -673,15 +673,12 @@ impl ExecutableTarget {
         pb: &ProgressBar,
     ) -> Result<Self, Error> {
         pb.set_message(format!("Loading \"{}\" into memory", dir.display()));
-
-        let manifest_path = dir.join("wasmer.toml");
-        let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?;
-        let container = Container::from(webc);
-
         pb.set_message("Resolving dependencies");
         let inner_runtime = runtime.clone();
-        let pkg = runtime.task_manager().spawn_and_block_on(async move {
-            BinaryPackage::from_webc(&container, inner_runtime.as_ref()).await
+        let pkg = runtime.task_manager().spawn_and_block_on({
+            let path = dir.to_path_buf();
+
+            async move { BinaryPackage::from_dir(&path, inner_runtime.as_ref()).await }
         })??;
 
         Ok(ExecutableTarget::Package(pkg))
@@ -707,7 +704,7 @@ impl ExecutableTarget {
 
                 Ok(ExecutableTarget::WebAssembly {
                     module,
-                    module_hash: ModuleHash::hash(&wasm),
+                    module_hash: ModuleHash::xxhash(&wasm),
                     path: path.to_path_buf(),
                 })
             }
@@ -717,7 +714,7 @@ impl ExecutableTarget {
                 let module = unsafe { Module::deserialize_from_file(&engine, path)? };
                 let module_hash = {
                     let wasm = std::fs::read(path)?;
-                    ModuleHash::hash(wasm)
+                    ModuleHash::xxhash(wasm)
                 };
 
                 Ok(ExecutableTarget::WebAssembly {

lib/journal/src/concrete/archived.rs

@@ -950,7 +950,7 @@ pub enum ArchivedJournalEntry<'a> {
 #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)]
 #[archive_attr(derive(CheckBytes), repr(align(8)))]
 pub struct JournalEntryInitModuleV1 {
-    pub wasm_hash: [u8; 8],
+    pub wasm_hash: Box<[u8]>,
 }
 
 #[repr(C)]

lib/journal/src/concrete/archived_from.rs

@@ -649,7 +649,7 @@ impl<'a> TryFrom<ArchivedJournalEntry<'a>> for JournalEntry<'a> {
         Ok(match value {
             ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => {
                 Self::InitModuleV1 {
-                    wasm_hash: *wasm_hash,
+                    wasm_hash: Box::from(wasm_hash.get()),
                 }
             }
             ArchivedJournalEntry::ClearEtherealV1(ArchivedJournalEntryClearEtherealV1 {

lib/journal/src/concrete/tests.rs

@@ -49,7 +49,7 @@ pub fn run_test<'a>(record: JournalEntry<'a>) {
 #[test]
 pub fn test_record_init_module() {
     run_test(JournalEntry::InitModuleV1 {
-        wasm_hash: [13u8; 8],
+        wasm_hash: Box::new([13u8; 8]),
     });
 }
 

lib/journal/src/entry.rs

@@ -84,7 +84,7 @@ pub enum SocketOptTimeType {
 #[serde(rename_all = "snake_case")]
 pub enum JournalEntry<'a> {
     InitModuleV1 {
-        wasm_hash: [u8; 8],
+        wasm_hash: Box<[u8]>,
     },
     ClearEtherealV1,
     UpdateMemoryRegionV1 {

lib/wasix/src/bin_factory/binary_package.rs

@@ -1,8 +1,9 @@
-use std::sync::Arc;
+use std::{path::Path, sync::Arc};
 
 use anyhow::Context;
 use derivative::*;
 use once_cell::sync::OnceCell;
+use sha2::Digest;
 use virtual_fs::FileSystem;
 use wasmer_config::package::{PackageHash, PackageId, PackageSource};
 use webc::{compat::SharedBytes, Container};
@@ -22,16 +23,21 @@ pub struct BinaryPackageCommand {
     metadata: webc::metadata::Command,
     #[derivative(Debug = "ignore")]
     pub(crate) atom: SharedBytes,
-    hash: OnceCell<ModuleHash>,
+    hash: ModuleHash,
 }
 
 impl BinaryPackageCommand {
-    pub fn new(name: String, metadata: webc::metadata::Command, atom: SharedBytes) -> Self {
+    pub fn new(
+        name: String,
+        metadata: webc::metadata::Command,
+        atom: SharedBytes,
+        hash: ModuleHash,
+    ) -> Self {
         Self {
             name,
             metadata,
             atom,
-            hash: OnceCell::new(),
+            hash,
         }
     }
 
@@ -52,7 +58,7 @@ impl BinaryPackageCommand {
     }
 
     pub fn hash(&self) -> &ModuleHash {
-        self.hash.get_or_init(|| ModuleHash::hash(self.atom()))
+        &self.hash
     }
 }
 
@@ -74,6 +80,36 @@ pub struct BinaryPackage {
 }
 
 impl BinaryPackage {
+    #[tracing::instrument(level = "debug", skip_all)]
+    pub async fn from_dir(
+        dir: &Path,
+        rt: &(dyn Runtime + Send + Sync),
+    ) -> Result<Self, anyhow::Error> {
+        let source = rt.source();
+
+        // since each package must be in its own directory, hash of the `dir` should provide a good enough
+        // unique identifier for the package
+        let hash = sha2::Sha256::digest(dir.display().to_string().as_bytes()).into();
+        let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash));
+
+        let manifest_path = dir.join("wasmer.toml");
+        let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?;
+        let container = Container::from(webc);
+        let manifest = container.manifest();
+
+        let root = PackageInfo::from_manifest(id, manifest, container.version())?;
+        let root_id = root.id.clone();
+
+        let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?;
+        let pkg = rt
+            .package_loader()
+            .load_package_tree(&container, &resolution)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))?;
+
+        Ok(pkg)
+    }
+
     /// Load a [`webc::Container`] and all its dependencies into a
     /// [`BinaryPackage`].
     #[tracing::instrument(level = "debug", skip_all)]
@@ -84,14 +120,13 @@ impl BinaryPackage {
         let source = rt.source();
 
         let manifest = container.manifest();
-        let id = match PackageInfo::package_id_from_manifest(manifest)? {
-            Some(id) => id,
-            None => PackageId::Hash(PackageHash::from_sha256_bytes(
+        let id = PackageInfo::package_id_from_manifest(manifest)?
+            .or_else(|| {
                 container
                     .webc_hash()
-                    .ok_or(anyhow::anyhow!("No webc hash was provided"))?,
-            )),
-        };
+                    .map(|hash| PackageId::Hash(PackageHash::from_sha256_bytes(hash)))
+            })
+            .ok_or_else(|| anyhow::Error::msg("webc file did not provide its hash"))?;
 
         let root = PackageInfo::from_manifest(id, manifest, container.version())?;
         let root_id = root.id.clone();
@@ -151,16 +186,17 @@ impl BinaryPackage {
     pub fn hash(&self) -> ModuleHash {
         *self.hash.get_or_init(|| {
             if let Some(entry) = self.entrypoint_bytes() {
-                ModuleHash::hash(entry)
+                ModuleHash::xxhash(entry)
             } else {
-                ModuleHash::hash(self.id.to_string())
+                ModuleHash::xxhash(self.id.to_string())
             }
         })
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use sha2::Digest;
     use tempfile::TempDir;
     use virtual_fs::AsyncReadExt;
 
@@ -203,17 +239,16 @@ mod tests {
         std::fs::create_dir_all(&out).unwrap();
         let file_txt = "Hello, World!";
         std::fs::write(out.join("file.txt"), file_txt).unwrap();
-        let webc: Container = webc::wasmer_package::Package::from_manifest(manifest)
-            .unwrap()
-            .into();
         let tasks = task_manager();
         let mut runtime = PluggableRuntime::new(tasks);
         runtime.set_package_loader(
             BuiltinPackageLoader::new()
                 .with_shared_http_client(runtime.http_client().unwrap().clone()),
         );
 
-        let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap();
+        let pkg = BinaryPackage::from_dir(&temp.path(), &runtime)
+            .await
+            .unwrap();
 
         // We should have mapped "./out/file.txt" on the host to
         // "/public/file.txt" on the guest.
@@ -227,4 +262,54 @@ mod tests {
         f.read_to_string(&mut buffer).await.unwrap();
         assert_eq!(buffer, file_txt);
     }
+
+    #[tokio::test]
+    #[cfg_attr(
+        not(feature = "sys-thread"),
+        ignore = "The tokio task manager isn't available on this platform"
+    )]
+    async fn commands_use_the_atom_signature() {
+        let temp = TempDir::new().unwrap();
+        let wasmer_toml = r#"
+            [package]
+            name = "some/package"
+            version = "0.0.0"
+            description = "a dummy package"
+
+            [[module]]
+            name = "foo"
+            source = "foo.wasm"
+            abi = "wasi"
+            
+            [[command]]
+            name = "cmd"
+            module = "foo"     
+        "#;
+        let manifest = temp.path().join("wasmer.toml");
+        std::fs::write(&manifest, wasmer_toml).unwrap();
+
+        let atom_path = temp.path().join("foo.wasm");
+        std::fs::write(&atom_path, b"").unwrap();
+
+        let webc: Container = webc::wasmer_package::Package::from_manifest(&manifest)
+            .unwrap()
+            .into();
+
+        let tasks = task_manager();
+        let mut runtime = PluggableRuntime::new(tasks);
+        runtime.set_package_loader(
+            BuiltinPackageLoader::new()
+                .with_shared_http_client(runtime.http_client().unwrap().clone()),
+        );
+
+        let pkg = BinaryPackage::from_dir(&temp.path(), &runtime)
+            .await
+            .unwrap();
+
+        assert_eq!(pkg.commands.len(), 1);
+        let command = pkg.get_command("cmd").unwrap();
+        let atom_sha256_hash: [u8; 32] = sha2::Sha256::digest(webc.get_atom("foo").unwrap()).into();
+        let module_hash = ModuleHash::sha256_from_bytes(atom_sha256_hash);
+        assert_eq!(command.hash(), &module_hash);
+    }
 }

lib/wasix/src/os/task/control_plane.rs

@@ -225,7 +225,7 @@ mod tests {
             enable_exponential_cpu_backoff: None,
         });
 
-        let p1 = p.new_process(ModuleHash::random()).unwrap();
+        let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap();
         let _t1 = p1
             .new_thread(WasiMemoryLayout::default(), ThreadStartType::MainThread)
             .unwrap();
@@ -234,7 +234,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(
-            p.new_process(ModuleHash::random()).unwrap_err(),
+            p.new_process(ModuleHash::xxhash_random()).unwrap_err(),
             ControlPlaneError::TaskLimitReached { max: 2 }
         );
     }
@@ -248,7 +248,7 @@ mod tests {
             enable_exponential_cpu_backoff: None,
         });
 
-        let p1 = p.new_process(ModuleHash::random()).unwrap();
+        let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap();
 
         for _ in 0..10 {
             let _thread = p1
@@ -264,7 +264,7 @@ mod tests {
             .unwrap();
 
         assert_eq!(
-            p.new_process(ModuleHash::random()).unwrap_err(),
+            p.new_process(ModuleHash::xxhash_random()).unwrap_err(),
             ControlPlaneError::TaskLimitReached { max: 2 }
         );
     }

lib/wasix/src/runtime/mod.rs

@@ -104,7 +104,7 @@ where
     fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result<Module, SpawnError>> {
         let engine = self.engine();
         let module_cache = self.module_cache();
-        let hash = ModuleHash::hash(wasm);
+        let hash = ModuleHash::xxhash(wasm);
 
         let task = async move { load_module(&engine, &module_cache, wasm, hash).await };
 
@@ -553,7 +553,7 @@ impl Runtime for OverriddenRuntime {
         if self.engine.is_some() || self.module_cache.is_some() {
             let engine = self.engine();
             let module_cache = self.module_cache();
-            let hash = ModuleHash::hash(wasm);
+            let hash = ModuleHash::xxhash(wasm);
 
             let task = async move { load_module(&engine, &module_cache, wasm, hash).await };
             Box::pin(task)

lib/wasix/src/runtime/module_cache/fallback.rs

@@ -179,7 +179,7 @@ mod tests {
     async fn load_from_primary() {
         let engine = Engine::default();
         let module = Module::new(&engine, ADD_WAT).unwrap();
-        let key = ModuleHash::from_bytes([0; 8]);
+        let key = ModuleHash::xxhash_from_bytes([0; 8]);
         let primary = SharedCache::default();
         let fallback = SharedCache::default();
         primary.save(key, &engine, &module).await.unwrap();
@@ -204,7 +204,7 @@ mod tests {
     async fn loading_from_fallback_also_populates_primary() {
         let engine = Engine::default();
         let module = Module::new(&engine, ADD_WAT).unwrap();
-        let key = ModuleHash::from_bytes([0; 8]);
+        let key = ModuleHash::xxhash_from_bytes([0; 8]);
         let primary = SharedCache::default();
         let fallback = SharedCache::default();
         fallback.save(key, &engine, &module).await.unwrap();
@@ -230,7 +230,7 @@ mod tests {
     async fn saving_will_update_both() {
         let engine = Engine::default();
         let module = Module::new(&engine, ADD_WAT).unwrap();
-        let key = ModuleHash::from_bytes([0; 8]);
+        let key = ModuleHash::xxhash_from_bytes([0; 8]);
         let primary = SharedCache::default();
         let fallback = SharedCache::default();
         let cache = FallbackCache::new(&primary, &fallback);