From c01f7908d0209fbafa62e8679a210c7eb64d5c44 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 5 Dec 2025 12:51:34 -0800 Subject: [PATCH] Optimize runtime of `test-artifacts` build script * Convert core wasms to components in parallel * Skip the core wasm to component part if the artifacts are up-to-date (based on mtime) * Build the C/C++ programs in parallel. The goal here is to pick some low-hanging fruit to prevent this from being such a bottleneck in local development, but there's more that can be done if necessary (e.g. reading the dep files and calculating that all manually). Right now for example the longer steps are invoking Cargo which does nothing and invoking the C/C++ compilers unconditionally, but solving that makes this more of a "build a build system" script and I feel like we haven't quite crossed that threshold yet. --- Cargo.lock | 1 + crates/test-programs/artifacts/Cargo.toml | 1 + crates/test-programs/artifacts/build.rs | 147 +++++++++++++++------- 3 files changed, 103 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6b26aaa8fe0..c25835df7296 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,6 +3803,7 @@ version = "0.0.0" dependencies = [ "cargo_metadata", "heck 0.5.0", + "rayon", "serde", "serde_derive", "wasmtime", diff --git a/crates/test-programs/artifacts/Cargo.toml b/crates/test-programs/artifacts/Cargo.toml index 6481b4d44247..d838499ec786 100644 --- a/crates/test-programs/artifacts/Cargo.toml +++ b/crates/test-programs/artifacts/Cargo.toml @@ -21,3 +21,4 @@ cargo_metadata = "0.19.2" wasmtime-test-util = { workspace = true, features = ['wast'] } serde_derive = { workspace = true } serde = { workspace = true } +rayon = { workspace = true } diff --git a/crates/test-programs/artifacts/build.rs b/crates/test-programs/artifacts/build.rs index cb9467aa882f..37b632c9412b 100644 --- a/crates/test-programs/artifacts/build.rs +++ b/crates/test-programs/artifacts/build.rs @@ -1,9 +1,11 @@ use heck::*; +use rayon::prelude::*; use std::collections::{BTreeMap, HashSet}; use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::SystemTime; use wit_component::ComponentEncoder; fn main() { @@ -62,6 +64,7 @@ impl Artifacts { let mut kinds = BTreeMap::new(); let missing_sdk_path = PathBuf::from("Asset not compiled, WASI_SDK_PATH missing at compile time"); + let mut components = Vec::new(); for test in tests.iter() { let shouty_snake = test.name.to_shouty_snake_case(); let snake = test.name.to_snake_case(); @@ -122,14 +125,20 @@ impl Artifacts { { continue; } - let adapter = match test.name.as_str() { + let (adapter, mtime) = match test.name.as_str() { "reactor" => &reactor_adapter, s if s.starts_with("p3_") => &reactor_adapter, s if s.starts_with("p2_api_proxy") => &proxy_adapter, _ => &command_adapter, }; let path = match &test.core_wasm { - Some(path) => self.compile_component(path, adapter), + Some(path) => { + let out_dir = path.parent().unwrap(); + let stem = path.file_stem().unwrap().to_str().unwrap(); + let component_path = out_dir.join(format!("{stem}.component.wasm")); + components.push((path, adapter, mtime.as_ref(), component_path.clone())); + component_path + } None => missing_sdk_path.clone(), }; generated_code += @@ -141,6 +150,12 @@ impl Artifacts { ); } + components + .par_iter() + .for_each(|(wasm, adapter, mtime, component_path)| { + self.compile_component(wasm, adapter, *mtime, component_path); + }); + for (kind, targets) in kinds { generated_code += &format!("#[macro_export]"); generated_code += &format!("macro_rules! foreach_{kind} {{\n"); @@ -206,13 +221,15 @@ impl Artifacts { generated_code: &mut String, name: &str, features: &[&str], - ) -> Vec { + ) -> (Vec, Option) { let mut cmd = cargo(); cmd.arg("build") .arg("--release") + .arg("-vv") .arg("--package=wasi-preview1-component-adapter") .arg("--target=wasm32-unknown-unknown") - .env("CARGO_TARGET_DIR", &self.out_dir) + .env("CARGO_BUILD_BUILD_DIR", &self.out_dir) + .env("CARGO_TARGET_DIR", &self.out_dir.join(name)) .env("RUSTFLAGS", rustflags()) .env_remove("CARGO_ENCODED_RUSTFLAGS"); for f in features { @@ -224,24 +241,55 @@ impl Artifacts { let artifact = self .out_dir + .join(name) .join("wasm32-unknown-unknown") .join("release") .join("wasi_snapshot_preview1.wasm"); let adapter = self .out_dir .join(format!("wasi_snapshot_preview1.{name}.wasm")); - std::fs::copy(&artifact, &adapter).unwrap(); + + if let Ok(prev) = std::fs::read(&adapter) + && let Ok(cur) = std::fs::read(&artifact) + && prev == cur + { + // nothing to do ... + } else { + if adapter.exists() { + fs::remove_file(&adapter).unwrap(); + } + std::fs::hard_link(&artifact, &adapter) + .or_else(|_| std::fs::copy(&artifact, &adapter).map(|_| ())) + .unwrap(); + } self.read_deps_of(&artifact); println!("wasi {name} adapter: {:?}", &adapter); generated_code.push_str(&format!( "pub const ADAPTER_{}: &'static str = {adapter:?};\n", name.to_shouty_snake_case(), )); - fs::read(&adapter).unwrap() + (fs::read(&adapter).unwrap(), mtime(&adapter)) } // Compile a component, return the path of the binary: - fn compile_component(&self, wasm: &Path, adapter: &[u8]) -> PathBuf { + fn compile_component( + &self, + wasm: &Path, + adapter: &[u8], + adapter_mtime: Option<&SystemTime>, + component_path: &Path, + ) { + // If the component exists and was last updated after the inputs that + // make it up then there's no need to recreate it. + if let Some(wasm_mtime) = mtime(wasm) + && let Some(adapter_mtime) = adapter_mtime + && let Some(component_mtime) = mtime(&component_path) + && wasm_mtime < component_mtime + && *adapter_mtime < component_mtime + { + println!("reusing cached component for {wasm:?}"); + return; + } println!("creating a component from {wasm:?}"); let module = fs::read(wasm).expect("read wasm module"); let component = ComponentEncoder::default() @@ -252,51 +300,55 @@ impl Artifacts { .unwrap() .encode() .expect("module can be translated to a component"); - let out_dir = wasm.parent().unwrap(); - let stem = wasm.file_stem().unwrap().to_str().unwrap(); - let component_path = out_dir.join(format!("{stem}.component.wasm")); fs::write(&component_path, component).expect("write component to disk"); - component_path } fn build_non_rust_tests(&mut self, tests: &mut Vec) { const ASSETS_REL_SRC_DIR: &'static str = "../src/bin"; println!("cargo:rerun-if-changed={ASSETS_REL_SRC_DIR}"); - for entry in fs::read_dir(ASSETS_REL_SRC_DIR).unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - let name = path.file_stem().unwrap().to_str().unwrap().to_owned(); - match path.extension().and_then(|s| s.to_str()) { - // Compile C/C++ tests with clang - Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name, tests), - - // just a header, part of another test. - Some("h") => {} - - // Convert the text format to binary and use it as a test. - Some("wat") => { - let wasm = wat::parse_file(&path).unwrap(); - let core_wasm = self.out_dir.join(&name).with_extension("wasm"); - fs::write(&core_wasm, &wasm).unwrap(); - tests.push(Test { - name, - core_wasm: Some(core_wasm), - }); + let entries = fs::read_dir(ASSETS_REL_SRC_DIR) + .unwrap() + .map(|e| e.unwrap()) + .collect::>(); + let mut c_tests = entries + .par_iter() + .flat_map(|entry| { + let path = entry.path(); + let name = path.file_stem().unwrap().to_str().unwrap().to_owned(); + match path.extension().and_then(|s| s.to_str()) { + // Compile C/C++ tests with clang + Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name), + + // just a header, part of another test. + Some("h") => None, + + // Convert the text format to binary and use it as a test. + Some("wat") => { + let wasm = wat::parse_file(&path).unwrap(); + let core_wasm = self.out_dir.join(&name).with_extension("wasm"); + fs::write(&core_wasm, &wasm).unwrap(); + Some(Test { + name, + core_wasm: Some(core_wasm), + }) + } + + // these are built above in `build_rust_tests` + Some("rs") => None, + + // Prevent stray files for now that we don't understand. + Some(_) => panic!("unknown file extension on {path:?}"), + + None => unreachable!("no extension in path {path:?}"), } - - // these are built above in `build_rust_tests` - Some("rs") => {} - - // Prevent stray files for now that we don't understand. - Some(_) => panic!("unknown file extension on {path:?}"), - - None => unreachable!("no extension in path {path:?}"), - } - } + }) + .collect::>(); + c_tests.sort_by_key(|t| t.name.clone()); + tests.extend(c_tests); } - fn build_c_or_cpp_test(&mut self, path: PathBuf, name: String, tests: &mut Vec) { + fn build_c_or_cpp_test(&self, path: PathBuf, name: String) -> Option { println!("compiling {path:?}"); println!("cargo:rerun-if-changed={}", path.display()); let contents = std::fs::read_to_string(&path).unwrap(); @@ -304,7 +356,7 @@ impl Artifacts { wasmtime_test_util::wast::parse_test_config::(&contents, "//!").unwrap(); if config.skip { - return; + return None; } // The debug tests relying on these assets are ignored by default, @@ -317,11 +369,10 @@ impl Artifacts { let wasi_sdk_path = match env::var_os("WASI_SDK_PATH") { Some(path) => PathBuf::from(path), None => { - tests.push(Test { + return Some(Test { name, core_wasm: None, }); - return; } }; @@ -352,7 +403,7 @@ impl Artifacts { assert!(dwp.status().expect("failed to spawn llvm-dwp").success()); } - tests.push(Test { + return Some(Test { name, core_wasm: Some(wasm_path), }); @@ -425,3 +476,7 @@ fn rustflags() -> &'static str { _ => "", } } + +fn mtime(path: &Path) -> Option { + path.metadata().ok()?.modified().ok() +}