diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..043cdfee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,112 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + zigbuild: + strategy: + matrix: + target: + - aarch64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - x86_64-apple-darwin + - aarch64-apple-darwin + - x86_64-pc-windows-gnu + - wasm32-unknown-unknown + + runs-on: ubuntu-latest + container: + image: ghcr.io/rust-cross/cargo-zigbuild:sha-59b27c3 + + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Install CMake + if: matrix.target != 'wasm32-unknown-unknown' + run: | + apt-get update -y && apt-get install -y --no-install-recommends --no-install-suggests cmake + + - name: Add wasm target + if: matrix.target == 'wasm32-unknown-unknown' + run: rustup target add wasm32-unknown-unknown + + - name: Install clippy + run: rustup component add clippy + + - name: Run cargo clippy + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo clippy --target ${{ matrix.target }} --features cmake -- -D warnings + + - name: Build + run: cargo zigbuild --target ${{ matrix.target }} + + - name: Build with thin feature + run: cargo zigbuild --target ${{ matrix.target }} --features thin + + - name: Build with cmake feature + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo zigbuild --target ${{ matrix.target }} --features cmake + + - name: Build with cmake feature (no default features) + if: matrix.target != 'wasm32-unknown-unknown' + run: cargo zigbuild --target ${{ matrix.target }} --no-default-features --features cmake,legacy,zdict_builder + + windows-msvc: + strategy: + matrix: + target: + - i686-pc-windows-msvc + - x86_64-pc-windows-msvc + + runs-on: windows-latest + + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - name: Setup Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable-${{ matrix.target }} + targets: ${{ matrix.target }} + + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose + + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose + + - name: Build with feature thin + run: cargo build --verbose --features thin + - name: Run tests with feature thin + run: cargo test --verbose --features thin + + - name: Build zstd-safe with feature seekable + run: cargo build --manifest-path zstd-safe/Cargo.toml --verbose --features seekable + - name: Run zstd-safe tests with feature seekable + run: cargo test --manifest-path zstd-safe/Cargo.toml --verbose --features seekable + - name: Build zstd-safe with features std and seekable + run: cargo build --manifest-path zstd-safe/Cargo.toml --verbose --features std,seekable + - name: Run zstd-safe tests with features std and seekable + run: cargo test --manifest-path zstd-safe/Cargo.toml --verbose --features std,seekable diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index a18723eb..00000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Linux - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - - runs-on: self-hosted - - steps: - - uses: actions/checkout@v6 - with: - submodules: recursive - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose - - - name: Build with feature thin - run: cargo build --verbose --features thin - - name: Run tests - run: cargo test --verbose --features thin - - - name: Build zstd-safe with feature seekable - run: cargo build --manifest-path zstd-safe/Cargo.toml --verbose --features seekable - - name: Run zstd-safe tests with feature seekable - run: cargo test --manifest-path zstd-safe/Cargo.toml --verbose --features seekable - - name: Build zstd-safe with features std and seekable - run: cargo build --manifest-path zstd-safe/Cargo.toml --verbose --features std,seekable - - name: Run zstd-safe tests with features std and seekable - run: cargo test --manifest-path zstd-safe/Cargo.toml --verbose --features std,seekable diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index 507b57f1..00000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: macOS - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - - runs-on: macos-latest - - steps: - - uses: actions/checkout@v6 - with: - submodules: recursive - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose - diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml deleted file mode 100644 index cc05d81c..00000000 --- a/.github/workflows/wasm.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Wasm - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - - runs-on: self-hosted - - steps: - - name: Wasm target - run: rustup target add wasm32-unknown-unknown - - uses: actions/checkout@v6 - with: - submodules: recursive - - - name: Build - run: cargo build --verbose --target wasm32-unknown-unknown - - name: Build with feature thin - run: cargo build --verbose --features thin --target wasm32-unknown-unknown diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 89e76913..00000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Windows - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - - strategy: - matrix: - target: - #- i686-pc-windows-gnu - - i686-pc-windows-msvc - #- x86_64-pc-windows-gnu - - x86_64-pc-windows-msvc - channel: [ stable ] - - runs-on: windows-latest - - steps: - - uses: actions/checkout@v6 - with: - submodules: recursive - - name: setup - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.channel }}-${{ matrix.target }} - targets: ${{ matrix.target }} - - - name: Add mingw32 to path for i686-gnu - run: | - echo "C:\msys64\mingw32\bin" >> $GITHUB_PATH - echo "C:\msys64\usr\bin" >> $GITHUB_PATH - if: matrix.target == 'i686-pc-windows-gnu' - shell: bash - - name: Add mingw64 to path for x86_64-gnu - run: | - echo "C:\msys64\mingw64\bin" >> $GITHUB_PATH - echo "C:\msys64\usr\bin" >> $GITHUB_PATH - if: matrix.target == 'x86_64-pc-windows-gnu' - shell: bash - - name: Update gcc - if: matrix.target == 'x86_64-pc-windows-gnu' - run: pacman.exe -Sy --noconfirm mingw-w64-x86_64-toolchain - - name: Update gcc - if: matrix.target == 'i686-pc-windows-gnu' - run: pacman.exe -Sy --noconfirm mingw-w64-i686-toolchain - - - name: Build - run: cargo build --verbose --verbose - - name: Run tests - run: cargo test --verbose diff --git a/Cargo.toml b/Cargo.toml index d9edf373..2cb672cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,10 @@ partial-io = "0.5" walkdir = "2.2" [features] -default = ["legacy", "arrays", "zdict_builder"] +default = ["legacy", "arrays", "zdict_builder", "cc"] bindgen = ["zstd-safe/bindgen"] +cc = ["zstd-safe/cc"] debug = ["zstd-safe/debug"] legacy = ["zstd-safe/legacy"] pkg-config = ["zstd-safe/pkg-config"] @@ -40,6 +41,7 @@ zstdmt = ["zstd-safe/zstdmt"] experimental = ["zstd-safe/experimental"] thin = ["zstd-safe/thin"] arrays = ["zstd-safe/arrays"] +cmake = ["zstd-safe/cmake"] no_asm = ["zstd-safe/no_asm"] doc-cfg = [] zdict_builder = ["zstd-safe/zdict_builder"] diff --git a/src/dict.rs b/src/dict.rs index e7d81c9f..e6c34d1a 100644 --- a/src/dict.rs +++ b/src/dict.rs @@ -100,7 +100,7 @@ impl<'a> DecoderDictionary<'a> { /// /// * `sample_data` is the concatenation of all sample data. /// * `sample_sizes` is the size of each sample in `sample_data`. -/// The sum of all `sample_sizes` should equal the length of `sample_data`. +/// The sum of all `sample_sizes` should equal the length of `sample_data`. /// * `max_size` is the maximum size of the dictionary to generate. /// /// The result is the dictionary data. You can, for example, feed it to [`CDict::create`]. @@ -222,7 +222,7 @@ where /// Train a dict from a list of files. /// /// * `filenames` is an iterator of files to load. Each file will be treated as an individual -/// sample. +/// sample. /// * `max_size` is the maximum size of the dictionary to generate. /// /// The result is the dictionary data. You can, for example, feed it to [`CDict::create`]. diff --git a/zstd-safe/Cargo.toml b/zstd-safe/Cargo.toml index 1843dc10..4ca08f3c 100644 --- a/zstd-safe/Cargo.toml +++ b/zstd-safe/Cargo.toml @@ -20,9 +20,10 @@ features = ["experimental", "arrays", "std", "zdict_builder", "doc-cfg"] zstd-sys = { path = "zstd-sys", version = "2.0.15", default-features = false } [features] -default = ["legacy", "arrays", "zdict_builder"] +default = ["legacy", "arrays", "zdict_builder", "cc"] bindgen = ["zstd-sys/bindgen"] +cc = ["zstd-sys/cc"] debug = ["zstd-sys/debug"] experimental = ["zstd-sys/experimental"] legacy = ["zstd-sys/legacy"] @@ -31,6 +32,7 @@ std = ["zstd-sys/std"] # Implements WriteBuf for std types like Cursor and Vec. zstdmt = ["zstd-sys/zstdmt"] thin = ["zstd-sys/thin"] arrays = [] +cmake = ["zstd-sys/cmake"] no_asm = ["zstd-sys/no_asm"] doc-cfg = [] zdict_builder = ["zstd-sys/zdict_builder"] diff --git a/zstd-safe/zstd-sys/Cargo.toml b/zstd-safe/zstd-sys/Cargo.toml index f880300d..73b222f0 100644 --- a/zstd-safe/zstd-sys/Cargo.toml +++ b/zstd-safe/zstd-sys/Cargo.toml @@ -34,8 +34,14 @@ include = [ "/zstd/lib/**/*.c", "/zstd/lib/**/*.h", "/zstd/lib/**/*.S", + "/zstd/lib/libzstd.pc.in", "/zstd/contrib/seekable_format/*.c", "/zstd/contrib/seekable_format/*.h", + "/zstd/build/cmake/CMakeLists.txt", + "/zstd/build/cmake/zstdConfig.cmake.in", + "/zstd/build/cmake/CMakeModules/*.cmake", + "/zstd/build/cmake/lib/CMakeLists.txt", + "/zstd/build/cmake/lib/*.in", ] # exclude = [ # "zstd", @@ -63,10 +69,17 @@ version = "0.3.28" [build-dependencies.cc] version = "1.0.45" features = ["parallel"] +optional = true + +[build-dependencies.cmake] +version = "0.1" +optional = true [features] -default = ["legacy", "zdict_builder", "bindgen"] +default = ["legacy", "zdict_builder", "bindgen", "cc"] +cc = ["dep:cc"] # Use cc crate to build zstd (default) +cmake = ["dep:cmake"] # Use CMake to build zstd (improves cross-compilation support) debug = [] # Enable zstd debug logs experimental = [] # Expose experimental ZSTD API legacy = [] # Enable legacy ZSTD support (for versions < zstd-0.8) diff --git a/zstd-safe/zstd-sys/build.rs b/zstd-safe/zstd-sys/build.rs index b7f7e6b8..d9f16cd5 100644 --- a/zstd-safe/zstd-sys/build.rs +++ b/zstd-safe/zstd-sys/build.rs @@ -1,7 +1,228 @@ +#[cfg(all(feature = "cc", not(feature = "cmake")))] use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::{env, fmt, fs}; +/// Search for an executable in PATH. +#[cfg(feature = "cmake")] +fn find_in_path(name: &str) -> Option { + env::var_os("PATH").and_then(|paths| { + env::split_paths(&paths).find_map(|dir| { + let full_path = dir.join(name); + if full_path.is_file() { + Some(full_path) + } else { + None + } + }) + }) +} + +/// Map a Rust target triple to a zig target triple. +#[cfg(feature = "cmake")] +fn rust_target_to_zig_target(rust_target: &str) -> Option { + let arch = rust_target.split('-').next()?; + + if rust_target.contains("apple-darwin") { + Some(format!("{}-macos-none", arch)) + } else if rust_target.contains("windows-gnu") { + Some(format!("{}-windows-gnu", arch)) + } else if rust_target.contains("linux-musl") { + Some(format!("{}-linux-musl", arch)) + } else if rust_target.contains("linux-gnu") { + Some(format!("{}-linux-gnu", arch)) + } else { + None + } +} + +/// When cross-compiling with the cmake feature and no explicit CC is set, +/// auto-detect zig and use it as the C/C++ cross-compiler. +/// This enables `cargo zigbuild` to work out of the box. +#[cfg(feature = "cmake")] +fn setup_zig_cross_compiler() { + let target = env::var("TARGET").unwrap_or_default(); + let host = env::var("HOST").unwrap_or_default(); + + if target == host { + return; + } + + // Respect explicit compiler settings + let cc_target_env = format!("CC_{}", target.replace('-', "_")); + if env::var_os(&cc_target_env).is_some() + || env::var_os("TARGET_CC").is_some() + || env::var_os("CC").is_some() + { + return; + } + + let zig_path = match find_in_path("zig") { + Some(p) => p, + None => return, + }; + + let zig_target = match rust_target_to_zig_target(&target) { + Some(t) => t, + None => return, + }; + + // Generate wrapper scripts in OUT_DIR so cmake can invoke zig + // as a single-path compiler command. + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + // The wrapper scripts filter out --target= flags that cmake-rs may add + // (e.g. --target=x86_64-apple-macosx) since these conflict with zig's + // own target specification. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + // Use bash for array support (safe argument handling with spaces) + let wrapper_template = |cmd: &str| { + format!( + concat!( + "#!/bin/bash\n", + "args=()\n", + "for arg in \"$@\"; do\n", + " case \"$arg\" in\n", + " --target=*) ;;\n", + " *) args+=(\"$arg\") ;;\n", + " esac\n", + "done\n", + "exec \"{}\" {} -target {} \"${{args[@]}}\"\n", + ), + zig_path.display(), + cmd, + zig_target + ) + }; + + let cc_wrapper = out_dir.join("zig-cc"); + fs::write(&cc_wrapper, wrapper_template("cc")).unwrap(); + fs::set_permissions(&cc_wrapper, fs::Permissions::from_mode(0o755)) + .unwrap(); + + let cxx_wrapper = out_dir.join("zig-cxx"); + fs::write(&cxx_wrapper, wrapper_template("c++")).unwrap(); + fs::set_permissions(&cxx_wrapper, fs::Permissions::from_mode(0o755)) + .unwrap(); + + // Set target-specific env vars so cmake-rs (via cc crate) picks them up + env::set_var( + format!("CC_{}", target.replace('-', "_")), + &cc_wrapper, + ); + env::set_var( + format!("CXX_{}", target.replace('-', "_")), + &cxx_wrapper, + ); + } + + #[cfg(windows)] + { + // On Windows, filter --target= via a PowerShell one-liner in a .bat wrapper + let wrapper_template = |cmd: &str| { + format!( + "@echo off\r\nsetlocal enabledelayedexpansion\r\nset \"ARGS=\"\r\nfor %%a in (%*) do (\r\n echo %%a | findstr /b /c:\"--target=\" >nul || set \"ARGS=!ARGS! %%a\"\r\n)\r\n\"{}\" {} -target {} !ARGS!\r\n", + zig_path.display(), + cmd, + zig_target + ) + }; + + let cc_wrapper = out_dir.join("zig-cc.bat"); + fs::write(&cc_wrapper, wrapper_template("cc")).unwrap(); + + let cxx_wrapper = out_dir.join("zig-cxx.bat"); + fs::write(&cxx_wrapper, wrapper_template("c++")).unwrap(); + + env::set_var( + format!("CC_{}", target.replace('-', "_")), + &cc_wrapper, + ); + env::set_var( + format!("CXX_{}", target.replace('-', "_")), + &cxx_wrapper, + ); + } +} + +#[cfg(feature = "cmake")] +fn compile_zstd_cmake() { + setup_zig_cross_compiler(); + + let mut config = cmake::Config::new("zstd/build/cmake"); + + // Build only the static library + config.define("ZSTD_BUILD_SHARED", "OFF"); + config.define("ZSTD_BUILD_STATIC", "ON"); + config.define("ZSTD_BUILD_PROGRAMS", "OFF"); + config.define("ZSTD_BUILD_TESTS", "OFF"); + config.define("ZSTD_BUILD_CONTRIB", "OFF"); + + // Legacy support + if cfg!(feature = "legacy") { + config.define("ZSTD_LEGACY_SUPPORT", "ON"); + } else { + config.define("ZSTD_LEGACY_SUPPORT", "OFF"); + } + + // Multi-threading support + if cfg!(feature = "zstdmt") { + config.define("ZSTD_MULTITHREAD_SUPPORT", "ON"); + } else { + config.define("ZSTD_MULTITHREAD_SUPPORT", "OFF"); + } + + // Dictionary builder + if cfg!(feature = "zdict_builder") { + config.define("ZSTD_BUILD_DICTBUILDER", "ON"); + } else { + config.define("ZSTD_BUILD_DICTBUILDER", "OFF"); + } + + // Hide symbols so we can coexist with another zstd-linking lib + config.define("ZSTDLIB_VISIBLE", "hidden"); + config.define("ZSTDERRORLIB_VISIBLE", "hidden"); + config.define("ZDICTLIB_VISIBLE", "hidden"); + + let dst = config.build(); + + // Tell cargo where to find the built library. + // CMake may place it in lib/ or lib64/ depending on the platform. + let lib_dir = if dst.join("lib64").join("libzstd.a").exists() + || dst.join("lib64").join("zstd_static.lib").exists() + { + dst.join("lib64") + } else { + dst.join("lib") + }; + cargo_print(&format_args!( + "rustc-link-search=native={}", + lib_dir.display() + )); + + // On MSVC, the static library is named zstd_static + if cfg!(target_env = "msvc") { + cargo_print(&"rustc-link-lib=static=zstd_static"); + } else { + cargo_print(&"rustc-link-lib=static=zstd"); + } + + // Copy headers for downstream consumers + let src = env::current_dir().unwrap().join("zstd").join("lib"); + let include = dst.join("include"); + fs::create_dir_all(&include).unwrap(); + fs::copy(src.join("zstd.h"), include.join("zstd.h")).unwrap(); + fs::copy(src.join("zstd_errors.h"), include.join("zstd_errors.h")) + .unwrap(); + #[cfg(feature = "zdict_builder")] + fs::copy(src.join("zdict.h"), include.join("zdict.h")).unwrap(); + cargo_print(&format_args!("root={}", dst.display())); + cargo_print(&format_args!("include={}", include.display())); +} + #[cfg(feature = "bindgen")] fn generate_bindings(defs: Vec<&str>, headerpaths: Vec) { use bindgen::RustTarget; @@ -61,33 +282,34 @@ fn pkg_config() -> (Vec<&'static str>, Vec) { (vec!["PKG_CONFIG"], library.include_paths) } -#[cfg(not(feature = "legacy"))] +#[cfg(all(feature = "cc", not(feature = "legacy"), not(feature = "cmake")))] fn set_legacy(_config: &mut cc::Build) {} -#[cfg(feature = "legacy")] +#[cfg(all(feature = "cc", feature = "legacy", not(feature = "cmake")))] fn set_legacy(config: &mut cc::Build) { config.define("ZSTD_LEGACY_SUPPORT", Some("1")); config.include("zstd/lib/legacy"); } -#[cfg(feature = "zstdmt")] +#[cfg(all(feature = "cc", feature = "zstdmt", not(feature = "cmake")))] fn set_pthread(config: &mut cc::Build) { config.flag("-pthread"); } -#[cfg(not(feature = "zstdmt"))] +#[cfg(all(feature = "cc", not(feature = "zstdmt"), not(feature = "cmake")))] fn set_pthread(_config: &mut cc::Build) {} -#[cfg(feature = "zstdmt")] +#[cfg(all(feature = "cc", feature = "zstdmt", not(feature = "cmake")))] fn enable_threading(config: &mut cc::Build) { config.define("ZSTD_MULTITHREAD", Some("")); } -#[cfg(not(feature = "zstdmt"))] +#[cfg(all(feature = "cc", not(feature = "zstdmt"), not(feature = "cmake")))] fn enable_threading(_config: &mut cc::Build) {} /// This function would find the first flag in `flags` that is supported /// and add that to `config`. +#[cfg(all(feature = "cc", not(feature = "cmake")))] #[allow(dead_code)] fn flag_if_supported_with_fallbacks(config: &mut cc::Build, flags: &[&str]) { let option = flags @@ -99,6 +321,7 @@ fn flag_if_supported_with_fallbacks(config: &mut cc::Build, flags: &[&str]) { } } +#[cfg(all(feature = "cc", not(feature = "cmake")))] fn compile_zstd() { let mut config = cc::Build::new(); @@ -289,7 +512,18 @@ fn main() { .expect("Manifest dir is always set by cargo"), ); - compile_zstd(); + #[cfg(feature = "cmake")] + { + compile_zstd_cmake(); + } + #[cfg(all(feature = "cc", not(feature = "cmake")))] + { + compile_zstd(); + } + #[cfg(not(any(feature = "cmake", feature = "cc")))] + { + panic!("Either the `cmake` or `cc` feature must be enabled to compile zstd from source."); + } (vec![], vec![manifest_dir.join("zstd/lib")]) };