Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 81 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ arrow-array = "59.2.0"
arrow-schema = "59.2.0"
num-traits = "0.2.19"
parquet = "59.2.0"
mockable = "3.0.0"

[workspace.lints.clippy]
pedantic = { level = "warn", priority = -1 }
disallowed_methods = "deny"

# 1. hygiene
allow_attributes = "deny"
Expand Down
2 changes: 2 additions & 0 deletions chutoro-benches/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ chutoro-core = { path = "../chutoro-core" }
chutoro-providers-dense = { path = "../chutoro-providers/dense" }
criterion = { version = "0.5.1", features = ["html_reports"] }
flate2 = "1.1.9"
mockable = { workspace = true }
rand = { version = "0.8.5", features = ["small_rng"] }
strsim = "0.11.1"
thiserror = "2.0.17"
Expand Down Expand Up @@ -59,6 +60,7 @@ harness = false
# and updated to stay in sync.
[lints.clippy]
pedantic = { level = "warn", priority = -1 }
disallowed_methods = "deny"

# 1. hygiene
allow_attributes = "deny"
Expand Down
26 changes: 19 additions & 7 deletions chutoro-benches/benches/hnsw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use criterion::{
BatchSize, BenchmarkGroup, BenchmarkId, Criterion, black_box, criterion_main,
measurement::WallTime,
};
use mockable::{DefaultEnv, Env};

use chutoro_benches::{
criterion_support::{
Expand Down Expand Up @@ -257,8 +258,8 @@ fn hnsw_build(c: &mut Criterion) {
}
}

fn should_collect_memory_profile() -> bool {
if let Ok(value) = std::env::var("CHUTORO_BENCH_HNSW_MEMORY_PROFILE") {
fn should_collect_memory_profile_with_env(env: &dyn Env) -> bool {
if let Some(value) = env.string("CHUTORO_BENCH_HNSW_MEMORY_PROFILE") {
let normalized = value.trim().to_ascii_lowercase();
if matches!(normalized.as_str(), "0" | "false" | "off") {
return false;
Expand All @@ -270,17 +271,21 @@ fn should_collect_memory_profile() -> bool {
!is_benchmark_discovery() && !is_exact_benchmark_probe()
}

fn memory_report_path() -> PathBuf {
std::env::var_os("CHUTORO_BENCH_HNSW_MEMORY_REPORT_PATH")
fn memory_report_path_with_env(env: &dyn Env) -> PathBuf {
env.os_string("CHUTORO_BENCH_HNSW_MEMORY_REPORT_PATH")
.map_or_else(|| PathBuf::from(MEMORY_REPORT_PATH), PathBuf::from)
}

fn profile_hnsw_memory_impl() -> Result<Option<PathBuf>, BenchSetupError> {
if !should_collect_memory_profile() {
profile_hnsw_memory_impl_with_env(&DefaultEnv)
}

fn profile_hnsw_memory_impl_with_env(env: &dyn Env) -> Result<Option<PathBuf>, BenchSetupError> {
if !should_collect_memory_profile_with_env(env) {
return Ok(None);
}

let report_path = memory_report_path();
let report_path = memory_report_path_with_env(env);
let mut records = Vec::new();

for &point_count in POINT_COUNTS {
Expand Down Expand Up @@ -330,6 +335,13 @@ fn hnsw_build_with_edges(c: &mut Criterion) {
}

fn hnsw_build_diverse_sources_impl(c: &mut Criterion) -> Result<(), BenchSetupError> {
hnsw_build_diverse_sources_impl_with_env(c, &DefaultEnv)
}

fn hnsw_build_diverse_sources_impl_with_env(
c: &mut Criterion,
env: &dyn Env,
) -> Result<(), BenchSetupError> {
let mut group = c.benchmark_group("hnsw_build_diverse_sources");
configure_hnsw_group(&mut group);

Expand Down Expand Up @@ -368,7 +380,7 @@ fn hnsw_build_diverse_sources_impl(c: &mut Criterion) -> Result<(), BenchSetupEr
&params,
);

if std::env::var("CHUTORO_BENCH_ENABLE_MNIST").as_deref() == Ok("1") {
if env.string("CHUTORO_BENCH_ENABLE_MNIST").as_deref() == Some("1") {
let mnist = SyntheticSource::load_mnist(&MnistConfig::default())?;
bench_build_source(
&mut group,
Expand Down
37 changes: 28 additions & 9 deletions chutoro-benches/benches/hnsw_ef_sweep.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use std::{num::NonZeroUsize, path::PathBuf, time::Duration, time::Instant};

use criterion::{BatchSize, BenchmarkId, Criterion, criterion_main};
use mockable::{DefaultEnv, Env};

use chutoro_benches::{
ef_sweep::{
Expand Down Expand Up @@ -82,8 +83,8 @@ fn warn_unrecognised_bool_env(env_var_name: &str, value: &str) {
);
}

fn parse_bool_env_var(env_var_name: &str) -> Option<bool> {
let value = std::env::var(env_var_name).ok()?;
fn parse_bool_env_var(env: &dyn Env, env_var_name: &str) -> Option<bool> {
let value = env.string(env_var_name)?;
let normalized = value.trim().to_ascii_lowercase();
if matches!(normalized.as_str(), "0" | "false" | "off") {
return Some(false);
Expand Down Expand Up @@ -122,24 +123,42 @@ fn ef_sweep_point_counts() -> &'static [usize] {
}

fn should_collect_recall_report() -> bool {
parse_bool_env_var("CHUTORO_BENCH_HNSW_RECALL_REPORT").unwrap_or_else(|| !is_discovery_mode())
should_collect_recall_report_with_env(&DefaultEnv)
}

fn should_collect_recall_report_with_env(env: &dyn Env) -> bool {
parse_bool_env_var(env, "CHUTORO_BENCH_HNSW_RECALL_REPORT")
.unwrap_or_else(|| !is_discovery_mode())
}

fn recall_report_path() -> PathBuf {
std::env::var_os("CHUTORO_BENCH_HNSW_RECALL_REPORT_PATH")
recall_report_path_with_env(&DefaultEnv)
}

fn recall_report_path_with_env(env: &dyn Env) -> PathBuf {
env.os_string("CHUTORO_BENCH_HNSW_RECALL_REPORT_PATH")
.map_or_else(|| PathBuf::from(RECALL_REPORT_PATH), PathBuf::from)
}

fn should_collect_cluster_quality_report() -> bool {
parse_bool_env_var("CHUTORO_BENCH_HNSW_CLUSTER_QUALITY_REPORT")
should_collect_cluster_quality_report_with_env(&DefaultEnv)
}

fn should_collect_cluster_quality_report_with_env(env: &dyn Env) -> bool {
parse_bool_env_var(env, "CHUTORO_BENCH_HNSW_CLUSTER_QUALITY_REPORT")
.unwrap_or_else(|| !is_discovery_mode())
}

fn cluster_quality_report_path() -> PathBuf {
std::env::var_os("CHUTORO_BENCH_HNSW_CLUSTER_QUALITY_REPORT_PATH").map_or_else(
|| PathBuf::from(CLUSTERING_QUALITY_REPORT_PATH),
PathBuf::from,
)
cluster_quality_report_path_with_env(&DefaultEnv)
}

fn cluster_quality_report_path_with_env(env: &dyn Env) -> PathBuf {
env.os_string("CHUTORO_BENCH_HNSW_CLUSTER_QUALITY_REPORT_PATH")
.map_or_else(
|| PathBuf::from(CLUSTERING_QUALITY_REPORT_PATH),
PathBuf::from,
)
}

/// Returns an evenly-spaced query index for deterministic recall sampling.
Expand Down
7 changes: 6 additions & 1 deletion chutoro-benches/src/criterion_support.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use std::{fmt::Display, time::Duration};

use criterion::{BenchmarkGroup, BenchmarkId, Criterion, measurement::WallTime};
use mockable::{DefaultEnv, Env};

/// Returns whether the current command line includes `flag`.
///
Expand Down Expand Up @@ -111,9 +112,13 @@ pub fn is_exact_benchmark_probe() -> bool {
/// ```
#[must_use]
pub fn is_nextest_exact_benchmark_probe() -> bool {
is_nextest_exact_benchmark_probe_with_env(&DefaultEnv)
}

fn is_nextest_exact_benchmark_probe_with_env(env: &dyn Env) -> bool {
is_nextest_exact_benchmark_probe_args(
std::env::args(),
std::env::var_os("NEXTEST_TEST_NAME").is_some(),
env.os_string("NEXTEST_TEST_NAME").is_some(),
)
}

Expand Down
8 changes: 6 additions & 2 deletions chutoro-benches/src/neighbour_scoring/benchmark_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use chutoro_core::DataSource;
use criterion::{
BenchmarkGroup, BenchmarkId, Criterion, Throughput, black_box, measurement::WallTime,
};
use mockable::{DefaultEnv, Env};

use super::{
CandidateBucket, ScoringFixture, benchmark_support::BenchError, benchmark_support::BenchResult,
Expand All @@ -34,9 +35,12 @@ fn should_use_short_measurement_value(value: Option<&str>) -> bool {
}

fn should_use_short_measurement() -> bool {
should_use_short_measurement_value(std::env::var(SHORT_MEASUREMENT_ENV).ok().as_deref())
should_use_short_measurement_with_env(&DefaultEnv)
}

fn should_use_short_measurement_with_env(env: &dyn Env) -> bool {
should_use_short_measurement_value(env.string(SHORT_MEASUREMENT_ENV).as_deref())
}
fn score_candidates(
scoring_fixture: &ScoringFixture,
candidates: &[usize],
Expand Down Expand Up @@ -109,7 +113,7 @@ fn neighbour_scoring_impl_with(
) -> BenchResult<()> {
let report_parent_dir = report_parent_dir();
let build_profile_target = build_profile_report_target_value(
std::env::var(BUILD_PROFILE_ENV).ok().as_deref(),
DefaultEnv.string(BUILD_PROFILE_ENV).as_deref(),
&report_parent_dir,
);
let build_profile_report_dir = build_profile_target
Expand Down
Loading
Loading