From 30225970255a0723e6b8e97c2ae8d51bbdf04f98 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 12:05:47 +0100 Subject: [PATCH 01/17] bench: netbench: default to 10000 bytes per round IMHO it doesn't make sense to benchmark a single byte transfer. Can still be overridden. --- benches/netbench/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/netbench/src/config.rs b/benches/netbench/src/config.rs index 16436e9773..ee640706f7 100644 --- a/benches/netbench/src/config.rs +++ b/benches/netbench/src/config.rs @@ -21,7 +21,7 @@ pub struct Config { #[arg( long = "bytes", short = 'k', - default_value_t = 1, + default_value_t = 10000, help = "number of bytes to transfer every round" )] pub n_bytes: usize, From 1ea4d7353a3a5388867e1ace56cf562430ccc1f3 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 12:08:32 +0100 Subject: [PATCH 02/17] bench: netbench-server: fix log benchmark name error The server has to be called server, independent of the OS --- benches/netbench/src/rust-tcp-bw/server.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 46dbbbf666..22c81d98d9 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -30,19 +30,11 @@ fn main() { let end = Instant::now(); let duration = end.duration_since(start); - #[cfg(target_os = "hermit")] log_benchmark_data( "TCP server", "Mbit/s", (tot_bytes as f64 * 8.0f64) / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()), ); - #[cfg(not(target_os = "hermit"))] - log_benchmark_data( - "TCP client", - "Mbit/s", - (tot_bytes as f64 * 8.0f64) / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()), - ); - connection::close_connection(&stream); } From a8fa976fd9f8cefe4ba89b0a2f981623155200a9 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 12:14:30 +0100 Subject: [PATCH 03/17] benches: netbench-tcp-bw: Proper average throughput calculation --- benches/netbench/src/rust-tcp-bw/server.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 22c81d98d9..047914168d 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -10,14 +10,13 @@ use rust_tcp_io_perf::connection; fn main() { let args = Config::parse(); - let tot_bytes = args.n_rounds * args.n_bytes; let mut buf = vec![0; args.n_bytes]; + let mut durations = Vec::with_capacity(args.n_rounds); let mut stream = connection::server_listen_and_get_first_connection(&args.port.to_string()); connection::setup(&args, &stream); - let start = Instant::now(); for i in 0..args.n_rounds { print!("round {i}: "); let round_start = Instant::now(); @@ -25,15 +24,13 @@ fn main() { let round_end = Instant::now(); let duration = round_end.duration_since(round_start); let mbits = buf.len() as f64 * 8.0f64 / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()); - println!("{mbits} Mbit/s"); + durations.push(mbits); } - let end = Instant::now(); - let duration = end.duration_since(start); log_benchmark_data( "TCP server", "Mbit/s", - (tot_bytes as f64 * 8.0f64) / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()), + durations.iter().sum::() / durations.len() as f64, ); connection::close_connection(&stream); From 9080dcc420feab13316e396404b5656a5a9a559d Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 12:15:20 +0100 Subject: [PATCH 04/17] benches: netbench-tcp-bw: Gracefully end benchmark on early client disconnect --- benches/netbench/src/rust-tcp-bw/server.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 047914168d..d8e3bfde87 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -1,4 +1,4 @@ -use std::io::Read; +use std::io::{ErrorKind, Read}; use std::time::Instant; use clap::Parser; @@ -20,7 +20,14 @@ fn main() { for i in 0..args.n_rounds { print!("round {i}: "); let round_start = Instant::now(); - stream.read_exact(&mut buf).unwrap(); + if let Err(e) = stream.read_exact(&mut buf){ + if e.kind() == ErrorKind::UnexpectedEof { + println!("Client ended transmission after {i} rounds"); + break; + } else { + panic!("Error in reading from stream: {}", e.kind()); + } + } let round_end = Instant::now(); let duration = round_end.duration_since(round_start); let mbits = buf.len() as f64 * 8.0f64 / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()); From ced2d473732f9988df5b081e678906abddc24438 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 12:17:11 +0100 Subject: [PATCH 05/17] benches: netbench-tcp-bw: Reduce stdout noise --- benches/netbench/src/rust-tcp-bw/server.rs | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index d8e3bfde87..5c71d9897a 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -14,13 +14,32 @@ fn main() { let mut buf = vec![0; args.n_bytes]; let mut durations = Vec::with_capacity(args.n_rounds); + println!( + "starting server with {} bytes and {} rounds", + args.n_bytes, args.n_rounds + ); let mut stream = connection::server_listen_and_get_first_connection(&args.port.to_string()); connection::setup(&args, &stream); + let progress_prints = [ + 1, + args.n_rounds / 10, + args.n_rounds / 10 * 2, + args.n_rounds / 10 * 3, + args.n_rounds / 10 * 4, + args.n_rounds / 10 * 5, + args.n_rounds / 10 * 6, + args.n_rounds / 10 * 7, + args.n_rounds / 10 * 8, + args.n_rounds / 10 * 9, + ]; + for i in 0..args.n_rounds { - print!("round {i}: "); + if progress_prints.contains(&i) { + println!("round {i}/{}", args.n_rounds) + } let round_start = Instant::now(); - if let Err(e) = stream.read_exact(&mut buf){ + if let Err(e) = stream.read_exact(&mut buf) { if e.kind() == ErrorKind::UnexpectedEof { println!("Client ended transmission after {i} rounds"); break; From abf3540d026c4d007e07f8450ccd240f49933068 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 14:48:34 +0100 Subject: [PATCH 06/17] benches: netbench-tcp-bw: Calculate std-dev --- benches/netbench/src/rust-tcp-bw/server.rs | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 5c71d9897a..268d0e858e 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -8,6 +8,32 @@ use hermit_bench_output::log_benchmark_data; use rust_tcp_io_perf::config::Config; use rust_tcp_io_perf::connection; +fn mean(data: &[f64]) -> Option { + let sum = data.iter().sum::(); + match data.len() { + positive if positive > 0 => Some(sum / positive as f64), + _ => None, + } +} + +fn std_deviation(data: &[f64]) -> Option { + match (mean(data), data.len()) { + (Some(data_mean), count) if count > 0 => { + let variance = + data.iter() + .map(|value| { + let diff = data_mean - *value; + + diff * diff + }) + .sum::() / count as f64; + + Some(variance.sqrt()) + } + _ => None, + } +} + fn main() { let args = Config::parse(); @@ -53,10 +79,12 @@ fn main() { durations.push(mbits); } + log_benchmark_data("TCP server", "Mbit/s", mean(&durations).unwrap()); + log_benchmark_data( - "TCP server", + "TCP server-StdDev", "Mbit/s", - durations.iter().sum::() / durations.len() as f64, + std_deviation(&durations).unwrap(), ); connection::close_connection(&stream); From 676dcdd6a347136b670a127fe26f7c50c22d082d Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 14:49:23 +0100 Subject: [PATCH 07/17] benches: netbench-tcp-bw: Added Warmup runs --- benches/netbench/src/config.rs | 7 +++ benches/netbench/src/rust-tcp-bw/client.rs | 36 ++++++++----- benches/netbench/src/rust-tcp-bw/server.rs | 61 +++++++++++++--------- 3 files changed, 65 insertions(+), 39 deletions(-) diff --git a/benches/netbench/src/config.rs b/benches/netbench/src/config.rs index ee640706f7..b49d5f4540 100644 --- a/benches/netbench/src/config.rs +++ b/benches/netbench/src/config.rs @@ -47,6 +47,13 @@ pub struct Config { help = "id of process to pin thread to, -1 for no pinning" )] pub p_id: i8, + #[arg( + long = "warmup", + short = 'w', + default_value_t = 50, + help = "nr. of warmup runs" + )] + pub warmup: usize, } impl Config { diff --git a/benches/netbench/src/rust-tcp-bw/client.rs b/benches/netbench/src/rust-tcp-bw/client.rs index 203816ca38..7e3d97377b 100644 --- a/benches/netbench/src/rust-tcp-bw/client.rs +++ b/benches/netbench/src/rust-tcp-bw/client.rs @@ -1,4 +1,5 @@ use std::io::{self, Write}; +use std::net::TcpStream; use clap::Parser; #[cfg(target_os = "hermit")] @@ -6,6 +7,25 @@ use hermit as _; use rust_tcp_io_perf::config::Config; use rust_tcp_io_perf::connection; +fn send_rounds(stream: &mut TcpStream, rounds: usize, bytes: usize) { + // Create a buffer of 0s, size n_bytes, to be sent over multiple times + let buf = vec![0; bytes]; + + for _i in 0..rounds { + let mut pos = 0; + + while pos < buf.len() { + let bytes_written = match stream.write(&buf[pos..]) { + Ok(len) => len, + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => 0, + Err(e) => panic!("encountered IO error: {e}"), + }; + pos += bytes_written; + } + } + stream.flush().expect("Unexpected behaviour"); +} + fn main() { let args = Config::parse(); @@ -15,21 +35,9 @@ fn main() { connection::setup(&args, &stream); println!("Connection established! Ready to send..."); - // Create a buffer of 0s, size n_bytes, to be sent over multiple times - let buf = vec![0; args.n_bytes]; + send_rounds(&mut stream, args.warmup, args.n_bytes); + send_rounds(&mut stream, args.n_rounds, args.n_bytes); - for _i in 0..args.n_rounds { - let mut pos = 0; - - while pos < buf.len() { - let bytes_written = match stream.write(&buf[pos..]) { - Ok(len) => len, - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => 0, - Err(e) => panic!("encountered IO error: {e}"), - }; - pos += bytes_written; - } - } stream.flush().expect("Unexpected behaviour"); connection::close_connection(&stream); diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 268d0e858e..f8d24a0f33 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -1,4 +1,5 @@ use std::io::{ErrorKind, Read}; +use std::net::TcpStream; use std::time::Instant; use clap::Parser; @@ -34,35 +35,30 @@ fn std_deviation(data: &[f64]) -> Option { } } -fn main() { - let args = Config::parse(); - - let mut buf = vec![0; args.n_bytes]; - let mut durations = Vec::with_capacity(args.n_rounds); - - println!( - "starting server with {} bytes and {} rounds", - args.n_bytes, args.n_rounds - ); - let mut stream = connection::server_listen_and_get_first_connection(&args.port.to_string()); - connection::setup(&args, &stream); +fn receive_rounds( + stream: &mut TcpStream, + rounds: usize, + bytes: usize, + progress_print: bool, +) -> Vec { + let mut buf = vec![0; bytes]; + let mut durations = Vec::with_capacity(rounds); let progress_prints = [ 1, - args.n_rounds / 10, - args.n_rounds / 10 * 2, - args.n_rounds / 10 * 3, - args.n_rounds / 10 * 4, - args.n_rounds / 10 * 5, - args.n_rounds / 10 * 6, - args.n_rounds / 10 * 7, - args.n_rounds / 10 * 8, - args.n_rounds / 10 * 9, + rounds / 10, + rounds / 10 * 2, + rounds / 10 * 3, + rounds / 10 * 4, + rounds / 10 * 5, + rounds / 10 * 6, + rounds / 10 * 7, + rounds / 10 * 8, + rounds / 10 * 9, ]; - - for i in 0..args.n_rounds { - if progress_prints.contains(&i) { - println!("round {i}/{}", args.n_rounds) + for i in 0..rounds { + if progress_print && progress_prints.contains(&i) { + println!("round {i}/{}", rounds) } let round_start = Instant::now(); if let Err(e) = stream.read_exact(&mut buf) { @@ -78,6 +74,21 @@ fn main() { let mbits = buf.len() as f64 * 8.0f64 / (1024.0f64 * 1024.0f64 * duration.as_secs_f64()); durations.push(mbits); } + durations +} + +fn main() { + let args = Config::parse(); + + println!( + "starting server with {} bytes, {} warmup rounds and {} rounds", + args.n_bytes, args.warmup, args.n_rounds + ); + let mut stream = connection::server_listen_and_get_first_connection(&args.port.to_string()); + connection::setup(&args, &stream); + + let _ = receive_rounds(&mut stream, args.warmup, args.n_bytes, false); + let durations = receive_rounds(&mut stream, args.n_rounds, args.n_bytes, true); log_benchmark_data("TCP server", "Mbit/s", mean(&durations).unwrap()); From bb81b160e1be0d768cbfcba978f3f4cc990628f1 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 19:14:44 +0100 Subject: [PATCH 08/17] benches: netbench-tcp-bw: Added statistics for boxplot --- benches/netbench/src/rust-tcp-bw/server.rs | 57 ++++++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index f8d24a0f33..14809b8909 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -35,6 +35,53 @@ fn std_deviation(data: &[f64]) -> Option { } } +#[derive(Debug)] +#[allow(dead_code)] +struct BoxplotValues { + whisk_min: f64, + whisk_max: f64, + median: f64, + q1: f64, + q3: f64, + nr_outliers: usize, + mean: f64, + std_deviation: f64, +} + +fn calculate_boxplot(durations: &[f64]) -> BoxplotValues { + let mut durations = Vec::from(durations); + durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let q1 = *durations.get(durations.len() / 4).unwrap(); + let q3 = *durations.get(durations.len() * 3 / 4).unwrap(); + let outlier_min = q1 - 1.5 * (q3 - q1); + let outlier_max = q3 + 1.5 * (q3 - q1); + let filtered_durations = durations + .iter() + .filter(|&x| *x >= outlier_min && *x <= outlier_max) + .collect::>(); + + let min = *filtered_durations[0]; + let max = *filtered_durations[filtered_durations.len() - 1]; + let median = **filtered_durations + .get(filtered_durations.len() / 2) + .unwrap_or(&&0.0); + let outliers = durations + .iter() + .filter(|&x| *x < outlier_min || *x > outlier_max) + .copied() + .collect::>(); + BoxplotValues { + whisk_min: min, + whisk_max: max, + median, + q1, + q3, + nr_outliers: outliers.len(), + std_deviation: std_deviation(&durations).unwrap(), + mean: mean(&durations).unwrap(), + } +} + fn receive_rounds( stream: &mut TcpStream, rounds: usize, @@ -92,10 +139,12 @@ fn main() { log_benchmark_data("TCP server", "Mbit/s", mean(&durations).unwrap()); - log_benchmark_data( - "TCP server-StdDev", - "Mbit/s", - std_deviation(&durations).unwrap(), + let statistics = calculate_boxplot(&durations); + println!("{statistics:#.2?}"); + println!( + "{} outliers ({:.1}%)", + statistics.nr_outliers, + 100.0 * statistics.nr_outliers as f64 / durations.len() as f64 ); connection::close_connection(&stream); From 99884fec9af8f81c5d83cd73ff479a00c3711188 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Wed, 11 Feb 2026 20:46:01 +0100 Subject: [PATCH 09/17] bench: moved statistics fn to print_utils --- benches/netbench/src/print_utils.rs | 73 +++++++++++++++++++++ benches/netbench/src/rust-tcp-bw/server.rs | 74 +--------------------- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/benches/netbench/src/print_utils.rs b/benches/netbench/src/print_utils.rs index 631f319426..6d29e8353b 100644 --- a/benches/netbench/src/print_utils.rs +++ b/benches/netbench/src/print_utils.rs @@ -18,3 +18,76 @@ pub fn print_summary(hist: hdrhist::HDRHist) { println!("{entry:?}"); } } + +pub fn mean(data: &[f64]) -> Option { + let sum = data.iter().sum::(); + match data.len() { + positive if positive > 0 => Some(sum / positive as f64), + _ => None, + } +} + +pub fn std_deviation(data: &[f64]) -> Option { + match (mean(data), data.len()) { + (Some(data_mean), count) if count > 0 => { + let variance = + data.iter() + .map(|value| { + let diff = data_mean - *value; + + diff * diff + }) + .sum::() / count as f64; + + Some(variance.sqrt()) + } + _ => None, + } +} + +#[derive(Debug)] +#[allow(dead_code)] +pub struct BoxplotValues { + pub whisk_min: f64, + pub whisk_max: f64, + pub median: f64, + pub q1: f64, + pub q3: f64, + pub nr_outliers: usize, + pub mean: f64, + pub std_deviation: f64, +} + +pub fn calculate_boxplot(durations: &[f64]) -> BoxplotValues { + let mut durations = Vec::from(durations); + durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let q1 = *durations.get(durations.len() / 4).unwrap(); + let q3 = *durations.get(durations.len() * 3 / 4).unwrap(); + let outlier_min = q1 - 1.5 * (q3 - q1); + let outlier_max = q3 + 1.5 * (q3 - q1); + let filtered_durations = durations + .iter() + .filter(|&x| *x >= outlier_min && *x <= outlier_max) + .collect::>(); + + let min = *filtered_durations[0]; + let max = *filtered_durations[filtered_durations.len() - 1]; + let median = **filtered_durations + .get(filtered_durations.len() / 2) + .unwrap_or(&&0.0); + let outliers = durations + .iter() + .filter(|&x| *x < outlier_min || *x > outlier_max) + .copied() + .collect::>(); + BoxplotValues { + whisk_min: min, + whisk_max: max, + median, + q1, + q3, + nr_outliers: outliers.len(), + std_deviation: std_deviation(&durations).unwrap(), + mean: mean(&durations).unwrap(), + } +} diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 14809b8909..52e608c403 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -8,79 +8,7 @@ use hermit as _; use hermit_bench_output::log_benchmark_data; use rust_tcp_io_perf::config::Config; use rust_tcp_io_perf::connection; - -fn mean(data: &[f64]) -> Option { - let sum = data.iter().sum::(); - match data.len() { - positive if positive > 0 => Some(sum / positive as f64), - _ => None, - } -} - -fn std_deviation(data: &[f64]) -> Option { - match (mean(data), data.len()) { - (Some(data_mean), count) if count > 0 => { - let variance = - data.iter() - .map(|value| { - let diff = data_mean - *value; - - diff * diff - }) - .sum::() / count as f64; - - Some(variance.sqrt()) - } - _ => None, - } -} - -#[derive(Debug)] -#[allow(dead_code)] -struct BoxplotValues { - whisk_min: f64, - whisk_max: f64, - median: f64, - q1: f64, - q3: f64, - nr_outliers: usize, - mean: f64, - std_deviation: f64, -} - -fn calculate_boxplot(durations: &[f64]) -> BoxplotValues { - let mut durations = Vec::from(durations); - durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let q1 = *durations.get(durations.len() / 4).unwrap(); - let q3 = *durations.get(durations.len() * 3 / 4).unwrap(); - let outlier_min = q1 - 1.5 * (q3 - q1); - let outlier_max = q3 + 1.5 * (q3 - q1); - let filtered_durations = durations - .iter() - .filter(|&x| *x >= outlier_min && *x <= outlier_max) - .collect::>(); - - let min = *filtered_durations[0]; - let max = *filtered_durations[filtered_durations.len() - 1]; - let median = **filtered_durations - .get(filtered_durations.len() / 2) - .unwrap_or(&&0.0); - let outliers = durations - .iter() - .filter(|&x| *x < outlier_min || *x > outlier_max) - .copied() - .collect::>(); - BoxplotValues { - whisk_min: min, - whisk_max: max, - median, - q1, - q3, - nr_outliers: outliers.len(), - std_deviation: std_deviation(&durations).unwrap(), - mean: mean(&durations).unwrap(), - } -} +use rust_tcp_io_perf::print_utils::{calculate_boxplot, mean}; fn receive_rounds( stream: &mut TcpStream, From 3f388c5b74e5ab10dc399a00f26797cf7bdb0689 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 10:09:36 +0100 Subject: [PATCH 10/17] tcp latency: Fix incorrect output names --- benches/netbench/src/rust-tcp-latency/client.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/benches/netbench/src/rust-tcp-latency/client.rs b/benches/netbench/src/rust-tcp-latency/client.rs index 44116b4327..eb67407e32 100644 --- a/benches/netbench/src/rust-tcp-latency/client.rs +++ b/benches/netbench/src/rust-tcp-latency/client.rs @@ -53,26 +53,11 @@ fn main() { } connection::close_connection(&stream); - #[cfg(not(target_os = "hermit"))] - hermit_bench_output::log_benchmark_data( - "95th percentile TCP Server Latency", - "ns", - get_percentiles(hist.summary(), 0.95), - ); - #[cfg(not(target_os = "hermit"))] - hermit_bench_output::log_benchmark_data( - "Max TCP Server Latency", - "ns", - get_percentiles(hist.summary(), 1.0), - ); - - #[cfg(target_os = "hermit")] hermit_bench_output::log_benchmark_data( "95th percentile TCP Client Latency", "ns", get_percentiles(hist.summary(), 0.95), ); - #[cfg(target_os = "hermit")] hermit_bench_output::log_benchmark_data( "Max TCP Client Latency", "ns", From 29cad806c7a158b7284901c670786805b3583eb2 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 10:11:09 +0100 Subject: [PATCH 11/17] bench: tcp latency: use warmup runs parameter --- .../netbench/src/rust-tcp-latency/client.rs | 18 ++++++++++-------- .../netbench/src/rust-tcp-latency/server.rs | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/benches/netbench/src/rust-tcp-latency/client.rs b/benches/netbench/src/rust-tcp-latency/client.rs index eb67407e32..d2f0e3a532 100644 --- a/benches/netbench/src/rust-tcp-latency/client.rs +++ b/benches/netbench/src/rust-tcp-latency/client.rs @@ -18,7 +18,7 @@ fn main() { let wbuf: Vec = vec![0; n_bytes]; let mut rbuf: Vec = vec![0; n_bytes]; - let progress_tracking_percentage = (n_rounds * 2) / 100; + let progress_tracking_percentage = (n_rounds) / 100; let mut connected = false; @@ -32,19 +32,21 @@ fn main() { println!("Connection established! Ready to send..."); - // To avoid TCP slowstart we do double iterations and measure only the second half - for i in 0..(n_rounds * 2) { + for _ in 0..(args.warmup) { + connection::send_message(n_bytes, &mut stream, &wbuf); + connection::receive_message(n_bytes, &mut stream, &mut rbuf); + } + + for i in 0..n_rounds { let start = Instant::now(); connection::send_message(n_bytes, &mut stream, &wbuf); connection::receive_message(n_bytes, &mut stream, &mut rbuf); let duration = Instant::now().duration_since(start); - if i >= n_rounds { - hist.add_value( - duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64, - ); - } + hist.add_value( + duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64, + ); if i % progress_tracking_percentage == 0 { // Track progress on screen diff --git a/benches/netbench/src/rust-tcp-latency/server.rs b/benches/netbench/src/rust-tcp-latency/server.rs index 93b334fcc3..6baeb4d3c0 100644 --- a/benches/netbench/src/rust-tcp-latency/server.rs +++ b/benches/netbench/src/rust-tcp-latency/server.rs @@ -15,7 +15,7 @@ fn main() { threading::setup(&args); // Make sure n_rounds is the same between client and server - for _i in 0..(n_rounds * 2) { + for _i in 0..(n_rounds + args.warmup) { connection::receive_message(n_bytes, &mut stream, &mut buf); connection::send_message(n_bytes, &mut stream, &buf); } From 0f39aa3fbb05724d3cabd81f9f868d81dadae443 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 10:23:54 +0100 Subject: [PATCH 12/17] bench: tcp latency: refactored connection code --- .../netbench/src/rust-tcp-latency/client.rs | 85 +++++++++---------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/benches/netbench/src/rust-tcp-latency/client.rs b/benches/netbench/src/rust-tcp-latency/client.rs index d2f0e3a532..a6928d1413 100644 --- a/benches/netbench/src/rust-tcp-latency/client.rs +++ b/benches/netbench/src/rust-tcp-latency/client.rs @@ -20,58 +20,55 @@ fn main() { let progress_tracking_percentage = (n_rounds) / 100; - let mut connected = false; - - while !connected { + let mut stream = loop { match connection::client_connect(args.address_and_port()) { - Ok(mut stream) => { - connection::setup(&args, &stream); - threading::setup(&args); - connected = true; - let mut hist = hdrhist::HDRHist::new(); - - println!("Connection established! Ready to send..."); - - for _ in 0..(args.warmup) { - connection::send_message(n_bytes, &mut stream, &wbuf); - connection::receive_message(n_bytes, &mut stream, &mut rbuf); - } - - for i in 0..n_rounds { - let start = Instant::now(); - - connection::send_message(n_bytes, &mut stream, &wbuf); - connection::receive_message(n_bytes, &mut stream, &mut rbuf); - - let duration = Instant::now().duration_since(start); - hist.add_value( - duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64, - ); - - if i % progress_tracking_percentage == 0 { - // Track progress on screen - println!("{}% completed", i / progress_tracking_percentage); - } - } - connection::close_connection(&stream); - - hermit_bench_output::log_benchmark_data( - "95th percentile TCP Client Latency", - "ns", - get_percentiles(hist.summary(), 0.95), - ); - hermit_bench_output::log_benchmark_data( - "Max TCP Client Latency", - "ns", - get_percentiles(hist.summary(), 1.0), - ); + Ok(stream) => { + break stream; } Err(error) => { println!("Couldn't connect to server, retrying... Error {error}"); thread::sleep(time::Duration::from_secs(1)); } } + }; + + connection::setup(&args, &stream); + threading::setup(&args); + let mut hist = hdrhist::HDRHist::new(); + + println!("Connection established! Ready to send..."); + + for _ in 0..(args.warmup) { + connection::send_message(n_bytes, &mut stream, &wbuf); + connection::receive_message(n_bytes, &mut stream, &mut rbuf); + } + + for i in 0..n_rounds { + let start = Instant::now(); + + connection::send_message(n_bytes, &mut stream, &wbuf); + connection::receive_message(n_bytes, &mut stream, &mut rbuf); + + let duration = Instant::now().duration_since(start); + hist.add_value(duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64); + + if i % progress_tracking_percentage == 0 { + // Track progress on screen + println!("{}% completed", i / progress_tracking_percentage); + } } + connection::close_connection(&stream); + + hermit_bench_output::log_benchmark_data( + "95th percentile TCP Client Latency", + "ns", + get_percentiles(hist.summary(), 0.95), + ); + hermit_bench_output::log_benchmark_data( + "Max TCP Client Latency", + "ns", + get_percentiles(hist.summary(), 1.0), + ); } fn get_percentiles(summary: impl Iterator, percentile: f64) -> f64 { From 6a866133e1dbf1ea8a8bde8b7a71f3ed9ef88cd9 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 10:29:05 +0100 Subject: [PATCH 13/17] bench: tcp latency: limit connection retries --- .../netbench/src/rust-tcp-latency/client.rs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/benches/netbench/src/rust-tcp-latency/client.rs b/benches/netbench/src/rust-tcp-latency/client.rs index a6928d1413..9409fac128 100644 --- a/benches/netbench/src/rust-tcp-latency/client.rs +++ b/benches/netbench/src/rust-tcp-latency/client.rs @@ -20,17 +20,24 @@ fn main() { let progress_tracking_percentage = (n_rounds) / 100; - let mut stream = loop { - match connection::client_connect(args.address_and_port()) { - Ok(stream) => { - break stream; + const MAX_RETRIES: i32 = 30; + let mut retries = 0; + let mut stream = + loop { + match connection::client_connect(args.address_and_port()) { + Ok(stream) => { + break stream; + } + Err(error) => { + retries += 1; + println!("Couldn't connect to server, retrying ({retries}/{MAX_RETRIES})... ({error})"); + if retries >= MAX_RETRIES { + panic!("Can't establish connection to server. Aborting after {MAX_RETRIES} attempts"); + } + thread::sleep(time::Duration::from_secs(1)); + } } - Err(error) => { - println!("Couldn't connect to server, retrying... Error {error}"); - thread::sleep(time::Duration::from_secs(1)); - } - } - }; + }; connection::setup(&args, &stream); threading::setup(&args); From c770dd10855eed16e1d3278bc1e9fd470439ac0e Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 11:43:35 +0100 Subject: [PATCH 14/17] bench: made BoxplotValues generic over T --- benches/netbench/src/print_utils.rs | 149 ++++++++++++++------- benches/netbench/src/rust-tcp-bw/server.rs | 6 +- 2 files changed, 102 insertions(+), 53 deletions(-) diff --git a/benches/netbench/src/print_utils.rs b/benches/netbench/src/print_utils.rs index 6d29e8353b..a26443bb13 100644 --- a/benches/netbench/src/print_utils.rs +++ b/benches/netbench/src/print_utils.rs @@ -19,19 +19,24 @@ pub fn print_summary(hist: hdrhist::HDRHist) { } } -pub fn mean(data: &[f64]) -> Option { - let sum = data.iter().sum::(); - match data.len() { - positive if positive > 0 => Some(sum / positive as f64), - _ => None, - } +trait StatCalcs: Sized { + fn mean(data: &[Self]) -> Option; + fn std_deviation(data: &[Self]) -> Option; } +impl StatCalcs for f64 { + fn mean(data: &[f64]) -> Option { + let sum = data.iter().sum::(); + match data.len() { + positive if positive > 0 => Some(sum / positive as f64), + _ => None, + } + } -pub fn std_deviation(data: &[f64]) -> Option { - match (mean(data), data.len()) { - (Some(data_mean), count) if count > 0 => { - let variance = - data.iter() + fn std_deviation(data: &[f64]) -> Option { + match (Self::mean(data), data.len()) { + (Some(data_mean), count) if count > 0 => { + let variance = data + .iter() .map(|value| { let diff = data_mean - *value; @@ -39,55 +44,99 @@ pub fn std_deviation(data: &[f64]) -> Option { }) .sum::() / count as f64; - Some(variance.sqrt()) + Some(variance.sqrt()) + } + _ => None, + } + } +} +impl StatCalcs for u64 { + fn mean(data: &[u64]) -> Option { + let sum = data.iter().sum::(); + match data.len() { + positive if positive > 0 => Some(sum / positive as u64), + _ => None, + } + } + + fn std_deviation(data: &[u64]) -> Option { + match (Self::mean(data), data.len()) { + (Some(data_mean), count) if count > 0 => { + let variance = data + .iter() + .map(|value| { + let diff = data_mean as f64 - *value as f64; + + diff * diff + }) + .sum::() / count as f64; + + Some((variance as f64).sqrt()) + } + _ => None, } - _ => None, } } #[derive(Debug)] #[allow(dead_code)] -pub struct BoxplotValues { - pub whisk_min: f64, - pub whisk_max: f64, - pub median: f64, - pub q1: f64, - pub q3: f64, +pub struct BoxplotValues { + pub whisk_min: T, + pub whisk_max: T, + pub median: T, + pub q1: T, + pub q3: T, pub nr_outliers: usize, - pub mean: f64, + pub mean: T, pub std_deviation: f64, } +impl< + T: Clone + + PartialOrd + + std::ops::Mul + + std::ops::Sub + + std::ops::Add + + std::ops::Div + + std::marker::Copy + + Default + + StatCalcs + + std::convert::From, + > From<&[T]> for BoxplotValues +{ + fn from(value: &[T]) -> Self { + let mut durations = Vec::from(value); + durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let median = *durations + .get(durations.len() / 2) + .unwrap_or(&&Default::default()); -pub fn calculate_boxplot(durations: &[f64]) -> BoxplotValues { - let mut durations = Vec::from(durations); - durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let q1 = *durations.get(durations.len() / 4).unwrap(); - let q3 = *durations.get(durations.len() * 3 / 4).unwrap(); - let outlier_min = q1 - 1.5 * (q3 - q1); - let outlier_max = q3 + 1.5 * (q3 - q1); - let filtered_durations = durations - .iter() - .filter(|&x| *x >= outlier_min && *x <= outlier_max) - .collect::>(); + let q1 = *durations.get(durations.len() / 4).unwrap(); + let q3 = *durations.get(durations.len() * 3 / 4).unwrap(); + let iqr: T = >::into(3_u8) * (q3 - q1) / >::into(2_u8); + let outlier_min = q1 - iqr.into(); + let outlier_max = q3 + iqr.into(); + let outliers = durations + .iter() + .filter(|&x| *x < outlier_min || *x > outlier_max) + .copied() + .collect::>(); - let min = *filtered_durations[0]; - let max = *filtered_durations[filtered_durations.len() - 1]; - let median = **filtered_durations - .get(filtered_durations.len() / 2) - .unwrap_or(&&0.0); - let outliers = durations - .iter() - .filter(|&x| *x < outlier_min || *x > outlier_max) - .copied() - .collect::>(); - BoxplotValues { - whisk_min: min, - whisk_max: max, - median, - q1, - q3, - nr_outliers: outliers.len(), - std_deviation: std_deviation(&durations).unwrap(), - mean: mean(&durations).unwrap(), + let filtered_durations = durations + .iter() + .filter(|&x| *x >= outlier_min && *x <= outlier_max) + .collect::>(); + let whisk_min = *filtered_durations[0]; + let whisk_max = *filtered_durations[filtered_durations.len() - 1]; + + Self { + whisk_min, + whisk_max, + median, + q1, + q3, + nr_outliers: outliers.len(), + std_deviation: T::std_deviation(&durations).unwrap(), + mean: T::mean(&durations).unwrap(), + } } } diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 52e608c403..232149ef16 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -8,7 +8,7 @@ use hermit as _; use hermit_bench_output::log_benchmark_data; use rust_tcp_io_perf::config::Config; use rust_tcp_io_perf::connection; -use rust_tcp_io_perf::print_utils::{calculate_boxplot, mean}; +use rust_tcp_io_perf::print_utils::BoxplotValues; fn receive_rounds( stream: &mut TcpStream, @@ -65,9 +65,9 @@ fn main() { let _ = receive_rounds(&mut stream, args.warmup, args.n_bytes, false); let durations = receive_rounds(&mut stream, args.n_rounds, args.n_bytes, true); - log_benchmark_data("TCP server", "Mbit/s", mean(&durations).unwrap()); + let statistics = BoxplotValues::::from(durations.as_slice()); + log_benchmark_data("TCP server", "Mbit/s", statistics.mean); - let statistics = calculate_boxplot(&durations); println!("{statistics:#.2?}"); println!( "{} outliers ({:.1}%)", From 0df1cfbe06f2edd2d81fbb4a49eccb63e24f0354 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 12:02:38 +0100 Subject: [PATCH 15/17] benches: tcp-latency: Quick and Dirty add of boxplot values --- benches/netbench/src/rust-tcp-latency/client.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benches/netbench/src/rust-tcp-latency/client.rs b/benches/netbench/src/rust-tcp-latency/client.rs index 9409fac128..a6101bed60 100644 --- a/benches/netbench/src/rust-tcp-latency/client.rs +++ b/benches/netbench/src/rust-tcp-latency/client.rs @@ -5,6 +5,7 @@ use clap::Parser; #[cfg(target_os = "hermit")] use hermit as _; use rust_tcp_io_perf::config::Config; +use rust_tcp_io_perf::print_utils::BoxplotValues; use rust_tcp_io_perf::{connection, threading}; fn main() { @@ -42,6 +43,7 @@ fn main() { connection::setup(&args, &stream); threading::setup(&args); let mut hist = hdrhist::HDRHist::new(); + let mut latencies = Vec::with_capacity(n_rounds); println!("Connection established! Ready to send..."); @@ -57,7 +59,9 @@ fn main() { connection::receive_message(n_bytes, &mut stream, &mut rbuf); let duration = Instant::now().duration_since(start); - hist.add_value(duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64); + let duration_u64 = duration.as_secs() * 1_000_000_000u64 + duration.subsec_nanos() as u64; + hist.add_value(duration_u64); + latencies.push(duration_u64); if i % progress_tracking_percentage == 0 { // Track progress on screen @@ -76,6 +80,9 @@ fn main() { "ns", get_percentiles(hist.summary(), 1.0), ); + + let statistics = BoxplotValues::from(latencies.as_slice()); + println!("{statistics:#.2?}"); } fn get_percentiles(summary: impl Iterator, percentile: f64) -> f64 { From 69bb28080194dcb18105191684fb344eb27016b3 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 12:10:44 +0100 Subject: [PATCH 16/17] benches: end server when connection is terminated --- benches/netbench/src/connection.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benches/netbench/src/connection.rs b/benches/netbench/src/connection.rs index 03bc5da3dd..cc9b3116cf 100644 --- a/benches/netbench/src/connection.rs +++ b/benches/netbench/src/connection.rs @@ -27,7 +27,14 @@ pub fn receive_message(n_bytes: usize, stream: &mut TcpStream, rbuf: &mut [u8]) let mut recv = 0; while recv < n_bytes { match stream.read(&mut rbuf[recv..]) { - Ok(n) => recv += n, + Ok(n) => { + if n == 0 { + // TODO: Return err instead and handle gracefully + panic!("Connection closed prematurely") + } else { + recv += n + } + } Err(err) => match err.kind() { WouldBlock => {} _ => panic!("Error occurred while reading: {err:?}"), From 16d1eed59f0d5ea9486ad56e4a32107bad667171 Mon Sep 17 00:00:00 2001 From: Jonathan Klimt Date: Thu, 12 Feb 2026 12:12:51 +0100 Subject: [PATCH 17/17] benches: tcp-bw-server: enable thread pinning --- benches/netbench/src/rust-tcp-bw/server.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benches/netbench/src/rust-tcp-bw/server.rs b/benches/netbench/src/rust-tcp-bw/server.rs index 232149ef16..f5f444d340 100644 --- a/benches/netbench/src/rust-tcp-bw/server.rs +++ b/benches/netbench/src/rust-tcp-bw/server.rs @@ -7,8 +7,8 @@ use clap::Parser; use hermit as _; use hermit_bench_output::log_benchmark_data; use rust_tcp_io_perf::config::Config; -use rust_tcp_io_perf::connection; use rust_tcp_io_perf::print_utils::BoxplotValues; +use rust_tcp_io_perf::{connection, threading}; fn receive_rounds( stream: &mut TcpStream, @@ -61,6 +61,7 @@ fn main() { ); let mut stream = connection::server_listen_and_get_first_connection(&args.port.to_string()); connection::setup(&args, &stream); + threading::setup(&args); let _ = receive_rounds(&mut stream, args.warmup, args.n_bytes, false); let durations = receive_rounds(&mut stream, args.n_rounds, args.n_bytes, true);