Skip to content

Commit 4dc0cf7

Browse files
authored
Add MiniExcel NuGet stress benchmark (#6)
1 parent 4dd6870 commit 4dc0cf7

7 files changed

Lines changed: 644 additions & 0 deletions

File tree

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: NuGet Benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
rows:
7+
description: Workbook row count
8+
required: true
9+
default: '100000'
10+
type: string
11+
columns:
12+
description: Workbook column count
13+
required: true
14+
default: '10'
15+
type: string
16+
iterations:
17+
description: Fresh processes per runtime and scenario
18+
required: true
19+
default: '5'
20+
type: string
21+
schedule:
22+
- cron: '17 4 * * 1'
23+
24+
permissions:
25+
contents: read
26+
27+
env:
28+
CARGO_TERM_COLOR: always
29+
BENCHMARK_ROWS: ${{ inputs.rows || '100000' }}
30+
BENCHMARK_COLUMNS: ${{ inputs.columns || '10' }}
31+
BENCHMARK_ITERATIONS: ${{ inputs.iterations || '5' }}
32+
PACKAGE_VERSION: 0.1.0-benchmark.${{ github.run_number }}
33+
34+
jobs:
35+
benchmark:
36+
name: Benchmark ${{ matrix.rid }}
37+
strategy:
38+
fail-fast: false
39+
matrix:
40+
include:
41+
- runner: windows-latest
42+
rid: win-x64
43+
- runner: windows-11-arm
44+
rid: win-arm64
45+
- runner: ubuntu-latest
46+
rid: linux-x64
47+
- runner: ubuntu-24.04-arm
48+
rid: linux-arm64
49+
- runner: macos-15-intel
50+
rid: osx-x64
51+
- runner: macos-latest
52+
rid: osx-arm64
53+
runs-on: ${{ matrix.runner }}
54+
steps:
55+
- uses: actions/checkout@v4
56+
- uses: dtolnay/rust-toolchain@1.85.0
57+
- uses: actions/setup-dotnet@v4
58+
with:
59+
dotnet-version: 8.0.x
60+
- uses: Swatinem/rust-cache@v2
61+
with:
62+
key: nuget-benchmark-${{ matrix.rid }}
63+
- name: Run NuGet stress benchmark
64+
shell: pwsh
65+
run: >-
66+
./scripts/compare-nuget-v1-rust.ps1
67+
-Rid '${{ matrix.rid }}'
68+
-Rows $env:BENCHMARK_ROWS
69+
-Columns $env:BENCHMARK_COLUMNS
70+
-Iterations $env:BENCHMARK_ITERATIONS
71+
-MiniExcelRustVersion $env:PACKAGE_VERSION
72+
- uses: actions/upload-artifact@v4
73+
with:
74+
name: nuget-benchmark-${{ matrix.rid }}
75+
path: |
76+
target/benchmarks/nuget-v1/benchmark-${{ matrix.rid }}.json
77+
target/benchmarks/nuget-v1/benchmark-${{ matrix.rid }}.md
78+
if-no-files-found: error

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/target/
22
/benchmarks/dotnet-v1-query/bin/
33
/benchmarks/dotnet-v1-query/obj/
4+
/benchmarks/nuget-v1-query/bin/
5+
/benchmarks/nuget-v1-query/obj/
46
/dotnet/**/bin/
57
/dotnet/**/obj/
68
/web-demo/dist/
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<MiniExcelRustPackageVersion Condition="'$(MiniExcelRustPackageVersion)' == ''">0.1.0-benchmark</MiniExcelRustPackageVersion>
9+
<MiniExcelVersion Condition="'$(MiniExcelVersion)' == ''">1.46.0</MiniExcelVersion>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<PackageReference Include="MiniExcel" Version="[$(MiniExcelVersion)]" />
14+
<PackageReference Include="MiniExcel.Rust" Version="$(MiniExcelRustPackageVersion)" />
15+
</ItemGroup>
16+
17+
</Project>
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
using System.Diagnostics;
2+
using System.Globalization;
3+
using System.IO.Compression;
4+
using System.Security.Cryptography;
5+
using System.Text;
6+
using System.Text.Json;
7+
using MiniExcelLibs;
8+
using ManagedMiniExcel = MiniExcelLibs.MiniExcel;
9+
10+
if (args.Length == 0)
11+
return Usage();
12+
13+
return args[0].ToLowerInvariant() switch
14+
{
15+
"generate" => Generate(args),
16+
"verify" => Verify(args),
17+
"managed" => Benchmark(args, useRust: false),
18+
"rust" => Benchmark(args, useRust: true),
19+
_ => Usage()
20+
};
21+
22+
static int Generate(string[] arguments)
23+
{
24+
if (arguments.Length != 4 ||
25+
!int.TryParse(arguments[2], out var rowCount) || rowCount < 1 ||
26+
!int.TryParse(arguments[3], out var columnCount) || columnCount is < 1 or > 26)
27+
return Usage();
28+
29+
CreateWorkbook(Path.GetFullPath(arguments[1]), rowCount, columnCount);
30+
return 0;
31+
}
32+
33+
static int Verify(string[] arguments)
34+
{
35+
if (arguments.Length != 2)
36+
return Usage();
37+
38+
var path = Path.GetFullPath(arguments[1]);
39+
using var managed = Query(path, useRust: false).GetEnumerator();
40+
using var rust = Query(path, useRust: true).GetEnumerator();
41+
long rowIndex = 0;
42+
while (true)
43+
{
44+
var hasManaged = managed.MoveNext();
45+
var hasRust = rust.MoveNext();
46+
Require(hasManaged == hasRust, $"Row count differs after row {rowIndex}.");
47+
if (!hasManaged)
48+
break;
49+
CompareRows(managed.Current, rust.Current, rowIndex);
50+
rowIndex++;
51+
}
52+
53+
Console.WriteLine($"Verified {rowIndex} rows against MiniExcel 1.46.0.");
54+
return 0;
55+
}
56+
57+
static int Benchmark(string[] arguments, bool useRust)
58+
{
59+
if (arguments.Length is < 2 or > 4 ||
60+
arguments.Length >= 3 && (!int.TryParse(arguments[2], out var passes) || passes < 1) ||
61+
arguments.Length >= 4 && (!int.TryParse(arguments[3], out var warmups) || warmups < 0))
62+
return Usage();
63+
64+
var path = Path.GetFullPath(arguments[1]);
65+
var measuredPasses = arguments.Length >= 3 ? int.Parse(arguments[2], CultureInfo.InvariantCulture) : 1;
66+
var warmupPasses = arguments.Length >= 4 ? int.Parse(arguments[3], CultureInfo.InvariantCulture) : 0;
67+
for (var pass = 0; pass < warmupPasses; pass++)
68+
Consume(path, useRust);
69+
70+
GC.Collect();
71+
GC.WaitForPendingFinalizers();
72+
GC.Collect();
73+
var allocatedBefore = GC.GetTotalAllocatedBytes(precise: true);
74+
var stopwatch = Stopwatch.StartNew();
75+
var firstRowMilliseconds = 0d;
76+
long rows = 0;
77+
long cells = 0;
78+
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
79+
for (var pass = 0; pass < measuredPasses; pass++)
80+
{
81+
foreach (var row in Query(path, useRust))
82+
{
83+
if (rows == 0)
84+
firstRowMilliseconds = stopwatch.Elapsed.TotalMilliseconds;
85+
rows++;
86+
cells += row.Count;
87+
AppendRow(hash, row);
88+
}
89+
}
90+
stopwatch.Stop();
91+
92+
Console.WriteLine(JsonSerializer.Serialize(new BenchmarkResult(
93+
useRust ? "MiniExcel.Rust" : "MiniExcel",
94+
Environment.Version.ToString(),
95+
measuredPasses,
96+
rows,
97+
cells,
98+
Convert.ToHexString(hash.GetHashAndReset()),
99+
stopwatch.Elapsed.TotalMilliseconds,
100+
firstRowMilliseconds,
101+
GC.GetTotalAllocatedBytes(precise: true) - allocatedBefore)));
102+
return 0;
103+
}
104+
105+
static IEnumerable<IDictionary<string, object?>> Query(string path, bool useRust)
106+
{
107+
if (useRust)
108+
return MiniExcelRust.Query(path, useHeaderRow: false);
109+
return ManagedMiniExcel.Query(path, useHeaderRow: false)
110+
.Cast<IDictionary<string, object?>>();
111+
}
112+
113+
static void Consume(string path, bool useRust)
114+
{
115+
foreach (var row in Query(path, useRust))
116+
_ = row.Count;
117+
}
118+
119+
static void CompareRows(
120+
IDictionary<string, object?> managed,
121+
IDictionary<string, object?> rust,
122+
long rowIndex)
123+
{
124+
Require(managed.Keys.SequenceEqual(rust.Keys, StringComparer.Ordinal),
125+
$"Column order differs at row {rowIndex}.");
126+
foreach (var key in managed.Keys)
127+
{
128+
Require(Normalize(managed[key]) == Normalize(rust[key]),
129+
$"Value differs at row {rowIndex}, column {key}: managed={managed[key]}, rust={rust[key]}.");
130+
}
131+
}
132+
133+
static void AppendRow(IncrementalHash hash, IDictionary<string, object?> row)
134+
{
135+
foreach (var cell in row)
136+
{
137+
AppendText(hash, cell.Key);
138+
AppendText(hash, Normalize(cell.Value));
139+
}
140+
}
141+
142+
static void AppendText(IncrementalHash hash, string value)
143+
{
144+
var bytes = Encoding.UTF8.GetBytes(value);
145+
hash.AppendData(BitConverter.GetBytes(bytes.Length));
146+
hash.AppendData(bytes);
147+
}
148+
149+
static string Normalize(object? value) => value switch
150+
{
151+
null or DBNull => "null",
152+
IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty,
153+
_ => value.ToString() ?? string.Empty
154+
};
155+
156+
static void CreateWorkbook(string path, int rows, int columns)
157+
{
158+
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
159+
if (File.Exists(path))
160+
File.Delete(path);
161+
using var archive = ZipFile.Open(path, ZipArchiveMode.Create);
162+
AddEntry(archive, "[Content_Types].xml", """
163+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
164+
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
165+
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
166+
<Default Extension="xml" ContentType="application/xml"/>
167+
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
168+
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
169+
</Types>
170+
""");
171+
AddEntry(archive, "_rels/.rels", """
172+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
173+
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
174+
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
175+
</Relationships>
176+
""");
177+
AddEntry(archive, "xl/workbook.xml", """
178+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
179+
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
180+
<sheets><sheet name="Sheet1" sheetId="1" r:id="rId1"/></sheets>
181+
</workbook>
182+
""");
183+
AddEntry(archive, "xl/_rels/workbook.xml.rels", """
184+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
185+
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
186+
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
187+
</Relationships>
188+
""");
189+
190+
var entry = archive.CreateEntry("xl/worksheets/sheet1.xml", CompressionLevel.Fastest);
191+
using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(false));
192+
writer.Write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData>");
193+
for (var row = 1; row <= rows; row++)
194+
{
195+
writer.Write($"<row r=\"{row}\">");
196+
for (var column = 1; column <= columns; column++)
197+
{
198+
var reference = $"{(char)('A' + column - 1)}{row}";
199+
var value = (long)(row - 1) * columns + column;
200+
writer.Write($"<c r=\"{reference}\"><v>{value}</v></c>");
201+
}
202+
writer.Write("</row>");
203+
}
204+
writer.Write("</sheetData></worksheet>");
205+
}
206+
207+
static void AddEntry(ZipArchive archive, string name, string contents)
208+
{
209+
var entry = archive.CreateEntry(name, CompressionLevel.Fastest);
210+
using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(false));
211+
writer.Write(contents);
212+
}
213+
214+
static void Require(bool condition, string message)
215+
{
216+
if (!condition)
217+
throw new InvalidOperationException(message);
218+
}
219+
220+
static int Usage()
221+
{
222+
Console.Error.WriteLine("Usage:");
223+
Console.Error.WriteLine(" NuGetV1Query generate <xlsx-path> <rows> <columns>");
224+
Console.Error.WriteLine(" NuGetV1Query verify <xlsx-path>");
225+
Console.Error.WriteLine(" NuGetV1Query <managed|rust> <xlsx-path> [passes] [warmup-passes]");
226+
return 2;
227+
}
228+
229+
internal sealed record BenchmarkResult(
230+
string Runtime,
231+
string DotNetRuntime,
232+
int Passes,
233+
long Rows,
234+
long Cells,
235+
string ContentHash,
236+
double ElapsedMilliseconds,
237+
double FirstRowMilliseconds,
238+
long AllocatedBytes);

‎docs/dotnet-v1-query-benchmark.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,25 @@ This benchmark compares dynamic, headerless XLSX streaming over the same workboo
1111

1212
Both runners enumerate every returned row without retaining the complete worksheet. Save performance, typed mapping, formulas, and other APIs are outside this comparison.
1313

14+
### NuGet-To-NuGet Stress Harness
15+
16+
To benchmark the distributable package rather than the Rust CLI, run:
17+
18+
```powershell
19+
pwsh ./scripts/compare-nuget-v1-rust.ps1
20+
```
21+
22+
This harness builds a local `MiniExcel.Rust` package, resolves the latest stable public MiniExcel v1
23+
package, restores both into an isolated `net8.0` consumer, and compares `MiniExcel.Query` with
24+
`MiniExcelRust.Query`. Before timing, it verifies every row, column, and normalized value. Cold and
25+
steady scenarios run in alternating fresh processes and report query time, first-row latency,
26+
managed allocation, peak working set, and peak private memory to
27+
`target/benchmarks/nuget-v1/benchmark-<rid>.{json,md}`.
28+
29+
Use `-Rows`, `-Columns`, `-Iterations`, `-Passes`, and `-WarmupPasses` to change the load. The
30+
`NuGet Benchmark` GitHub workflow runs the same harness on Windows, Linux, and macOS for x64 and
31+
Arm64. Pass `-MiniExcelVersion 1.46.0` to pin a historical baseline for reproducible comparisons.
32+
1433
## Fairness Controls
1534

1635
- Both runners use Release builds, the same workbook, and equivalent public dynamic Query APIs.

‎docs/dotnet-v1-query-benchmark.zh-CN.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,23 @@
1111

1212
两个 runner 都会遍历所有返回行,但不会把完整 worksheet 保留在内存中。本测试不包含 Save、类型映射、公式及其他 API。
1313

14+
### NuGet 对 NuGet 压力测试
15+
16+
如需比较实际发布包而不是 Rust CLI,请运行:
17+
18+
```powershell
19+
pwsh ./scripts/compare-nuget-v1-rust.ps1
20+
```
21+
22+
该脚本先构建本地 `MiniExcel.Rust` 包,自动解析 NuGet 上最新的稳定 MiniExcel v1,再创建隔离的
23+
`net8.0` 消费者并对比 `MiniExcel.Query` 与 `MiniExcelRust.Query`。计时前会逐行、逐列、
24+
逐值验证结果;Cold 与 Steady 场景使用交替的新进程执行,并将 Query 耗时、首行延迟、托管分配、
25+
峰值工作集与峰值私有内存写入 `target/benchmarks/nuget-v1/benchmark-<rid>.{json,md}`。
26+
27+
可通过 `-Rows`、`-Columns`、`-Iterations`、`-Passes` 和 `-WarmupPasses` 调整压力。
28+
GitHub 的 `NuGet Benchmark` workflow 会在 Windows、Linux、macOS 的 x64 与 Arm64 环境运行同一套测试。
29+
如需复现历史结果,可传入 `-MiniExcelVersion 1.46.0` 固定基线版本。
30+
1431
## 公平性控制
1532

1633
- 两个 runner 都使用 Release 构建、同一份工作簿和语义等价的公开动态 Query API。

0 commit comments

Comments
 (0)