Skip to content

Commit 1d59f05

Browse files
849 by category or by render type (#903)
1 parent 89f9148 commit 1d59f05

166 files changed

Lines changed: 153663 additions & 269352 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CONTRIBUTING.md‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,11 @@ commands for reproducing each job are in [Measure Framework](./.github/workflows
100100

101101
### Result Schemas
102102

103-
[`schemas.ts`](./packages/results/src/schemas.ts) defines the Zod schemas used
104-
to validate benchmark output and derive TypeScript types. Record what actually
105-
ran, the samples, and the summaries in the benchmark result using the fields
106-
defined by its schema. Update `FrameworkSchema` and `PackagePathSchema` when
107-
adding a framework.
108-
[`storage.ts`](./packages/results/src/storage.ts) uses those schemas to validate
109-
the combined JSON and associate each measurement with its starter or app.
103+
`benchmark-schemas.ts` in `packages/results/src/` defines the schemas for individual
104+
benchmark artifacts and the assembled framework result. `benchmark-types.ts`
105+
defines framework names, package paths, and metric units. Each benchmark records
106+
its project, samples, and summaries. CI validates artifacts and assembles the
107+
latest and versioned results.
110108

111109
### How the Docs Use Results
112110

@@ -134,10 +132,10 @@ For a newer patch within a major or minor version already tracked (for example,
134132
SolidStart `2.0.0` to `2.0.5`), update the exact framework version in both the
135133
starter and app `package.json` files, regenerate both lockfiles, and update the
136134
starter's `first-party-dependencies.json`. Under `packages/results/data/<framework>/`,
137-
update `frameworkVersion` and the framework dependency version for each affected
138-
project in `latest.json` and `versions/<old-version>.json`. Rename the versioned
139-
file to the new patch version to replace the old entry in the major or minor
140-
history.
135+
update the top-level `frameworkVersion` and the framework dependency version
136+
in each affected project's `packageJson` in `latest.json` and
137+
`versions/<old-version>.json`. Rename the versioned file to the new patch
138+
version to replace the old entry in the major or minor history.
141139
Leave measured values and diagnostic messages unchanged; they still reflect the
142140
previous patch until the Benchmarks workflow reruns after the change reaches
143141
`main` and opens a stats update PR.

‎packages/docs/src/benchmarks/browser-baseline/FrameworkBrowserBaselineVersionTable.astro‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@ import {
33
formatBaselineReason,
44
formatBaselineStatus,
55
} from '../../lib/prepare-table'
6-
import type { FrameworkResult } from '@framework-performance/results'
6+
import type { BrowserBaselineBenchmark } from '@framework-performance/results/benchmark-schemas'
7+
import type { z } from 'astro/zod'
78
import StatsTable, { type TableColumn } from '../../components/StatsTable.astro'
89
910
interface BrowserBaselineTests {
1011
baselineStatus?: 'high' | 'low' | false | null
1112
baselineYear?: number | null
1213
baselineReason?: string | string[] | null
13-
baselineFeatureCount?: NonNullable<
14-
FrameworkResult['browserBaseline']
14+
baselineFeatureCount?: z.output<
15+
typeof BrowserBaselineBenchmark
1516
>['baselineFeatureCount']
1617
}
1718

‎packages/docs/src/benchmarks/dependency-counts/FirstPartyDependencyCharts.astro‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,14 @@ import ComparisonBarChart from '../../components/ComparisonBarChart.astro'
1414
import ComparisonLineChart from '../../components/ComparisonLineChart.astro'
1515
1616
type DependencyMetric =
17-
| 'totalDependencyCount'
1817
| 'developmentDependencyCount'
1918
| 'productionDependencyCount'
2019
| 'duplicateDependencyCount'
2120
2221
function getDependencyChartData(metric: DependencyMetric) {
2322
return starterStats
2423
.flatMap((framework) => {
25-
const value = framework.firstPartyDependencies[metric]?.value
24+
const value = framework.firstPartyDependencies?.[metric]?.value
2625
if (value == null || !Number.isFinite(value)) return []
2726
2827
return [

‎packages/docs/src/benchmarks/initial-load/Details.astro‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
---
2-
import type { InitialLoadStats } from '@framework-performance/results'
2+
import type { SSRInitialLoadBenchmark } from '@framework-performance/results/benchmark-schemas'
3+
import type { z } from 'astro/zod'
34
import { Content as Description } from './Description.mdx'
45
import StackedAreaChart from '../../components/StackedAreaChart.astro'
56
import { initialLoadSeries } from '../../lib/prepare-chart'
67
import { prepareInitialLoadResources } from '../../lib/initial-load'
78
interface Props {
8-
versions: { frameworkVersion?: string; initialLoad?: InitialLoadStats }[]
9+
versions: {
10+
frameworkVersion?: string
11+
initialLoad?: z.output<typeof SSRInitialLoadBenchmark>
12+
}[]
913
}
1014
const data = Astro.props.versions.flatMap((version) => {
1115
const resources = version.initialLoad?.resources

‎packages/docs/src/content/docs/methodology.md‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -222,11 +222,10 @@ pnpm dlx create-void starter-void --toolchain vite --framework react --starter s
222222

223223
- **Core package:** Measures the direct production and development dependencies
224224
declared by the framework package itself.
225-
- **First-party packages:** Measures the framework-maintained packages listed in
226-
each starter project's `first-party-dependencies.json` and their resolved
227-
dependency graph, including duplicate versions.
228-
- **Starter project:** Measures the full starter project's direct dependencies,
229-
resolved dependency graph, duplicates, and install size.
225+
- **First-party packages:** Counts the direct dependencies listed in
226+
`first-party-dependencies.json` and duplicate versions in their resolved graph.
227+
- **Starter project:** Counts direct dependencies and duplicate versions in the
228+
full resolved graph, and measures install size.
230229

231230
### Node Modules Size
232231

‎packages/docs/src/lib/chart-measurements.ts‎

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -158,20 +158,6 @@ export function prepareFrameworkDependenciesDevDependenciesHistory(
158158
)
159159
}
160160

161-
export function prepareFrameworkDependenciesAllDependenciesHistory(
162-
records: readonly ChartMeasurement[],
163-
): VersionLineChartPayload {
164-
return history(
165-
metricCatalogue['frameworkDependencies.totalDependencyCount'],
166-
records.map((record) => ({
167-
record,
168-
value: finiteValue(
169-
record.frameworkDependencies?.totalDependencyCount?.value,
170-
),
171-
})),
172-
)
173-
}
174-
175161
export function prepareProdDependenciesHistory(
176162
records: readonly ChartMeasurement[],
177163
): VersionLineChartPayload {
@@ -248,18 +234,6 @@ export function prepareDevDependenciesMonthlyComparison(
248234
)
249235
}
250236

251-
export function prepareAllDependenciesHistory(
252-
records: readonly ChartMeasurement[],
253-
): VersionLineChartPayload {
254-
return history(
255-
metricCatalogue['totalDependencyCount'],
256-
records.map((record) => ({
257-
record,
258-
value: finiteValue(record.totalDependencyCount?.value),
259-
})),
260-
)
261-
}
262-
263237
export function prepareDuplicateDependenciesHistory(
264238
records: readonly ChartMeasurement[],
265239
): VersionLineChartPayload {
@@ -427,7 +401,7 @@ export function prepareInstallTimeHistory(
427401
metricCatalogue['install.duration'],
428402
records.map((record) => ({
429403
record,
430-
value: finiteValue(record.install.duration.median, 1000),
404+
value: finiteValue(record.install?.duration?.median, 1000),
431405
})),
432406
)
433407
}
@@ -439,7 +413,7 @@ export function prepareInstallTimeComparison(
439413
metricCatalogue['install.duration'],
440414
records.map((record) => ({
441415
record,
442-
value: finiteValue(record.install.duration.median, 1000),
416+
value: finiteValue(record.install?.duration?.median, 1000),
443417
})),
444418
)
445419
}
@@ -744,7 +718,7 @@ export function prepareNodeModulesSizeHistory(
744718
metricCatalogue['nodeModulesSize'],
745719
records.map((record) => ({
746720
record,
747-
value: finiteValue(record.install.nodeModulesSize?.value, 1048576),
721+
value: finiteValue(record.install?.nodeModulesSize?.value, 1048576),
748722
})),
749723
)
750724
}
@@ -756,7 +730,7 @@ export function prepareNodeModulesSizeComparison(
756730
metricCatalogue['nodeModulesSize'],
757731
records.map((record) => ({
758732
record,
759-
value: finiteValue(record.install.nodeModulesSize?.value, 1048576),
733+
value: finiteValue(record.install?.nodeModulesSize?.value, 1048576),
760734
})),
761735
)
762736
}

‎packages/docs/src/lib/initial-load.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import type { InitialLoadStats } from '@framework-performance/results'
1+
import type { SSRInitialLoadBenchmark } from '@framework-performance/results/benchmark-schemas'
2+
import type { z } from 'astro/zod'
23

34
export function prepareInitialLoadResources(
4-
resources: InitialLoadStats['resources'],
5+
resources: z.output<typeof SSRInitialLoadBenchmark>['resources'],
56
) {
67
return {
78
html: resources.document.transferSize.value / 1024,

‎packages/docs/src/lib/interaction-history.test.ts‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
prepareLighthouseClientSideRenderedTestsInteractionHistory,
55
prepareLighthouseServerSideRenderedTestsInteractionHistory,
66
} from './chart-measurements.ts'
7-
import { formatRendered } from './result-content.ts'
7+
import { formatLighthouse } from './result-content.ts'
88

99
const timings = (
1010
inputDelay: number,
@@ -32,31 +32,34 @@ test('rendered tests take interaction phases from the run with the median total'
3232
timings(5, 40, 20),
3333
timings(3, 12, 10),
3434
]
35-
const formatted = formatRendered({
36-
name: 'clientSideRendered',
37-
runs: runs.length,
38-
iterations: runs.map((interactionTests) => ({
39-
firstPaint: { value: 100, unit: 'ms' },
40-
firstContentfulPaint: { value: 100, unit: 'ms' },
41-
interactionTests: {
42-
inputDelay: { value: interactionTests.inputDelay, unit: 'ms' },
43-
processingDuration: {
44-
value: interactionTests.processingDuration,
45-
unit: 'ms',
35+
const formatted = formatLighthouse({
36+
name: 'csrLighthouse',
37+
project: 'app',
38+
iterations: {
39+
runs: runs.length,
40+
iterations: runs.map((interactionTests) => ({
41+
firstPaint: { value: 100, unit: 'ms' },
42+
firstContentfulPaint: { value: 100, unit: 'ms' },
43+
interaction: {
44+
inputDelay: { value: interactionTests.inputDelay, unit: 'ms' },
45+
processingDuration: {
46+
value: interactionTests.processingDuration,
47+
unit: 'ms',
48+
},
49+
presentationDelay: {
50+
value: interactionTests.presentationDelay,
51+
unit: 'ms',
52+
},
53+
interactionLatency: {
54+
value: interactionTests.interactionLatency,
55+
unit: 'ms',
56+
},
4657
},
47-
presentationDelay: {
48-
value: interactionTests.presentationDelay,
49-
unit: 'ms',
50-
},
51-
interactionLatency: {
52-
value: interactionTests.interactionLatency,
53-
unit: 'ms',
54-
},
55-
},
56-
})),
58+
})),
59+
},
5760
firstPaint: metric(100),
5861
firstContentfulPaint: metric(100),
59-
interactionTests: {
62+
interaction: {
6063
interactionLatency: metric(30),
6164
inputDelay: metric(2),
6265
processingDuration: metric(20),

‎packages/docs/src/lib/metrics.ts‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,6 @@ export const metricCatalogue = {
2020
valueFormat: 'count',
2121
yAxisLabel: 'Dependencies',
2222
},
23-
'frameworkDependencies.totalDependencyCount': {
24-
id: 'frameworkDependencies.totalDependencyCount',
25-
title: 'Core Package \u00b7 All Dependencies',
26-
valueFormat: 'count',
27-
yAxisLabel: 'Dependencies',
28-
},
2923
productionDependencyCount: {
3024
id: 'productionDependencyCount',
3125
title: 'Starter Project \u00b7 Production Dependencies',
@@ -38,12 +32,6 @@ export const metricCatalogue = {
3832
valueFormat: 'count',
3933
yAxisLabel: 'Dependencies',
4034
},
41-
totalDependencyCount: {
42-
id: 'totalDependencyCount',
43-
title: 'Starter Project \u00b7 All Dependencies',
44-
valueFormat: 'count',
45-
yAxisLabel: 'Dependencies',
46-
},
4735
duplicateDependencyCount: {
4836
id: 'duplicateDependencyCount',
4937
title: 'Starter Project \u00b7 Duplicate Dependencies',

‎packages/docs/src/lib/prepare-chart.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,33 @@ import { prepareInitialLoadResources } from './initial-load'
44
import { getCollection } from 'astro:content'
55
import { frameworksConfig } from '@framework-performance/results/frameworks-config'
66
export * from './chart-measurements'
7-
import type { CombinedResult } from '@framework-performance/results/storage'
7+
import { BenchmarkResultSchema } from '@framework-performance/results/benchmark-schemas'
88
import { prepareRuntimeResult, prepareStarterResult } from './result-content'
99
import type { ChartMeasurement, StackedAreaChartPayload } from './types'
1010

11-
const resultFiles = import.meta.glob<CombinedResult>(
11+
const resultFiles = import.meta.glob<unknown>(
1212
'../../../results/data/**/*.json',
1313
{ eager: true, import: 'default' },
1414
)
1515
const latest = Object.entries(resultFiles)
1616
.filter(([path]) => path.endsWith('/latest.json'))
17-
.map(([, result]) => result)
17+
.map(([, result]) => BenchmarkResultSchema.parse(result))
1818
.sort(
1919
(a, b) =>
2020
frameworksConfig.findIndex((f) => f.name === a.framework) -
2121
frameworksConfig.findIndex((f) => f.name === b.framework),
2222
)
2323
const history = Object.entries(resultFiles)
2424
.filter(([path]) => path.includes('/versions/'))
25-
.map(([, result]) => result)
25+
.map(([, result]) => BenchmarkResultSchema.parse(result))
2626
export const starterStats = latest
27-
.filter((result) => result.projects.starter)
27+
.filter((result) => result.framework !== 'baseline-html')
2828
.map(prepareStarterResult)
2929
export const ssrRequestThroughputStats = latest
3030
.filter((result) => result.projects.app)
3131
.map(prepareRuntimeResult)
3232
export const starterHistory = history
33-
.filter((result) => result.projects.starter)
33+
.filter((result) => result.framework !== 'baseline-html')
3434
.map(prepareStarterResult)
3535
export const runtimeHistory = history
3636
.filter((result) => result.projects.app)
@@ -188,7 +188,7 @@ export function prepareDependencyHistoryChart(
188188
{
189189
name: version.frameworkVersion ?? 'Unknown',
190190
values: { prod, dev },
191-
total: counts?.totalDependencyCount?.value ?? null,
191+
total: prod + dev,
192192
},
193193
]
194194
})
@@ -197,6 +197,6 @@ export function prepareDependencyHistoryChart(
197197
series: dependencySeries,
198198
valueFormat: 'count',
199199
yAxisLabel: 'Dependencies',
200-
totalLabel: 'Measured total',
200+
totalLabel: 'Direct dependencies',
201201
}
202202
}

0 commit comments

Comments
 (0)