Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/validate-stats-pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Validate Stats PR

on:
pull_request:
paths:
- 'packages/starter-*/ci-stats.json'
- 'packages/starter-*/stats/**'
- 'packages/app-*/ci-stats.json'
- 'packages/app-*/stats/**'
- 'packages/docs/src/content/devtime/**'
- 'packages/docs/src/content/runtime/**'
- 'packages/stats-generator/**'
- '.github/workflows/validate-stats-pr.yml'

jobs:
validate-stats-json:
runs-on: depot-ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- name: Setup pnpm
uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5

- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: '24'
cache: 'pnpm'

- name: Install workspace dependencies
run: pnpm install --frozen-lockfile

- name: Run test suite
run: pnpm --filter @framework-tracker/stats-generator test

- name: Validate stats JSON files
run: pnpm --filter @framework-tracker/stats-generator validate:json
1 change: 1 addition & 0 deletions packages/stats-generator/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"save:ci-stats": "node src/save-ci-stats.ts",
"generate:preview-comment": "node src/generate-preview-comment.ts",
"validate": "node src/validate-stats.ts",
"validate:json": "node src/validate-json-stats.ts",
"sync:versions": "node src/sync-versions.ts",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
Expand Down
61 changes: 61 additions & 0 deletions packages/stats-generator/src/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,65 @@ export const ServerSideRenderedStatsSchema = z.object({
browserVersion: z.string().min(1),
})

export const StarterCIStatsSchema = z.object({
timingMeasuredAt: z.string().min(1),
runner: z.string().min(1),
frameworkVersion: z.string().min(1),
installTime: TimeStatSchema,
coldBuildTime: TimeStatSchema,
warmBuildTime: TimeStatSchema,
nodeModulesSize: z.number().positive(),
buildOutputSize: z.number().positive(),
allDependencies: z.number().positive(),
prodDependencies: z.number().nonnegative(),
devDependencies: z.number().nonnegative(),
duplicateDependencies: z.number().nonnegative().optional(),
depInstallSize: z.number().positive().optional(),
browserBaselineTests: BrowserBaselineStatsSchema.optional(),
frameworkDependencies: z
.object({
prodDependencies: z.number().nonnegative().optional(),
devDependencies: z.number().nonnegative().optional(),
allDependencies: z.number().nonnegative().optional(),
})
.optional(),
packageJson: z
.object({
dependencies: z.record(z.string(), z.string()).optional(),
devDependencies: z.record(z.string(), z.string()).optional(),
})
.optional(),
})

export const AppCIStatsSchema = z
.object({
timingMeasuredAt: z.string().min(1),
runner: z.string().min(1),
frameworkVersion: z.string().min(1).optional(),
browserVersion: z.string().min(1).optional(),
ssrRequestThroughputTests:
SSRRequestThroughputStatsSchema.shape.ssrRequestThroughputTests.optional(),
ssrLoadTests: SSRLoadStatsSchema.shape.ssrLoadTests.optional(),
clientSideRenderedTests: RenderedTestsSchema.optional(),
serverSideRenderedTests: RenderedTestsSchema.optional(),
packageJson: z
.object({
dependencies: z.record(z.string(), z.string()).optional(),
devDependencies: z.record(z.string(), z.string()).optional(),
})
.optional(),
})
.refine(
(stats) =>
stats.ssrRequestThroughputTests !== undefined ||
stats.ssrLoadTests !== undefined ||
stats.clientSideRenderedTests !== undefined ||
stats.serverSideRenderedTests !== undefined,
{
message: 'Expected at least one benchmark test measurement in app stats',
},
)

export type InstallStats = z.infer<typeof InstallStatsSchema>
export type BuildStats = z.infer<typeof BuildStatsSchema>
export type BrowserBaselineStats = z.infer<typeof BrowserBaselineStatsSchema>
Expand All @@ -177,4 +236,6 @@ export type ClientSideRenderedStats = z.infer<
export type ServerSideRenderedStats = z.infer<
typeof ServerSideRenderedStatsSchema
>
export type StarterCIStats = z.infer<typeof StarterCIStatsSchema>
export type AppCIStats = z.infer<typeof AppCIStatsSchema>
export type TimeStat = z.infer<typeof TimeStatSchema>
88 changes: 88 additions & 0 deletions packages/stats-generator/src/validate-json-stats.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import { StarterCIStatsSchema, AppCIStatsSchema } from './schemas.ts'

describe('Stats Validation Schemas', () => {
it('validates a valid starter ci-stats payload', () => {
const validStarter = {
timingMeasuredAt: '2026-08-29T05:27:13.181Z',
runner: 'Depot runner',
frameworkVersion: '7.1.6',
installTime: {
avgMs: 2759.2,
standardDeviationMs: 679,
minMs: 1671,
maxMs: 3464,
samplesMs: [1671, 2624, 2947, 3464, 3090],
},
coldBuildTime: {
avgMs: 797.8,
standardDeviationMs: 5.07,
minMs: 793,
maxMs: 805,
samplesMs: [801, 805, 794, 796, 793],
},
warmBuildTime: {
avgMs: 764,
standardDeviationMs: 6.2,
minMs: 757,
maxMs: 774,
samplesMs: [763, 774, 764, 757, 762],
},
nodeModulesSize: 218034176,
buildOutputSize: 32768,
prodDependencies: 1,
devDependencies: 2,
allDependencies: 387,
}

const result = StarterCIStatsSchema.safeParse(validStarter)
assert.equal(result.success, true)
})

it('rejects starter stats with zero nodeModulesSize or missing installTime', () => {
const invalidStarter = {
timingMeasuredAt: '2026-08-29T05:27:13.181Z',
runner: 'Depot runner',
frameworkVersion: '7.1.6',
nodeModulesSize: 0,
buildOutputSize: 32768,
prodDependencies: 1,
devDependencies: 2,
allDependencies: 387,
}

const result = StarterCIStatsSchema.safeParse(invalidStarter)
assert.equal(result.success, false)
})

it('validates app stats with throughput or render tests', () => {
const validApp = {
timingMeasuredAt: '2026-08-29T05:27:13.181Z',
runner: 'Depot runner',
frameworkVersion: '7.1.6',
ssrRequestThroughputTests: {
opsPerSec: 1540.5,
avgLatencyMs: 0.65,
medianLatencyMs: 0.61,
samples: 100,
bodySizeKb: 14.2,
duplicationFactor: 1,
},
}

const result = AppCIStatsSchema.safeParse(validApp)
assert.equal(result.success, true)
})

it('rejects app stats with no benchmark tests', () => {
const emptyApp = {
timingMeasuredAt: '2026-08-29T05:27:13.181Z',
runner: 'Depot runner',
frameworkVersion: '7.1.6',
}

const result = AppCIStatsSchema.safeParse(emptyApp)
assert.equal(result.success, false)
})
})
157 changes: 157 additions & 0 deletions packages/stats-generator/src/validate-json-stats.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
import { join } from 'node:path'
import { readdirSync, existsSync } from 'node:fs'
import { z } from 'zod'
import { getFrameworks } from './get-frameworks.ts'
import { packagesDir } from './constants.ts'
import { readJsonFile } from './utils.ts'
import { StarterCIStatsSchema, AppCIStatsSchema } from './schemas.ts'

function validateFile(filePath: string, schema: z.ZodSchema): string[] {
const data = readJsonFile(filePath)
if (!data) {
return [`File not found or unparseable: ${filePath}`]
}

const result = schema.safeParse(data)
if (result.success) {
return []
}

return result.error.issues.map(
(issue) => `${issue.path.join('.') || 'root'}: ${issue.message}`,
)
}

function findJsonFilesInDir(dir: string): string[] {
if (!existsSync(dir)) return []
const files: string[] = []
const entries = readdirSync(dir, { withFileTypes: true })
for (const entry of entries) {
const fullPath = join(dir, entry.name)
if (entry.isDirectory()) {
files.push(...findJsonFilesInDir(fullPath))
} else if (entry.isFile() && entry.name.endsWith('.json')) {
files.push(fullPath)
}
}
return files
}

async function main() {
console.info('=== Validating Generated Stats JSON Files ===\n')

const frameworks = await getFrameworks()
let hasFailures = false
let totalFilesChecked = 0

for (const framework of frameworks) {
if (framework.starter) {
const pkg = framework.starter.package
const starterDir = join(packagesDir, pkg)
console.info(`Validating ${framework.displayName} starter (${pkg})...`)

const jsonFiles: string[] = []
const ciStats = join(starterDir, 'ci-stats.json')
if (existsSync(ciStats)) jsonFiles.push(ciStats)

const statsDir = join(starterDir, 'stats')
if (existsSync(statsDir)) {
jsonFiles.push(...findJsonFilesInDir(statsDir))
}

for (const filePath of jsonFiles) {
totalFilesChecked++
const errors = validateFile(filePath, StarterCIStatsSchema)
if (errors.length > 0) {
console.error(` ✗ ${filePath}`)
errors.forEach((e) => console.error(` ${e}`))
hasFailures = true
} else {
console.info(` ✓ ${filePath}`)
}
}
console.info('')
}

if (framework.app) {
const pkg = framework.app.package
const appDir = join(packagesDir, pkg)
console.info(`Validating ${framework.displayName} app (${pkg})...`)

const jsonFiles: string[] = []
const ciStats = join(appDir, 'ci-stats.json')
if (existsSync(ciStats)) jsonFiles.push(ciStats)

const statsDir = join(appDir, 'stats')
if (existsSync(statsDir)) {
jsonFiles.push(...findJsonFilesInDir(statsDir))
}

for (const filePath of jsonFiles) {
totalFilesChecked++
const errors = validateFile(filePath, AppCIStatsSchema)
if (errors.length > 0) {
console.error(` ✗ ${filePath}`)
errors.forEach((e) => console.error(` ${e}`))
hasFailures = true
} else {
console.info(` ✓ ${filePath}`)
}
}
console.info('')
}
}

// Validate Docs Devtime & Runtime JSON files
const docsDevtimeDir = join(packagesDir, 'docs', 'src', 'content', 'devtime')
if (existsSync(docsDevtimeDir)) {
console.info('Validating docs devtime content JSON files...')
const devtimeFiles = findJsonFilesInDir(docsDevtimeDir)
for (const filePath of devtimeFiles) {
totalFilesChecked++
const errors = validateFile(filePath, StarterCIStatsSchema)
if (errors.length > 0) {
console.error(` ✗ ${filePath}`)
errors.forEach((e) => console.error(` ${e}`))
hasFailures = true
} else {
console.info(` ✓ ${filePath}`)
}
}
console.info('')
}

const docsRuntimeDir = join(packagesDir, 'docs', 'src', 'content', 'runtime')
if (existsSync(docsRuntimeDir)) {
console.info('Validating docs runtime content JSON files...')
const runtimeFiles = findJsonFilesInDir(docsRuntimeDir)
for (const filePath of runtimeFiles) {
totalFilesChecked++
const errors = validateFile(filePath, AppCIStatsSchema)
if (errors.length > 0) {
console.error(` ✗ ${filePath}`)
errors.forEach((e) => console.error(` ${e}`))
hasFailures = true
} else {
console.info(` ✓ ${filePath}`)
}
}
console.info('')
}

if (hasFailures) {
console.error(
`Validation failed! Some JSON stats files contain invalid or missing data.`,
)
process.exit(1)
}

console.info(
`✓ Successfully validated all ${totalFilesChecked} JSON stats files!`,
)
}

main().catch((error) => {
console.error('Validation error:', error)
process.exit(1)
})