From 352fb74533a6d27851c56b63f27a7e463f81c720 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 07:49:57 +0900 Subject: [PATCH 1/8] feat(tools): add the stage performance statistics CLI amadeus-stage-stats.ts derives a per-stage duration baseline from the audit shards and the review-iteration distribution from the record artefacts. Stage windows pair STAGE_STARTED with STAGE_COMPLETED; the reported net duration subtracts approval waits, parks and session gaps, which is what makes raw wall-clock unusable as a performance signal. net is an estimate, and the header says so on every run. Exclusion buckets are layered by the three disjoint populations they count in, so the window identity stays checkable on the output: constructed windows = net population + unclosed-idle + zero-second. Every bucket is reported whatever its value, and a shard that exists but cannot be read is fail-loud at the exit code. Two-generation journal normalization is reused from amadeus-journal.ts rather than reimplemented; the tool imports no filesystem write API. Refs #2405 --- .../core/tools/amadeus-stage-stats.ts | 944 ++++++++++++++++++ tests/.coverage-registry.json | 4 + .../t485-stage-stats.integration.test.ts | 406 ++++++++ tests/unit/t484-stage-stats.test.ts | 523 ++++++++++ 4 files changed, 1877 insertions(+) create mode 100644 packages/framework/core/tools/amadeus-stage-stats.ts create mode 100644 tests/integration/t485-stage-stats.integration.test.ts create mode 100644 tests/unit/t484-stage-stats.test.ts diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts new file mode 100644 index 000000000..668d7d4e1 --- /dev/null +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -0,0 +1,944 @@ +// Stage performance statistics — the read-only aggregation CLI (#2405). +// +// Usage: bun amadeus-stage-stats.ts [--project-dir ] [--space ] +// [--format markdown|csv|json] [--json] +// +// One command derives the per-stage duration baseline from the audit shards +// (amadeus/spaces//intents/* /audit/*.jsonl) plus the §12a review +// iteration distribution from the record artefacts. Stage windows are +// STAGE_STARTED -> STAGE_COMPLETED pairs; the reported net duration subtracts +// the idle spans (approval waits, parks, session gaps) that make raw +// wall-clock unusable as a performance signal. +// +// net is an ESTIMATE. The header says so on every run: subtracting idle is a +// hypothesis about working time, not a measurement of it. +// +// Reading contract mirrored from the sibling amadeus-subagent-stats.ts: +// measurement ref first, absence recorded as absence (a row without Model is +// the UNKNOWN bucket, never dropped), and a shard that exists but cannot be +// read is fail-loud at the exit code. Nothing here writes. +// +// The two-generation journal normalization is NOT reimplemented here: schema +// v1 (event/fields) and v2 (eventName/attributes) both arrive through +// amadeus-journal.ts's exported reader and field accessor. This module +// deliberately does not import amadeus-lib.ts (the same dependency-direction +// ruling the sibling tool records), and it does not reference the test tree — +// the nearest-rank p95 is mirrored locally because the shipped surface must +// not reach into tests. + +import { readdirSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { type JournalRecord, journalRecordField, readJournalRecords } from "./amadeus-journal.ts"; + +// --- domain types ---------------------------------------------------------- + +/** One journal record together with the intent its shard path places it in. */ +export interface AttributedRecord { + readonly intent: string; + readonly record: JournalRecord; +} + +/** The scan phase's output: what was read, and what could not be. */ +export interface ScannedCorpus { + readonly records: readonly AttributedRecord[]; + readonly unreadableShardCount: number; + readonly brokenLineCount: number; + readonly shardCount: number; + readonly lineCount: number; +} + +/** A STAGE_STARTED -> STAGE_COMPLETED pair. Timestamps are second-grained. */ +export interface StageWindow { + readonly intent: string; + readonly stage: string; + readonly startedAt: string; + readonly completedAt: string; + readonly rawSeconds: number; +} + +/** A window with its idle span subtracted. Invariant: netSeconds >= 0. */ +export interface MeasuredWindow extends StageWindow { + readonly idleSeconds: number; + readonly netSeconds: number; +} + +// The closed set of exclusion buckets, layered by the three DISJOINT +// populations they count in. A flat record would invite the reader to add the +// seven numbers together, which is meaningless: corpus counts lines and shards +// that never became a window, review counts headings in a different data +// source entirely, and only two of the windowing buckets take part in the +// window identity. Every bucket is reported regardless (silence is not an +// option), and adding a bucket breaks the renderer at the type boundary. +export interface ExclusionCounts { + readonly corpus: { readonly brokenLine: number; readonly unreadableShard: number }; + readonly windowing: { + readonly unmatchedStart: number; + readonly orphanComplete: number; + readonly unclosedIdle: number; + readonly zeroSecond: number; + }; + readonly review: { readonly unparseableReviewHeading: number }; +} + +/** All buckets at zero — the identity element the folds start from. */ +export function emptyExclusions(): ExclusionCounts { + return { + corpus: { brokenLine: 0, unreadableShard: 0 }, + windowing: { unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0 }, + review: { unparseableReviewHeading: 0 }, + }; +} + +/** One row of the per-stage duration table. */ +export interface StageStat { + readonly stage: string; + readonly n: number; + readonly rawMedian: number; + readonly netMean: number; + readonly netMedian: number; + readonly netP95: number; +} + +// --- window construction --------------------------------------------------- + +/** The audit event name of a record, under either schema generation. */ +export function eventOf(record: JournalRecord): string | null { + return journalRecordField(record, "Event"); +} + +/** A record's payload attribute, under either schema generation. */ +export function attrOf(record: JournalRecord, key: string): string | null { + return journalRecordField(record, key); +} + +// Second-grained epoch. The audit writer strips milliseconds, so window +// arithmetic is integer seconds and sub-second work collapses to zero — the +// zero-second bucket exists to say so rather than to hide it. +function epochSeconds(timestamp: string): number { + return Math.floor(Date.parse(timestamp) / 1000); +} + +// Timestamp-ascending, with the original index as the tie-break so records +// stamped in the same second keep their shard order (START before COMPLETED). +function chronological(records: readonly AttributedRecord[]): AttributedRecord[] { + return records + .map((entry, index) => ({ entry, index })) + .sort((a, b) => a.entry.record.timestamp.localeCompare(b.entry.record.timestamp) || a.index - b.index) + .map((wrapped) => wrapped.entry); +} + +/** Pair STAGE_STARTED with the next STAGE_COMPLETED inside the same + * intent x stage. Unpaired events on either side are counted as events, not + * as windows — they never enter the window population. */ +export function buildWindows(records: readonly AttributedRecord[]): { windows: StageWindow[]; buckets: ExclusionCounts } { + const pending = new Map(); + const windows: StageWindow[] = []; + let unmatchedStart = 0; + let orphanComplete = 0; + + for (const attributed of chronological(records)) { + const event = eventOf(attributed.record); + if (event !== "STAGE_STARTED" && event !== "STAGE_COMPLETED") continue; + const stage = attrOf(attributed.record, "Stage"); + if (stage === null || stage === "") continue; + const key = `${attributed.intent}\u0000${stage}`; + if (event === "STAGE_STARTED") { + const queue = pending.get(key); + if (queue === undefined) pending.set(key, [attributed]); + else queue.push(attributed); + continue; + } + const queue = pending.get(key); + const started = queue?.shift(); + if (started === undefined) { + orphanComplete += 1; + continue; + } + windows.push({ + intent: attributed.intent, + stage, + startedAt: started.record.timestamp, + completedAt: attributed.record.timestamp, + rawSeconds: epochSeconds(attributed.record.timestamp) - epochSeconds(started.record.timestamp), + }); + } + for (const queue of pending.values()) unmatchedStart += queue.length; + + const zero = emptyExclusions(); + return { windows, buckets: { ...zero, windowing: { ...zero.windowing, unmatchedStart, orphanComplete } } }; +} + +// --- idle subtraction ------------------------------------------------------ + +/** One idle span, already closed by its terminating event. */ +export interface IdleInterval { + readonly kind: "awaiting" | "parked" | "session-gap"; + readonly start: number; + readonly end: number; +} + +/** Per-intent idle spans plus the timestamps of openers that never closed. */ +export interface IdleIndex { + readonly intervals: ReadonlyMap; + readonly unclosedOpeners: ReadonlyMap; +} + +// The three idle shapes, as opener -> accepted closers. An opener that never +// meets its closer inside the intent is NOT given an implied terminal: guessing +// one would silently invent working time, so the affected window leaves the +// population instead. +const IDLE_SHAPES: readonly { kind: IdleInterval["kind"]; open: string; close: readonly string[] }[] = [ + { kind: "awaiting", open: "STAGE_AWAITING_APPROVAL", close: ["GATE_APPROVED", "GATE_REJECTED"] }, + { kind: "parked", open: "WORKFLOW_PARKED", close: ["WORKFLOW_UNPARKED"] }, + { kind: "session-gap", open: "SESSION_ENDED", close: ["SESSION_STARTED", "SESSION_RESUMED"] }, +]; + +// The open-opener stacks, keyed by intent and idle kind. Kept as its own type +// so the fold below reads as three small steps instead of one nested sweep. +type OpenerStacks = Map>; + +/** The stack of unmatched openers for one intent x kind, created on demand. */ +function openerStack(stacks: OpenerStacks, intent: string, kind: IdleInterval["kind"]): number[] { + const perKind = stacks.get(intent) ?? new Map(); + stacks.set(intent, perKind); + const stack = perKind.get(kind) ?? []; + perKind.set(kind, stack); + return stack; +} + +/** Append one closed span to an intent's interval list. */ +function pushInterval(intervals: Map, intent: string, span: IdleInterval): void { + const bucket = intervals.get(intent) ?? []; + intervals.set(intent, bucket); + bucket.push(span); +} + +/** The idle shape this event participates in, or null when it is not idle. */ +function idleShapeFor(event: string): (typeof IDLE_SHAPES)[number] | null { + return IDLE_SHAPES.find((shape) => event === shape.open || shape.close.includes(event)) ?? null; +} + +/** Build the per-intent idle index from the corpus. Pure over the records. */ +export function indexIdle(records: readonly AttributedRecord[]): IdleIndex { + const intervals = new Map(); + const unclosedOpeners = new Map(); + const openPerIntent: OpenerStacks = new Map(); + + for (const attributed of chronological(records)) { + const event = eventOf(attributed.record); + const shape = event === null ? null : idleShapeFor(event); + if (shape === null) continue; + const at = epochSeconds(attributed.record.timestamp); + const stack = openerStack(openPerIntent, attributed.intent, shape.kind); + if (event === shape.open) { + stack.push(at); + continue; + } + const start = stack.shift(); + if (start !== undefined) pushInterval(intervals, attributed.intent, { kind: shape.kind, start, end: at }); + } + + for (const [intent, perKind] of openPerIntent) { + const leftovers: number[] = []; + for (const stack of perKind.values()) leftovers.push(...stack); + if (leftovers.length > 0) unclosedOpeners.set(intent, leftovers.sort((a, b) => a - b)); + } + return { intervals, unclosedOpeners }; +} + +// Clip each span to the window, then merge the survivors so overlapping spans +// are subtracted once. Clipping is what keeps net non-negative: the merged +// length can never exceed the window it was clipped into. +function clippedIdleSeconds(window: StageWindow, spans: readonly IdleInterval[]): number { + const from = epochSeconds(window.startedAt); + const to = epochSeconds(window.completedAt); + const clipped = spans + .map((span) => ({ start: Math.max(span.start, from), end: Math.min(span.end, to) })) + .filter((span) => span.end > span.start) + .sort((a, b) => a.start - b.start); + + let total = 0; + let cursor = from; + for (const span of clipped) { + const start = Math.max(span.start, cursor); + if (span.end > start) { + total += span.end - start; + cursor = span.end; + } + } + return total; +} + +/** Subtract idle from every window. A window is dropped from the population + * when an unclosed idle opener falls inside it (its idle is unknowable) or + * when it collapsed to zero seconds. The two are decided in that order, so a + * window that satisfies both is counted exactly once. */ +export function subtractIdle( + windows: readonly StageWindow[], + records: readonly AttributedRecord[], +): { measured: MeasuredWindow[]; buckets: ExclusionCounts } { + const index = indexIdle(records); + const measured: MeasuredWindow[] = []; + let unclosedIdle = 0; + let zeroSecond = 0; + + for (const window of windows) { + const from = epochSeconds(window.startedAt); + const to = epochSeconds(window.completedAt); + const openers = index.unclosedOpeners.get(window.intent) ?? []; + const shadowed = openers.some((at) => at >= from && at <= to); + if (shadowed) { + unclosedIdle += 1; + continue; + } + if (window.rawSeconds === 0) { + zeroSecond += 1; + continue; + } + const idleSeconds = clippedIdleSeconds(window, index.intervals.get(window.intent) ?? []); + measured.push({ ...window, idleSeconds, netSeconds: window.rawSeconds - idleSeconds }); + } + + const zero = emptyExclusions(); + return { measured, buckets: { ...zero, windowing: { ...zero.windowing, unclosedIdle, zeroSecond } } }; +} + +// --- review iterations ----------------------------------------------------- + +/** One §12a review block located in a record artefact. */ +export interface ReviewBlock { + readonly intent: string; + readonly stagePath: string; + readonly unit: string | null; + readonly iteration: number; +} + +// The reviewer runtime finds its own blocks with a permissive H2 scan followed +// by an exact marker comparison. The same two stages are mirrored here so a +// heading the writer would not recognize is surfaced as unparseable rather +// than quietly matched by a looser reader. +const REVIEW_HEADING = /^## Review(?:[ \t].*)?$/gm; +const REVIEW_MARKER = /^## Review — Iteration (\d+)$/; + +/** Iterations found in one artefact's text, plus headings the writer's exact + * marker would not accept (suffixed variants are real and stay visible). */ +export function parseReviewHeadings(content: string): { iterations: number[]; unparseable: number } { + const iterations: number[] = []; + let unparseable = 0; + for (const heading of content.matchAll(REVIEW_HEADING)) { + const marker = REVIEW_MARKER.exec(heading[0].trim()); + if (marker === null) { + unparseable += 1; + continue; + } + iterations.push(Number(marker[1])); + } + return { iterations, unparseable }; +} + +/** Stage path and unit for an artefact, taken from its intent-relative path. + * A literal `{unit-name}` directory is a real thing on disk; it is displayed + * verbatim rather than repaired into an invented unit name. */ +export function reviewAttribution(relativePath: string): { stagePath: string; unit: string | null } { + const segments = relativePath.split("/"); + const stagePath = segments.slice(0, -1).join("/"); + const unit = segments[0] === "construction" && segments.length >= 4 ? (segments[1] ?? null) : null; + return { stagePath, unit }; +} + +// --- sensors --------------------------------------------------------------- + +/** Sensor outcomes for one stage slug. */ +export interface SensorTally { + readonly stageSlug: string; + readonly fired: number; + readonly passed: number; + readonly failed: number; + readonly failedRate: number; +} + +const SENSOR_EVENTS: Record = { + SENSOR_FIRED: "fired", + SENSOR_PASSED: "passed", + SENSOR_FAILED: "failed", +}; + +/** Tally sensor outcomes by the `Stage slug` attribute. The lifecycle events' + * `Stage` attribute is a different key with different values and is never read + * here — mixing them would merge unrelated rows. */ +export function tallySensors(records: readonly AttributedRecord[]): SensorTally[] { + const counters = new Map(); + for (const { record } of records) { + const event = eventOf(record); + const outcome = event === null ? undefined : SENSOR_EVENTS[event]; + if (outcome === undefined) continue; + const slug = attrOf(record, "Stage slug"); + if (slug === null || slug === "") continue; + const counter = counters.get(slug) ?? { fired: 0, passed: 0, failed: 0 }; + counters.set(slug, counter); + counter[outcome] += 1; + } + return [...counters.entries()] + .map(([stageSlug, c]) => ({ stageSlug, ...c, failedRate: c.fired === 0 ? Number.NaN : c.failed / c.fired })) + .sort((a, b) => b.fired - a.fired || a.stageSlug.localeCompare(b.stageSlug)); +} + +// --- model attribution ----------------------------------------------------- + +/** Model breakdown of the subagent rows, with the unattributable ones kept. */ +export interface ModelAttribution { + readonly byModel: ReadonlyMap; + readonly byModelSource: ReadonlyMap; + readonly unresolvedCount: number; + readonly attributableCount: number; + readonly totalCount: number; +} + +/** Attribute subagent rows to models. A row without a usable `Model` is the + * UNKNOWN bucket: the absence of the attribute is itself the record, so it is + * reported rather than dropped from the denominator. */ +export function attributeModels(records: readonly AttributedRecord[]): ModelAttribution { + const byModel = new Map(); + const byModelSource = new Map(); + let unresolvedCount = 0; + let attributableCount = 0; + let totalCount = 0; + + for (const { record } of records) { + const event = eventOf(record); + if (event !== "SUBAGENT_COMPLETED" && event !== "SUBAGENT_STARTED") continue; + totalCount += 1; + const model = attrOf(record, "Model"); + if (model === null || model.trim() === "") { + unresolvedCount += 1; + continue; + } + attributableCount += 1; + byModel.set(model, (byModel.get(model) ?? 0) + 1); + const source = attrOf(record, "Model Source"); + if (source !== null && source.trim() !== "") byModelSource.set(source, (byModelSource.get(source) ?? 0) + 1); + } + return { byModel, byModelSource, unresolvedCount, attributableCount, totalCount }; +} + +// --- statistics ------------------------------------------------------------ + +// Nearest rank, no interpolation: the smallest sample whose rank covers 95% of +// the set. An empty set yields NaN rather than 0 so a hole in the measurement +// stays visibly non-finite instead of reading as a real zero. +export function nearestRankP95(values: readonly number[]): number { + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.ceil(sorted.length * 0.95) - 1] ?? Number.NaN; +} + +/** Arithmetic mean; NaN on an empty population (no 0 fallback). */ +export function mean(values: readonly number[]): number { + if (values.length === 0) return Number.NaN; + return values.reduce((a, b) => a + b, 0) / values.length; +} + +/** Median (mean of the two middle samples on an even count); NaN when empty. */ +export function median(values: readonly number[]): number { + if (values.length === 0) return Number.NaN; + const sorted = [...values].sort((left, right) => left - right); + const mid = sorted.length >> 1; + return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +/** Fold measured windows into the per-stage table. Ordering: count-desc, then + * stage-asc, so the same population always renders in the same order. */ +export function composeStageStats(measured: readonly MeasuredWindow[]): StageStat[] { + const byStage = new Map(); + for (const window of measured) { + const bucket = byStage.get(window.stage); + if (bucket === undefined) byStage.set(window.stage, [window]); + else bucket.push(window); + } + const rows: StageStat[] = []; + for (const [stage, windows] of byStage) { + const net = windows.map((w) => w.netSeconds); + rows.push({ + stage, + n: windows.length, + rawMedian: median(windows.map((w) => w.rawSeconds)), + netMean: mean(net), + netMedian: median(net), + netP95: nearestRankP95(net), + }); + } + return rows.sort((a, b) => b.n - a.n || a.stage.localeCompare(b.stage)); +} + +// --- report assembly ------------------------------------------------------- + +// The one sentence this tool must never ship without. net is derived by +// subtracting idle spans; nobody has verified that the remainder equals the +// time someone actually spent working, so the report says so where the reader +// cannot miss it. +export const HYPOTHESIS_NOTICE = + "net (working time) is an estimate produced by subtracting idle spans; its agreement with real working time is an unverified hypothesis."; + +/** Review blocks folded per stage path. */ +export interface ReviewBucket { + readonly stagePath: string; + readonly unit: string | null; + readonly blocks: number; + readonly maxIteration: number; +} + +/** Everything the renderers are allowed to draw from. The measurement ref and + * the hypothesis notice are required fields, so an output that omits them is + * not expressible. */ +export interface StageStatsReport { + readonly scanScope: string; + readonly hypothesisNotice: string; + readonly shardCount: number; + readonly lineCount: number; + readonly exclusions: ExclusionCounts; + readonly constructedWindowCount: number; + readonly populationCount: number; + readonly stages: readonly StageStat[]; + readonly sensors: readonly SensorTally[]; + readonly models: ModelAttribution; + readonly reviewBuckets: readonly ReviewBucket[]; +} + +/** Fold review blocks into per-stage-path buckets, count-desc then path-asc. */ +export function bucketReviews(blocks: readonly ReviewBlock[]): ReviewBucket[] { + const buckets = new Map(); + for (const block of blocks) { + const bucket = buckets.get(block.stagePath) ?? { unit: block.unit, blocks: 0, maxIteration: 0 }; + buckets.set(block.stagePath, { + unit: bucket.unit, + blocks: bucket.blocks + 1, + maxIteration: Math.max(bucket.maxIteration, block.iteration), + }); + } + return [...buckets.entries()] + .map(([stagePath, b]) => ({ stagePath, ...b })) + .sort((a, b) => b.blocks - a.blocks || a.stagePath.localeCompare(b.stagePath)); +} + +/** Assemble the whole report from a scanned corpus and the review blocks. + * Pure over its input: the same corpus always produces the same report. */ +export function composeReport(input: { + scanScope: string; + corpus: ScannedCorpus; + reviewBlocks: readonly ReviewBlock[]; + unparseableReviewHeadingCount: number; +}): StageStatsReport { + const built = buildWindows(input.corpus.records); + const subtracted = subtractIdle(built.windows, input.corpus.records); + return { + scanScope: input.scanScope, + hypothesisNotice: HYPOTHESIS_NOTICE, + shardCount: input.corpus.shardCount, + lineCount: input.corpus.lineCount, + exclusions: { + corpus: { brokenLine: input.corpus.brokenLineCount, unreadableShard: input.corpus.unreadableShardCount }, + windowing: { + unmatchedStart: built.buckets.windowing.unmatchedStart, + orphanComplete: built.buckets.windowing.orphanComplete, + unclosedIdle: subtracted.buckets.windowing.unclosedIdle, + zeroSecond: subtracted.buckets.windowing.zeroSecond, + }, + review: { unparseableReviewHeading: input.unparseableReviewHeadingCount }, + }, + constructedWindowCount: built.windows.length, + populationCount: subtracted.measured.length, + stages: composeStageStats(subtracted.measured), + sensors: tallySensors(input.corpus.records), + models: attributeModels(input.corpus.records), + reviewBuckets: bucketReviews(input.reviewBlocks), + }; +} + +// --- rendering ------------------------------------------------------------- + +// Values from the corpus reach a terminal here. The corpus is outside the +// trust boundary, so a stage slug or model name is reduced to its first line +// and stripped of control bytes at the render point only — the aggregation +// keys stay verbatim, and the JSON form keeps the raw value because its +// encoding already escapes control bytes and machine consumers need it whole. +const FIRST_PRINTABLE = 0x20; +const DELETE_CODE = 0x7f; + +function safe(value: string): string { + const firstLine = value.split("\n", 1)[0] ?? ""; + let out = ""; + for (const char of firstLine) { + const code = char.codePointAt(0) ?? 0; + if (code >= FIRST_PRINTABLE && code !== DELETE_CODE) out += char; + } + return out; +} + +// A number the reader must be able to tell apart from a real zero. +function num(value: number): string { + return Number.isNaN(value) ? "n/a" : `${Math.round(value * 100) / 100}`; +} + +/** The measurement ref lines, shared by every output form: what was scanned, + * and every exclusion bucket, whatever its value. */ +function measurementRefLines(report: StageStatsReport): string[] { + const { corpus, windowing, review } = report.exclusions; + return [ + `scan scope: ${safe(report.scanScope)}`, + `shards: ${report.shardCount}`, + `lines: ${report.lineCount}`, + `broken-line: ${corpus.brokenLine}`, + `unreadable-shard: ${corpus.unreadableShard}`, + `unmatched-start: ${windowing.unmatchedStart}`, + `orphan-complete: ${windowing.orphanComplete}`, + `unclosed-idle: ${windowing.unclosedIdle}`, + `zero-second: ${windowing.zeroSecond}`, + `unparseable-review-heading: ${review.unparseableReviewHeading}`, + `constructed-windows: ${report.constructedWindowCount}`, + `net-population: ${report.populationCount}`, + ]; +} + +/** Sorted entries of a tally map: count-desc, then key-asc. */ +function sortedEntries(tally: ReadonlyMap): [string, number][] { + return [...tally.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); +} + +/** The human-readable form. */ +export function renderMarkdown(report: StageStatsReport): string { + const lines: string[] = ["# stage-stats — measurement ref", ""]; + for (const line of measurementRefLines(report)) lines.push(`- ${line}`); + lines.push( + "", + `> ${HYPOTHESIS_NOTICE}`, + "", + `Identity W: ${report.constructedWindowCount} constructed = ${report.populationCount} population + ${report.exclusions.windowing.unclosedIdle} unclosed-idle + ${report.exclusions.windowing.zeroSecond} zero-second`, + `Identity M: ${report.models.totalCount} total = ${report.models.attributableCount} attributable + ${report.models.unresolvedCount} unresolved`, + "", + "## Stage durations (seconds)", + "", + "| stage | n | raw median | net mean | net median | net p95 |", + "| --- | --- | --- | --- | --- | --- |", + ); + if (report.stages.length === 0) lines.push("| (none) | 0 | n/a | n/a | n/a | n/a |"); + for (const stat of report.stages) { + lines.push(`| ${safe(stat.stage)} | ${stat.n} | ${num(stat.rawMedian)} | ${num(stat.netMean)} | ${num(stat.netMedian)} | ${num(stat.netP95)} |`); + } + + lines.push("", "## Sensor outcomes", "", "| stage slug | fired | passed | failed | failed rate |", "| --- | --- | --- | --- | --- |"); + if (report.sensors.length === 0) lines.push("| (none) | 0 | 0 | 0 | n/a |"); + for (const row of report.sensors) { + lines.push(`| ${safe(row.stageSlug)} | ${row.fired} | ${row.passed} | ${row.failed} | ${num(row.failedRate)} |`); + } + + lines.push("", "## Model attribution", "", `- attributable: ${report.models.attributableCount} / ${report.models.totalCount}`, `- unresolved (UNKNOWN): ${report.models.unresolvedCount}`); + for (const [model, count] of sortedEntries(report.models.byModel)) lines.push(`- model ${safe(model)}: ${count}`); + for (const [source, count] of sortedEntries(report.models.byModelSource)) lines.push(`- model-source ${safe(source)}: ${count}`); + + lines.push("", "## Review iterations", "", "| stage path | unit | blocks | max iteration |", "| --- | --- | --- | --- |"); + if (report.reviewBuckets.length === 0) lines.push("| (none) | | 0 | 0 |"); + for (const bucket of report.reviewBuckets) { + lines.push(`| ${safe(bucket.stagePath)} | ${safe(bucket.unit ?? "")} | ${bucket.blocks} | ${bucket.maxIteration} |`); + } + return `${lines.join("\n")}\n`; +} + +// CSV quoting: double the quotes and wrap, so a value containing a comma or a +// quote cannot forge an extra column. +function csvCell(value: string): string { + return `"${safe(value).replace(/"/g, '""')}"`; +} + +/** The spreadsheet form: the same measurement ref, then one section per axis. */ +export function renderCsv(report: StageStatsReport): string { + const lines: string[] = ["section,key,value"]; + lines.push(`meta,${csvCell("stage-stats")},${csvCell(report.scanScope)}`); + lines.push(`meta,${csvCell("hypothesis")},${csvCell(HYPOTHESIS_NOTICE)}`); + for (const line of measurementRefLines(report)) { + const split = line.indexOf(":"); + lines.push(`ref,${line.slice(0, split)},${csvCell(line.slice(split + 2))}`); + } + lines.push("", "stage,n,raw_median,net_mean,net_median,net_p95"); + for (const stat of report.stages) { + lines.push(`${csvCell(stat.stage)},${stat.n},${num(stat.rawMedian)},${num(stat.netMean)},${num(stat.netMedian)},${num(stat.netP95)}`); + } + lines.push("", "sensor_stage_slug,fired,passed,failed,failed_rate"); + for (const row of report.sensors) lines.push(`${csvCell(row.stageSlug)},${row.fired},${row.passed},${row.failed},${num(row.failedRate)}`); + lines.push("", "model,count"); + for (const [model, count] of sortedEntries(report.models.byModel)) lines.push(`${csvCell(model)},${count}`); + lines.push(`${csvCell("(unresolved)")},${report.models.unresolvedCount}`); + lines.push("", "review_stage_path,unit,blocks,max_iteration"); + for (const bucket of report.reviewBuckets) { + lines.push(`${csvCell(bucket.stagePath)},${csvCell(bucket.unit ?? "")},${bucket.blocks},${bucket.maxIteration}`); + } + return `${lines.join("\n")}\n`; +} + +/** The machine form. Maps become arrays in the same fixed order the text forms + * use, so a consumer sees one ordering contract, not two. */ +export function serializeJson(report: StageStatsReport): Record { + return { + scanScope: report.scanScope, + hypothesisNotice: report.hypothesisNotice, + shardCount: report.shardCount, + lineCount: report.lineCount, + exclusions: report.exclusions, + constructedWindowCount: report.constructedWindowCount, + populationCount: report.populationCount, + stages: report.stages, + sensors: report.sensors, + models: { + byModel: sortedEntries(report.models.byModel).map(([model, count]) => ({ model, count })), + byModelSource: sortedEntries(report.models.byModelSource).map(([source, count]) => ({ source, count })), + unresolvedCount: report.models.unresolvedCount, + attributableCount: report.models.attributableCount, + totalCount: report.models.totalCount, + }, + reviewBuckets: report.reviewBuckets, + }; +} + +// --- argv ------------------------------------------------------------------ + +/** Command line options, already validated into their narrow shapes. */ +export interface CliOptions { + readonly projectDir?: string; + readonly space?: string; + readonly format: "markdown" | "csv" | "json"; +} + +/** The one thing a usage error carries. */ +export interface UsageError { + readonly message: string; +} + +/** A parse outcome; the shell is the only place it becomes an exit code. */ +export type Result = { readonly ok: true; readonly value: T } | { readonly ok: false; readonly error: E }; + +const USAGE = + "Usage: bun amadeus-stage-stats.ts [--project-dir ] [--space ] [--format markdown|csv|json] [--json]"; + +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const FORMATS: readonly CliOptions["format"][] = ["markdown", "csv", "json"]; + +function isFormat(value: string): value is CliOptions["format"] { + return (FORMATS as readonly string[]).includes(value); +} + +const VALUE_FLAGS = ["--project-dir", "--space", "--format"] as const; +type ValueFlag = (typeof VALUE_FLAGS)[number]; + +function isValueFlag(arg: string): arg is ValueFlag { + return (VALUE_FLAGS as readonly string[]).includes(arg); +} + +function usageError(message: string): { ok: false; error: UsageError } { + return { ok: false, error: { message: `${message}\n${USAGE}` } }; +} + +/** Apply one already-paired flag/value to the accumulating options, or refuse + * the value. Keeping the per-flag rules here leaves the loop a plain walk. */ +function applyValueFlag(flag: ValueFlag, value: string, into: { projectDir?: string; space?: string; format: CliOptions["format"] }): UsageError | null { + if (flag === "--project-dir") { + into.projectDir = value; + return null; + } + if (flag === "--space") { + if (!SAFE_NAME.test(value)) return { message: `Invalid --space value: ${value}\n${USAGE}` }; + into.space = value; + return null; + } + if (!isFormat(value)) return { message: `Invalid --format value: ${value}\n${USAGE}` }; + into.format = value; + return null; +} + +/** Parse, do not validate: an unknown flag, a missing value, an unsupported + * format, or an unsafe space name never reaches the rest of the program. */ +export function parseArgs(argv: readonly string[]): Result { + const options: { projectDir?: string; space?: string; format: CliOptions["format"] } = { format: "markdown" }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] as string; + if (arg === "--json") { + options.format = "json"; + continue; + } + if (!isValueFlag(arg)) return usageError(`Unknown argument: ${arg}`); + const value = argv[i + 1]; + if (value === undefined) return usageError(`${arg} requires a value`); + i += 1; + const rejected = applyValueFlag(arg, value, options); + if (rejected !== null) return { ok: false, error: rejected }; + } + return { ok: true, value: options }; +} + +// --------------------------------------------------------------------------- +// Filesystem layer + CLI shell — the only process-boundary code in this module. +// --------------------------------------------------------------------------- + +/** Directory entries, or none when the directory is absent or unreadable. An + * absent tree and a tree that vanishes mid-scan are the same thing here: an + * empty corpus. The fail-loud case is a shard that EXISTS but cannot be read. */ +function listOrEmpty(dir: string): string[] { + try { + return readdirSync(dir).sort(); + } catch { + return []; + } +} + +/** One shard's bytes, or null when it exists but cannot be read. */ +function readOrNull(path: string): string | null { + try { + return readFileSync(path, "utf-8"); + } catch (e) { + const reason = e instanceof Error ? e.message : String(e); + const detail = safe(`${path}: ${reason}`); + process.stderr.write(`stage-stats: unreadable: ${detail}\n`); + return null; + } +} + +// Records in one shard, counting lines that do not decode. The journal reader +// is strict per buffer, so it is fed one line at a time: a single torn row then +// costs that row and nothing else, and the count of torn rows is reported. +function recordsFromShard(intent: string, body: string): { records: AttributedRecord[]; lines: number; broken: number } { + const records: AttributedRecord[] = []; + let lines = 0; + let broken = 0; + for (const line of body.split("\n")) { + if (line.trim() === "") continue; + lines += 1; + try { + for (const record of readJournalRecords(line)) records.push({ intent, record }); + } catch { + broken += 1; + continue; + } + } + return { records, lines, broken }; +} + +/** Read every audit shard under the space's intents tree. The intent an event + * belongs to comes from the shard's PATH, never from the recorded intentId — + * most historical v1 rows carry a degenerate value there. */ +export function scanCorpus(spaceRoot: string): ScannedCorpus { + const records: AttributedRecord[] = []; + let unreadableShardCount = 0; + let brokenLineCount = 0; + let shardCount = 0; + let lineCount = 0; + + const intentsRoot = join(spaceRoot, "intents"); + for (const intent of listOrEmpty(intentsRoot)) { + const auditDir = join(intentsRoot, intent, "audit"); + for (const shard of listOrEmpty(auditDir)) { + if (!shard.endsWith(".jsonl")) continue; + const body = readOrNull(join(auditDir, shard)); + if (body === null) { + unreadableShardCount += 1; + continue; + } + shardCount += 1; + const read = recordsFromShard(intent, body); + records.push(...read.records); + lineCount += read.lines; + brokenLineCount += read.broken; + } + } + return { records, unreadableShardCount, brokenLineCount, shardCount, lineCount }; +} + +/** Markdown artefacts under one intent, as intent-relative paths. */ +function markdownUnder(root: string, prefix: string, out: string[]): void { + for (const entry of listOrEmpty(root)) { + if (entry === "audit" || entry === "node_modules") continue; + const full = join(root, entry); + const relative = prefix === "" ? entry : `${prefix}/${entry}`; + if (entry.endsWith(".md")) out.push(relative); + else markdownUnder(full, relative, out); + } +} + +/** Collect every §12a review block from the record artefacts. This is the + * record tree, not the audit tree: iteration counts live in the artefacts the + * reviewer appended to, and nowhere else. */ +export function collectReviewBlocks(intentsRoot: string): { blocks: ReviewBlock[]; unparseableHeadingCount: number } { + const blocks: ReviewBlock[] = []; + let unparseableHeadingCount = 0; + for (const intent of listOrEmpty(intentsRoot)) { + const paths: string[] = []; + markdownUnder(join(intentsRoot, intent), "", paths); + for (const relative of paths) { + const body = readOrNull(join(intentsRoot, intent, relative)); + if (body === null) continue; + const parsed = parseReviewHeadings(body); + unparseableHeadingCount += parsed.unparseable; + const attribution = reviewAttribution(relative); + for (const iteration of parsed.iterations) { + blocks.push({ intent, stagePath: attribution.stagePath, unit: attribution.unit, iteration }); + } + } + } + return { blocks, unparseableHeadingCount }; +} + +// The two path idioms below mirror the sibling stats tool, reimplemented small +// because this module must not import amadeus-lib.ts. + +function resolveProjectDirLocal(explicitDir: string | undefined): string { + if (explicitDir) return explicitDir; + if (process.env.CLAUDE_PROJECT_DIR) return process.env.CLAUDE_PROJECT_DIR; + const scriptDir = dirname(fileURLToPath(import.meta.url)); + if (basename(scriptDir) === "tools") { + const leaf = basename(dirname(scriptDir)); + if (/^\.[a-z0-9][a-z0-9._-]*$/i.test(leaf)) return dirname(dirname(scriptDir)); + } + return process.cwd(); +} + +function activeSpaceLocal(projectDir: string): string { + try { + const raw = readFileSync(join(projectDir, "amadeus", "active-space"), "utf-8").trim(); + if (SAFE_NAME.test(raw)) return raw; + } catch { + return "default"; + } + return "default"; +} + +const RENDERERS: Record string> = { + markdown: renderMarkdown, + csv: renderCsv, + json: (report) => JSON.stringify(serializeJson(report), null, 2), +}; + +/** 0 normal, 1 the corpus had a hole, 2 the caller made a usage error. */ +export function main(argv: readonly string[]): number { + const parsed = parseArgs(argv); + if (!parsed.ok) { + process.stderr.write(`${parsed.error.message}\n`); + return 2; + } + const projectDir = resolveProjectDirLocal(parsed.value.projectDir); + const space = parsed.value.space ?? activeSpaceLocal(projectDir); + const spaceRoot = join(projectDir, "amadeus", "spaces", space); + + const corpus = scanCorpus(spaceRoot); + const reviews = collectReviewBlocks(join(spaceRoot, "intents")); + const report = composeReport({ + scanScope: `space "${space}" — amadeus/spaces/${space}/intents/*/audit/*.jsonl + record *.md`, + corpus, + reviewBlocks: reviews.blocks, + unparseableReviewHeadingCount: reviews.unparseableHeadingCount, + }); + process.stdout.write(`${RENDERERS[parsed.value.format](report)}`); + + // A shard that existed but could not be read means the report covers less + // than the observable corpus — say so in the exit code so a caller does not + // mistake a partial sweep for a complete one. + return corpus.unreadableShardCount > 0 ? 1 : 0; +} + +if (import.meta.main) process.exit(main(process.argv.slice(2))); diff --git a/tests/.coverage-registry.json b/tests/.coverage-registry.json index d10eb30ba..08ca276a3 100644 --- a/tests/.coverage-registry.json +++ b/tests/.coverage-registry.json @@ -2762,6 +2762,10 @@ "file": "tests/integration/t420-unchecked-cast-guard-cli.test.ts", "mechanism": "none" }, + { + "file": "tests/integration/t485-stage-stats.integration.test.ts", + "mechanism": "cli" + }, { "file": "tests/unit/t-graph-dispatch-seam.test.ts", "mechanism": "cli" diff --git a/tests/integration/t485-stage-stats.integration.test.ts b/tests/integration/t485-stage-stats.integration.test.ts new file mode 100644 index 000000000..046145d10 --- /dev/null +++ b/tests/integration/t485-stage-stats.integration.test.ts @@ -0,0 +1,406 @@ +// covers: function:scanCorpus, function:collectReviewBlocks, function:main +// size: medium +// +// t485 — Issue #2405 (U1 stage-stats). Integration half of the twin: the real +// filesystem scan, the CLI shell driven in-process (so bun --coverage measures +// it), and the exit ladder measured by an actual spawn. The pure aggregation +// core is t484's job. +// +// Every expectation is checked against an INDEPENDENT ORACLE — a walker +// written here in the test that counts shards, lines and events on its own — +// so a defect in the scanner cannot cancel out against the expectation. +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { collectReviewBlocks, composeReport, main, scanCorpus } from "../../packages/framework/core/tools/amadeus-stage-stats.ts"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const TOOL = join(REPO_ROOT, "packages", "framework", "core", "tools", "amadeus-stage-stats.ts"); + +const scratchDirs: string[] = []; +afterEach(() => { + while (scratchDirs.length > 0) rmSync(scratchDirs.pop()!, { recursive: true, force: true }); +}); + +function scratch(): string { + const dir = mkdtempSync(join(tmpdir(), "amadeus-stage-stats-")); + scratchDirs.push(dir); + return dir; +} + +// --- fixture builders ------------------------------------------------------- + +function v1Line(seq: number, event: string, timestamp: string, fields: Record = {}): string { + return JSON.stringify({ schemaVersion: 1, seq, cloneId: "c1", intentId: "intents", timestamp, heading: event, event, fields }); +} + +function v2Line(seq: number, event: string, timestamp: string, attrs: Record = {}): string { + return JSON.stringify({ + schemaVersion: 2, + eventId: `e${seq}`, + seq, + timestamp, + eventName: "amadeus.stage", + attributes: { Event: event, ...attrs }, + intentId: "intents", + space: "default", + cloneId: "c2", + traceId: null, + spanId: null, + traceFlags: 0, + idempotencyKey: `k${seq}`, + canonical: true, + }); +} + +function writeShard(spaceRoot: string, intent: string, name: string, lines: readonly string[]): string { + const dir = join(spaceRoot, "intents", intent, "audit"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, `${lines.join("\n")}\n`); + return path; +} + +function writeArtefact(spaceRoot: string, intent: string, relative: string, body: string): void { + const path = join(spaceRoot, "intents", intent, relative); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, body); +} + +// A corpus mixing both journal generations, a torn line, a degenerate +// intentId, an unmatched start, a zero-second window, an unclosed park, and +// review headings both canonical and suffixed. +function buildCorpus(projectDir: string): string { + const spaceRoot = join(projectDir, "amadeus", "spaces", "default"); + writeShard(spaceRoot, "alpha", "shard-a.jsonl", [ + v1Line(1, "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1Line(2, "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1Line(3, "GATE_APPROVED", "2026-01-01T00:00:50Z"), + v2Line(4, "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + "{ this line is not json", + v2Line(5, "SENSOR_FIRED", "2026-01-01T00:00:05Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2Line(6, "SENSOR_FAILED", "2026-01-01T00:00:06Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2Line(7, "SUBAGENT_COMPLETED", "2026-01-01T00:00:07Z", { Model: "opus", "Model Source": "pin" }), + v1Line(8, "SUBAGENT_COMPLETED", "2026-01-01T00:00:08Z"), + ]); + writeShard(spaceRoot, "beta", "shard-b.jsonl", [ + v1Line(1, "STAGE_STARTED", "2026-01-01T01:00:00Z", { Stage: "build" }), + v1Line(2, "STAGE_COMPLETED", "2026-01-01T01:00:00Z", { Stage: "build" }), + v1Line(3, "STAGE_STARTED", "2026-01-01T02:00:00Z", { Stage: "never-finished" }), + v1Line(4, "STAGE_STARTED", "2026-01-01T03:00:00Z", { Stage: "parked-forever" }), + v1Line(5, "WORKFLOW_PARKED", "2026-01-01T03:00:10Z"), + v1Line(6, "STAGE_COMPLETED", "2026-01-01T03:01:00Z", { Stage: "parked-forever" }), + ]); + writeArtefact(spaceRoot, "alpha", "construction/unit-x/functional-design/business-rules.md", "## Review — Iteration 1\nbody\n## Review — Iteration 2\n"); + writeArtefact(spaceRoot, "alpha", "construction/{unit-name}/code-generation/code-summary.md", "## Review — Iteration 1\n"); + writeArtefact(spaceRoot, "beta", "inception/requirements-analysis/requirements.md", "## Review — Iteration 1 (follow-up)\n"); + return spaceRoot; +} + +// --- independent oracles ---------------------------------------------------- + +// A walker written from scratch: it counts shards, non-empty lines, and torn +// lines without calling any function under test. +function oracleEventName(line: string): string { + const parsed = JSON.parse(line) as Record; + return parsed.schemaVersion === 2 ? String((parsed.attributes as Record).Event) : String(parsed.event); +} + +function oracleShardPaths(spaceRoot: string): string[] { + const intentsRoot = join(spaceRoot, "intents"); + const paths: string[] = []; + for (const intent of readdirSync(intentsRoot).sort()) { + const auditDir = join(intentsRoot, intent, "audit"); + let entries: string[] = []; + try { + entries = readdirSync(auditDir).sort(); + } catch { + entries = []; + } + for (const shard of entries) if (shard.endsWith(".jsonl")) paths.push(join(auditDir, shard)); + } + return paths; +} + +function oracleScan(spaceRoot: string): { shards: number; lines: number; broken: number; events: Record } { + let shards = 0; + let lines = 0; + let broken = 0; + const events: Record = {}; + for (const path of oracleShardPaths(spaceRoot)) { + let body: string; + try { + body = readFileSync(path, "utf-8"); + } catch { + continue; + } + shards += 1; + for (const line of body.split("\n")) { + if (line.trim() === "") continue; + lines += 1; + try { + const name = oracleEventName(line); + events[name] = (events[name] ?? 0) + 1; + } catch { + broken += 1; + } + } + } + return { shards, lines, broken, events }; +} + +// A second, independent heading counter: split on lines and compare the exact +// canonical marker, without touching the module's parser. +function oracleMarkdownPaths(dir: string, out: string[]): void { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === "audit") continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) oracleMarkdownPaths(full, out); + else if (entry.name.endsWith(".md")) out.push(full); + } +} + +function oracleReviewCount(spaceRoot: string): { canonical: number; suffixed: number } { + let canonical = 0; + let suffixed = 0; + const paths: string[] = []; + oracleMarkdownPaths(join(spaceRoot, "intents"), paths); + for (const path of paths) { + for (const line of readFileSync(path, "utf-8").split("\n")) { + // The permissive scan the writer uses: "## Review" followed by nothing, + // a space, or a tab. Anything else is a different heading entirely. + if (!/^## Review(?:[ \t].*)?$/.test(line)) continue; + if (/^## Review — Iteration \d+$/.test(line)) canonical += 1; + else suffixed += 1; + } + } + return { canonical, suffixed }; +} + +function capturedMain(argv: readonly string[]): { code: number; stdout: string } { + const chunks: string[] = []; + const original = process.stdout.write.bind(process.stdout); + (process.stdout as unknown as { write: (s: string) => boolean }).write = (s: string) => { + chunks.push(s); + return true; + }; + try { + return { code: main(argv), stdout: chunks.join("") }; + } finally { + (process.stdout as unknown as { write: typeof original }).write = original; + } +} + +// --- FR-1: corpus scan and attribution -------------------------------------- + +describe("scanCorpus — both journal generations, path-derived attribution", () => { + test("rows of both schema versions reach the aggregation, and torn lines are counted", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const oracle = oracleScan(spaceRoot); + const corpus = scanCorpus(spaceRoot); + + expect(corpus.shardCount).toBe(oracle.shards); + expect(corpus.lineCount).toBe(oracle.lines); + expect(corpus.brokenLineCount).toBe(oracle.broken); + expect(corpus.brokenLineCount).toBe(1); + // 15 written lines minus the one that does not decode. + expect(corpus.records).toHaveLength(oracle.lines - oracle.broken); + const versions = new Set(corpus.records.map((r) => r.record.schemaVersion)); + expect([...versions].sort()).toEqual([1, 2]); + }); + + test("a degenerate recorded intentId is overridden by the shard path", () => { + const projectDir = scratch(); + const corpus = scanCorpus(buildCorpus(projectDir)); + expect(corpus.records.every((r) => r.record.intentId === "intents")).toBe(true); + expect([...new Set(corpus.records.map((r) => r.intent))].sort()).toEqual(["alpha", "beta"]); + }); + + test("an unreadable shard is counted, and the rest of the corpus still scans", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const before = scanCorpus(spaceRoot); + // A dangling symlink is the portable way to make a directory entry that + // exists for readdir but cannot be read (a directory would raise EISDIR on + // macOS and return "" on Linux). + symlinkSync(join(spaceRoot, "intents", "alpha", "audit", "nowhere.jsonl"), join(spaceRoot, "intents", "alpha", "audit", "dangling.jsonl")); + const after = scanCorpus(spaceRoot); + expect(after.unreadableShardCount).toBe(1); + expect(after.records.length).toBe(before.records.length); + }); + + test("an absent space tree is an empty corpus, not a crash", () => { + const corpus = scanCorpus(join(scratch(), "no", "such", "space")); + expect(corpus).toEqual({ records: [], unreadableShardCount: 0, brokenLineCount: 0, shardCount: 0, lineCount: 0 }); + }); +}); + +// --- FR-3: review iterations from the record tree --------------------------- + +describe("collectReviewBlocks — record artefacts, literal unit dirs kept", () => { + test("canonical headings become blocks and suffixed headings are counted", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const oracle = oracleReviewCount(spaceRoot); + const collected = collectReviewBlocks(join(spaceRoot, "intents")); + + expect(collected.blocks).toHaveLength(oracle.canonical); + expect(collected.unparseableHeadingCount).toBe(oracle.suffixed); + expect(collected.unparseableHeadingCount).toBe(1); + expect(collected.blocks.map((b) => b.unit)).toContain("{unit-name}"); + expect(collected.blocks.map((b) => b.unit)).toContain("unit-x"); + }); +}); + +// --- FR-2 / FR-4 / FR-5 / FR-6 through the assembled report ----------------- + +describe("main — the whole pipeline over a real corpus", () => { + test("the report opens with the measurement ref and states the hypothesis", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { code, stdout } = capturedMain(["--project-dir", projectDir, "--space", "default"]); + expect(code).toBe(0); + const head = stdout.split("\n").slice(0, 16).join("\n"); + for (const key of ["shards:", "lines:", "broken-line:", "unreadable-shard:", "unmatched-start:", "orphan-complete:", "unclosed-idle:", "zero-second:", "unparseable-review-heading:"]) { + expect(head).toContain(key); + } + expect(stdout).toContain("unverified hypothesis"); + }); + + test("json carries the layered identities and the exclusion buckets", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { stdout } = capturedMain(["--project-dir", projectDir, "--space", "default", "--json"]); + const report = JSON.parse(stdout) as { + constructedWindowCount: number; + populationCount: number; + exclusions: { corpus: Record; windowing: Record; review: Record }; + models: { attributableCount: number; unresolvedCount: number; totalCount: number }; + stages: { stage: string; rawMedian: number; netMedian: number }[]; + sensors: { stageSlug: string; fired: number; failed: number }[]; + }; + // Identity W over the real corpus. + expect(report.populationCount + report.exclusions.windowing.unclosedIdle + report.exclusions.windowing.zeroSecond).toBe(report.constructedWindowCount); + // Identity M. + expect(report.models.attributableCount + report.models.unresolvedCount).toBe(report.models.totalCount); + expect(report.models.unresolvedCount).toBe(1); + // The fixture's shapes, each counted exactly once. + expect(report.exclusions.corpus.brokenLine).toBe(1); + expect(report.exclusions.windowing.unmatchedStart).toBe(1); + expect(report.exclusions.windowing.zeroSecond).toBe(1); + expect(report.exclusions.windowing.unclosedIdle).toBe(1); + expect(report.exclusions.review.unparseableReviewHeading).toBe(1); + // Idle subtraction: the design window is 100s raw, 30s of it approval wait. + const design = report.stages.find((s) => s.stage === "design"); + expect(design?.rawMedian).toBe(100); + expect(design?.netMedian).toBe(70); + // Sensors bucket on `Stage slug`, whose value differs from `Stage` here. + expect(report.sensors.map((s) => s.stageSlug)).toEqual(["design"]); + }); + + test("two runs over the same corpus produce byte-identical output in all three forms", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + for (const form of [[], ["--format", "csv"], ["--json"]]) { + const args = ["--project-dir", projectDir, "--space", "default", ...form]; + expect(capturedMain(args).stdout).toBe(capturedMain(args).stdout); + } + }); + + test("composeReport over the scanned corpus matches what main rendered", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const reviews = collectReviewBlocks(join(spaceRoot, "intents")); + const report = composeReport({ + scanScope: "x", + corpus: scanCorpus(spaceRoot), + reviewBlocks: reviews.blocks, + unparseableReviewHeadingCount: reviews.unparseableHeadingCount, + }); + const { stdout } = capturedMain(["--project-dir", projectDir, "--space", "default", "--json"]); + expect((JSON.parse(stdout) as { populationCount: number }).populationCount).toBe(report.populationCount); + }); + + test("scanning the real workspace stays well inside the sixty-second ceiling", () => { + const started = Date.now(); + const { code } = capturedMain(["--project-dir", REPO_ROOT, "--space", "default", "--json"]); + const elapsed = (Date.now() - started) / 1000; + expect(code === 0 || code === 1).toBe(true); + expect(elapsed).toBeLessThan(60); + }); +}); + +// --- FR-7: exit ladder and the read-only invariant -------------------------- + +describe("exit ladder — measured by spawning the CLI", () => { + test("a healthy corpus exits 0", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], { encoding: "utf-8", env: process.env }); + expect(run.status).toBe(0); + expect(run.stdout).toContain("hypothesisNotice"); + }); + + test("a corpus with an unreadable shard exits 1 while still printing the report", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + symlinkSync(join(spaceRoot, "intents", "alpha", "audit", "nowhere.jsonl"), join(spaceRoot, "intents", "alpha", "audit", "dangling.jsonl")); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], { encoding: "utf-8", env: process.env }); + expect(run.status).toBe(1); + expect(run.stdout).toContain('"unreadableShard": 1'); + }); + + test("an unknown flag exits 2 and prints usage", () => { + const run = spawnSync("bun", [TOOL, "--nope"], { encoding: "utf-8", env: process.env }); + expect(run.status).toBe(2); + expect(run.stderr).toContain("Unknown argument"); + }); +}); + +describe("read-only — the shipped source cannot write", () => { + const WRITE_APIS = [ + "writeFileSync", + "appendFileSync", + "mkdirSync", + "rmSync", + "unlinkSync", + "renameSync", + "cpSync", + "copyFileSync", + "mkdtempSync", + "chmodSync", + "createWriteStream", + "writeFile", + "appendFile", + ]; + + test("no filesystem write API is imported or referenced by the tool", () => { + const source = readFileSync(TOOL, "utf-8"); + const imports = [...source.matchAll(/import\s*\{([^}]*)\}\s*from\s*["']node:fs["']/g)].flatMap((m) => + (m[1] ?? "").split(",").map((s) => s.trim()).filter((s) => s !== ""), + ); + expect(imports.sort()).toEqual(["readFileSync", "readdirSync"]); + for (const api of WRITE_APIS) expect(source.includes(api)).toBe(false); + expect(source.includes("node:fs/promises")).toBe(false); + }); + + test("a run leaves the corpus byte-identical", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const snapshot = new Map(); + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else snapshot.set(full, readFileSync(full, "utf-8")); + } + }; + walk(spaceRoot); + capturedMain(["--project-dir", projectDir, "--space", "default"]); + for (const [path, body] of snapshot) expect(readFileSync(path, "utf-8")).toBe(body); + }); +}); diff --git a/tests/unit/t484-stage-stats.test.ts b/tests/unit/t484-stage-stats.test.ts new file mode 100644 index 000000000..8438dd4ef --- /dev/null +++ b/tests/unit/t484-stage-stats.test.ts @@ -0,0 +1,523 @@ +// covers: function:nearestRankP95, function:composeStageStats +// +// t484 — Issue #2405 (U1 stage-stats). Unit half of the twin: every pure +// function of amadeus-stage-stats.ts, driven in-process so bun --coverage +// measures the whole aggregation contract (the FS scan and the CLI spawn live +// in t485's integration layer — cid:code-generation:fs-tests-integration-first). +import { describe, expect, test } from "bun:test"; +import { + type AttributedRecord, + buildWindows, + composeStageStats, + emptyExclusions, + type MeasuredWindow, + nearestRankP95, + parseReviewHeadings, + reviewAttribution, + type StageWindow, + subtractIdle, + attributeModels, + composeReport, + HYPOTHESIS_NOTICE, + parseArgs, + renderCsv, + renderMarkdown, + serializeJson, + tallySensors, +} from "../../packages/framework/core/tools/amadeus-stage-stats.ts"; + +// Independent oracle for the corpus shape: the tests hand-build v1 and v2 +// journal records rather than routing through the module's own scanner, so a +// scanner defect cannot cancel against the expectation. +function v1(intent: string, event: string, timestamp: string, fields: Record = {}): AttributedRecord { + return { intent, record: { schemaVersion: 1, seq: 1, cloneId: "c", intentId: "intents", timestamp, heading: "h", event, fields } }; +} + +function v2(intent: string, event: string, timestamp: string, attrs: Record = {}): AttributedRecord { + return { + intent, + record: { + schemaVersion: 2, + eventId: "e", + seq: 1, + timestamp, + eventName: "amadeus.test", + attributes: { Event: event, ...attrs }, + intentId: "intents", + space: "default", + cloneId: "c", + traceId: null, + spanId: null, + traceFlags: 0, + idempotencyKey: `${timestamp}:${event}`, + canonical: true, + }, + }; +} + +// An independent oracle for the window shape: the tests build MeasuredWindow +// literals directly rather than calling the module's own constructor, so a +// constructor defect cannot cancel out against the expectation (BR: no +// self-referential comparison). +function win(stage: string, raw: number, net: number): MeasuredWindow { + return { intent: "i", stage, startedAt: "2026-01-01T00:00:00Z", completedAt: "2026-01-01T00:00:01Z", rawSeconds: raw, idleSeconds: raw - net, netSeconds: net }; +} + +describe("nearestRankP95 — nearest rank, NaN on empty", () => { + test("empty input propagates NaN instead of collapsing to 0", () => { + expect(nearestRankP95([])).toBeNaN(); + }); + test("nearest rank picks sorted[ceil(0.95n) - 1]", () => { + const values = Array.from({ length: 100 }, (_, i) => i + 1); + expect(nearestRankP95(values)).toBe(95); + }); + test("a single sample is its own p95", () => { + expect(nearestRankP95([42])).toBe(42); + }); + test("input order does not change the result", () => { + expect(nearestRankP95([9, 1, 5, 3, 7])).toBe(nearestRankP95([1, 3, 5, 7, 9])); + }); +}); + +describe("composeStageStats — per-stage mean/median/p95", () => { + test("groups by stage and reports n, raw median, and net statistics", () => { + const stats = composeStageStats([win("a", 10, 5), win("a", 30, 15), win("b", 100, 100)]); + expect(stats.map((s) => s.stage)).toEqual(["a", "b"]); + const a = stats[0]!; + expect(a.n).toBe(2); + expect(a.rawMedian).toBe(20); + expect(a.netMean).toBe(10); + expect(a.netMedian).toBe(10); + expect(a.netP95).toBe(15); + }); + test("empty population yields an empty table, never a zero row", () => { + expect(composeStageStats([])).toEqual([]); + }); + test("ordering is count-desc then key-asc", () => { + const stats = composeStageStats([win("zz", 1, 1), win("aa", 1, 1), win("aa", 1, 1)]); + expect(stats.map((s) => s.stage)).toEqual(["aa", "zz"]); + }); +}); + +describe("buildWindows — pairing START with COMPLETED per intent x stage", () => { + test("pairs both journal generations and computes raw seconds", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v2("i1", "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(1); + expect(windows[0]!.rawSeconds).toBe(100); + expect(windows[0]!.stage).toBe("design"); + expect(buckets.windowing.unmatchedStart).toBe(0); + expect(buckets.windowing.orphanComplete).toBe(0); + }); + + test("a START with no COMPLETED is the unmatched-start bucket, not a window", () => { + const { windows, buckets } = buildWindows([v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" })]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.unmatchedStart).toBe(1); + }); + + test("a COMPLETED with no START is the orphan-complete bucket", () => { + const { buckets } = buildWindows([v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:00Z", { Stage: "design" })]); + expect(buckets.windowing.orphanComplete).toBe(1); + }); + + test("the same stage in two intents does not cross-pair", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1("i2", "STAGE_COMPLETED", "2026-01-01T00:00:10Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.unmatchedStart).toBe(1); + expect(buckets.windowing.orphanComplete).toBe(1); + }); + + test("records are ordered by timestamp before pairing, not by input order", () => { + const { windows } = buildWindows([ + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:30Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(1); + expect(windows[0]!.rawSeconds).toBe(30); + }); + + test("emptyExclusions starts every bucket at zero", () => { + const zero = emptyExclusions(); + expect(zero.corpus).toEqual({ brokenLine: 0, unreadableShard: 0 }); + expect(zero.windowing).toEqual({ unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0 }); + expect(zero.review).toEqual({ unparseableReviewHeading: 0 }); + }); +}); + +// A stage window built straight from literals, so the idle tests do not depend +// on buildWindows being correct (independent oracle). +function stageWindow(startedAt: string, completedAt: string, rawSeconds: number): StageWindow { + return { intent: "i1", stage: "design", startedAt, completedAt, rawSeconds }; +} + +describe("subtractIdle — clip, merge, and exclude", () => { + test("an approval wait inside the window makes net strictly less than raw", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + ], + ); + expect(measured).toHaveLength(1); + expect(measured[0]!.idleSeconds).toBe(30); + expect(measured[0]!.netSeconds).toBe(70); + expect(measured[0]!.netSeconds).toBeLessThan(measured[0]!.rawSeconds); + }); + + test("a park span and a session gap both subtract", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:10Z"), + v1("i1", "WORKFLOW_UNPARKED", "2026-01-01T00:00:20Z"), + v2("i1", "SESSION_ENDED", "2026-01-01T00:00:30Z"), + v2("i1", "SESSION_RESUMED", "2026-01-01T00:00:45Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(25); + expect(measured[0]!.netSeconds).toBe(75); + }); + + test("overlapping idle spans merge — no double subtraction", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:10Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:20Z"), + v1("i1", "WORKFLOW_UNPARKED", "2026-01-01T00:00:40Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(40); + expect(measured[0]!.netSeconds).toBe(60); + }); + + test("idle extending past both window edges clips to the window, never negative", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:10Z", "2026-01-01T00:00:20Z", 10)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:00Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:01:00Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(10); + expect(measured[0]!.netSeconds).toBe(0); + expect(measured[0]!.netSeconds).toBeGreaterThanOrEqual(0); + }); + + test("idle abutting the window end (same-try-block gate approval) yields no negative net", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:30Z", 30)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:30Z"), + ], + ); + expect(measured[0]!.netSeconds).toBe(20); + }); + + test("an idle span in a different intent does not reach this window", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i2", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i2", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(0); + expect(measured[0]!.netSeconds).toBe(100); + }); + + test("an unclosed idle opener drops the window from the population and is counted", () => { + const { measured, buckets } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z")], + ); + expect(measured).toHaveLength(0); + expect(buckets.windowing.unclosedIdle).toBe(1); + expect(buckets.windowing.zeroSecond).toBe(0); + }); + + test("a zero-second window is excluded as below the timestamp resolution", () => { + const { measured, buckets } = subtractIdle([stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0)], []); + expect(measured).toHaveLength(0); + expect(buckets.windowing.zeroSecond).toBe(1); + }); + + test("zeroSecond and unclosedIdle are mutually exclusive — a window satisfying both counts once", () => { + const { measured, buckets } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0)], + [v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:00Z")], + ); + expect(measured).toHaveLength(0); + expect(buckets.windowing.unclosedIdle + buckets.windowing.zeroSecond).toBe(1); + expect(buckets.windowing.unclosedIdle).toBe(1); + }); + + test("identity W holds: constructed windows = population + unclosedIdle + zeroSecond", () => { + const windows = [ + stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:10Z", 10), + stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0), + stageWindow("2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z", 60), + ]; + const { measured, buckets } = subtractIdle(windows, [v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:01:10Z")]); + expect(measured.length + buckets.windowing.unclosedIdle + buckets.windowing.zeroSecond).toBe(windows.length); + }); +}); + +describe("parseReviewHeadings — two-stage match, suffixes are reported", () => { + test("a canonical heading yields its iteration", () => { + const { iterations, unparseable } = parseReviewHeadings("intro\n\n## Review — Iteration 2\n\nbody\n"); + expect(iterations).toEqual([2]); + expect(unparseable).toBe(0); + }); + test("a suffixed heading is counted as unparseable, never dropped in silence", () => { + const { iterations, unparseable } = parseReviewHeadings("## Review — Iteration 1 (follow-up)\n"); + expect(iterations).toEqual([]); + expect(unparseable).toBe(1); + }); + test("multiple canonical headings all count", () => { + const { iterations } = parseReviewHeadings("## Review — Iteration 1\nx\n## Review — Iteration 3\n"); + expect(iterations).toEqual([1, 3]); + }); + test("a line that only mentions the marker mid-text is not a heading", () => { + const { iterations, unparseable } = parseReviewHeadings("see ## Review — Iteration 1 above\n"); + expect(iterations).toEqual([]); + expect(unparseable).toBe(0); + }); + test("an unrelated H2 is not a review heading", () => { + expect(parseReviewHeadings("## Findings\n").unparseable).toBe(0); + }); +}); + +describe("reviewAttribution — stage and unit come from the path", () => { + test("a construction unit path keeps its unit segment", () => { + expect(reviewAttribution("construction/stage-stats-cli/functional-design/business-rules.md")).toEqual({ + stagePath: "construction/stage-stats-cli/functional-design", + unit: "stage-stats-cli", + }); + }); + test("a literal {unit-name} directory is displayed as-is, not corrected", () => { + expect(reviewAttribution("construction/{unit-name}/code-generation/code-summary.md").unit).toBe("{unit-name}"); + }); + test("a non-construction path has no unit", () => { + expect(reviewAttribution("inception/requirements-analysis/requirements.md")).toEqual({ + stagePath: "inception/requirements-analysis", + unit: null, + }); + }); +}); + +describe("tallySensors — bucketed by `Stage slug`, never by `Stage`", () => { + test("the `Stage slug` attribute wins over a differing `Stage`", () => { + const rows = tallySensors([ + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:00Z", { "Stage slug": "design", Stage: "Design Stage" }), + v1("i1", "SENSOR_PASSED", "2026-01-01T00:00:01Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2("i1", "SENSOR_FIRED", "2026-01-01T00:00:02Z", { "Stage slug": "design", Stage: "other" }), + v2("i1", "SENSOR_FAILED", "2026-01-01T00:00:03Z", { "Stage slug": "design", Stage: "other" }), + ]); + expect(rows).toHaveLength(1); + expect(rows[0]!.stageSlug).toBe("design"); + expect(rows[0]!.fired).toBe(2); + expect(rows[0]!.passed).toBe(1); + expect(rows[0]!.failed).toBe(1); + expect(rows[0]!.failedRate).toBe(0.5); + }); + test("a slug that never fired reports NaN, not a 0% pass", () => { + const rows = tallySensors([v1("i1", "SENSOR_FAILED", "2026-01-01T00:00:00Z", { "Stage slug": "x" })]); + expect(rows[0]!.failedRate).toBeNaN(); + }); + test("ordering is fired-desc then slug-asc", () => { + const rows = tallySensors([ + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:00Z", { "Stage slug": "zz" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:01Z", { "Stage slug": "aa" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:02Z", { "Stage slug": "aa" }), + ]); + expect(rows.map((r) => r.stageSlug)).toEqual(["aa", "zz"]); + }); +}); + +describe("attributeModels — absence is recorded as absence", () => { + test("attributable + unresolved equals the total (identity M)", () => { + const attribution = attributeModels([ + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:00Z", { Model: "opus", "Model Source": "pin" }), + v2("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:01Z", { Model: "sonnet", "Model Source": "default" }), + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:02Z", {}), + v1("i1", "SUBAGENT_STARTED", "2026-01-01T00:00:03Z", { Model: "opus" }), + ]); + expect(attribution.totalCount).toBe(4); + expect(attribution.attributableCount).toBe(3); + expect(attribution.unresolvedCount).toBe(1); + expect(attribution.attributableCount + attribution.unresolvedCount).toBe(attribution.totalCount); + expect(attribution.byModel.get("opus")).toBe(2); + expect(attribution.byModel.get("sonnet")).toBe(1); + expect(attribution.byModelSource.get("pin")).toBe(1); + }); + test("a blank Model is unresolved, not a model named empty string", () => { + const attribution = attributeModels([v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:00Z", { Model: " " })]); + expect(attribution.unresolvedCount).toBe(1); + expect(attribution.byModel.size).toBe(0); + }); + test("non-subagent events are outside the model population entirely", () => { + expect(attributeModels([v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z")]).totalCount).toBe(0); + }); +}); + +// A corpus exercising every reported axis at once: two paired windows, an idle +// span, an unmatched start, a zero-second window, sensors on two slugs, and one +// attributable plus one unattributable subagent row. +function mixedCorpus(): AttributedRecord[] { + return [ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:40Z"), + v2("i1", "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:02:00Z", { Stage: "build" }), + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:02:30Z", { Stage: "build" }), + v1("i2", "STAGE_STARTED", "2026-01-01T00:03:00Z", { Stage: "design" }), + v1("i2", "STAGE_COMPLETED", "2026-01-01T00:03:00Z", { Stage: "design" }), + v1("i2", "STAGE_STARTED", "2026-01-01T00:04:00Z", { Stage: "orphan" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:05Z", { "Stage slug": "design", Stage: "Design" }), + v1("i1", "SENSOR_FAILED", "2026-01-01T00:00:06Z", { "Stage slug": "design", Stage: "Design" }), + v2("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:07Z", { Model: "opus", "Model Source": "pin" }), + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:08Z", {}), + ]; +} + +function sampleReport() { + return composeReport({ + scanScope: "space \"default\"", + corpus: { records: mixedCorpus(), unreadableShardCount: 0, brokenLineCount: 2, shardCount: 3, lineCount: 13 }, + reviewBlocks: [ + { intent: "i1", stagePath: "construction/u/functional-design", unit: "u", iteration: 1 }, + { intent: "i1", stagePath: "construction/u/functional-design", unit: "u", iteration: 2 }, + { intent: "i1", stagePath: "construction/{unit-name}/code-generation", unit: "{unit-name}", iteration: 1 }, + ], + unparseableReviewHeadingCount: 1, + }); +} + +describe("composeReport — measurement ref, buckets, and the layered identity", () => { + test("carries every corpus counter into the report", () => { + const report = sampleReport(); + expect(report.shardCount).toBe(3); + expect(report.lineCount).toBe(13); + expect(report.exclusions.corpus.brokenLine).toBe(2); + expect(report.exclusions.corpus.unreadableShard).toBe(0); + expect(report.exclusions.review.unparseableReviewHeading).toBe(1); + }); + + test("identity W holds on the assembled report", () => { + const report = sampleReport(); + const w = report.exclusions.windowing; + expect(report.populationCount + w.unclosedIdle + w.zeroSecond).toBe(report.constructedWindowCount); + }); + + test("identity M holds on the assembled report", () => { + const m = sampleReport().models; + expect(m.attributableCount + m.unresolvedCount).toBe(m.totalCount); + }); + + test("the unmatched start is counted but is not part of identity W", () => { + const report = sampleReport(); + expect(report.exclusions.windowing.unmatchedStart).toBe(1); + expect(report.constructedWindowCount).toBe(3); + }); + + test("idle subtraction shows up as net below raw for the design stage", () => { + const design = sampleReport().stages.find((s) => s.stage === "design"); + expect(design?.netMedian).toBe(80); + expect(design?.rawMedian).toBe(100); + }); + + test("review iterations are bucketed by stage path, literal unit dirs kept", () => { + const buckets = sampleReport().reviewBuckets; + expect(buckets.map((b) => b.stagePath)).toContain("construction/{unit-name}/code-generation"); + const fd = buckets.find((b) => b.stagePath === "construction/u/functional-design"); + expect(fd?.blocks).toBe(2); + expect(fd?.maxIteration).toBe(2); + }); +}); + +describe("renderers — deterministic, header-first, hypothesis stated", () => { + test("markdown opens with the measurement ref and states the hypothesis", () => { + const text = renderMarkdown(sampleReport()); + const head = text.split("\n").slice(0, 20).join("\n"); + expect(head).toContain("shards: 3"); + expect(head).toContain("lines: 13"); + expect(head).toContain("broken-line: 2"); + expect(head).toContain("unreadable-shard: 0"); + expect(head).toContain("unmatched-start: 1"); + expect(head).toContain("orphan-complete: 0"); + expect(head).toContain("unclosed-idle: 0"); + expect(head).toContain("zero-second: 1"); + expect(head).toContain("unparseable-review-heading: 1"); + expect(text).toContain(HYPOTHESIS_NOTICE); + }); + + test("csv also carries the measurement ref and the hypothesis before the rows", () => { + const text = renderCsv(sampleReport()); + const lines = text.split("\n"); + expect(text).toContain(HYPOTHESIS_NOTICE); + expect(lines[0]).toBe("section,key,value"); + expect(lines[1]).toContain("stage-stats"); + expect(text).toContain("zero-second,"); + // Measurement ref precedes every data section, not merely present somewhere. + expect(text.indexOf("unparseable-review-heading")).toBeLessThan(text.indexOf("stage,n,raw_median")); + }); + + test("all three forms are byte-identical across two runs of the same input", () => { + expect(renderMarkdown(sampleReport())).toBe(renderMarkdown(sampleReport())); + expect(renderCsv(sampleReport())).toBe(renderCsv(sampleReport())); + expect(JSON.stringify(serializeJson(sampleReport()))).toBe(JSON.stringify(serializeJson(sampleReport()))); + }); + + test("json turns the maps into fixed-order arrays and keeps the notice", () => { + const json = serializeJson(sampleReport()) as Record; + expect(json.hypothesisNotice).toBe(HYPOTHESIS_NOTICE); + expect(Array.isArray((json.models as Record).byModel)).toBe(true); + expect(json.constructedWindowCount).toBe(3); + }); +}); + +describe("parseArgs — parse, do not validate", () => { + test("no arguments default to markdown", () => { + const parsed = parseArgs([]); + expect(parsed.ok).toBe(true); + if (parsed.ok) expect(parsed.value.format).toBe("markdown"); + }); + test("--json selects the json form", () => { + const parsed = parseArgs(["--json"]); + expect(parsed.ok && parsed.value.format).toBe("json"); + }); + test("--format csv selects the csv form", () => { + const parsed = parseArgs(["--format", "csv"]); + expect(parsed.ok && parsed.value.format).toBe("csv"); + }); + test("an unknown flag is a usage error, not a silently ignored argument", () => { + const parsed = parseArgs(["--nope"]); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error.message).toContain("--nope"); + }); + test("an option missing its value is a usage error", () => { + expect(parseArgs(["--space"]).ok).toBe(false); + }); + test("an unsupported --format value is a usage error", () => { + expect(parseArgs(["--format", "yaml"]).ok).toBe(false); + }); + test("an unsafe --space value is a usage error", () => { + expect(parseArgs(["--space", "../etc"]).ok).toBe(false); + }); + test("--project-dir and --space carry through when valid", () => { + const parsed = parseArgs(["--project-dir", "/tmp/x", "--space", "team-a"]); + expect(parsed.ok && parsed.value.projectDir).toBe("/tmp/x"); + expect(parsed.ok && parsed.value.space).toBe("team-a"); + }); +}); From 0a1daf3293499205b5e7b268149cfa0589fb9dac Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 08:06:07 +0900 Subject: [PATCH 2/8] test: renumber stage-stats twins to t486/t487 after base advance; regen coverage registry --- tests/.coverage-ratchet.json | 4 +- tests/.coverage-registry.json | 38 ++++--------------- ...s => t487-stage-stats.integration.test.ts} | 4 +- ...stats.test.ts => t486-stage-stats.test.ts} | 4 +- 4 files changed, 13 insertions(+), 37 deletions(-) rename tests/integration/{t485-stage-stats.integration.test.ts => t487-stage-stats.integration.test.ts} (99%) rename tests/unit/{t484-stage-stats.test.ts => t486-stage-stats.test.ts} (99%) diff --git a/tests/.coverage-ratchet.json b/tests/.coverage-ratchet.json index bce1c151a..072b63ccc 100644 --- a/tests/.coverage-ratchet.json +++ b/tests/.coverage-ratchet.json @@ -1,11 +1,11 @@ { "note": "Committed baseline: covered-unit count per class. The --check ratchet fails CI if any class's covered count DROPS below these numbers without a reviewed deferred entry. Monotonic anti-regression: you can cover more, never silently less. Regenerate with: bun tests/gen-coverage-registry.ts", "coveredByClass": { - "function": 181, + "function": 180, "audit": 44, "scope": 15, "stage": 8, - "hook": 15, + "hook": 14, "subcommand": 85, "render-surface": 7 } diff --git a/tests/.coverage-registry.json b/tests/.coverage-registry.json index 08ca276a3..5098b4a92 100644 --- a/tests/.coverage-registry.json +++ b/tests/.coverage-registry.json @@ -20,22 +20,22 @@ "render-surface": "tui" }, "counts": { - "total": 619, + "total": 617, "enumeratedByClass": { - "function": 350, + "function": 349, "audit": 90, "scope": 15, "stage": 32, - "hook": 15, + "hook": 14, "subcommand": 110, "render-surface": 7 }, "coveredByClass": { - "function": 181, + "function": 180, "audit": 44, "scope": 15, "stage": 8, - "hook": 15, + "hook": 14, "subcommand": 85, "render-surface": 7 } @@ -1935,18 +1935,6 @@ ], "status": "covered" }, - { - "unitClass": "function", - "unitId": "function:evaluateDispatchGuard", - "minMechanism": "none", - "coveredBy": [ - { - "file": "tests/integration/t485-subagent-model-guard.integration.test.ts", - "mechanism": "cli" - } - ], - "status": "covered" - }, { "unitClass": "function", "unitId": "function:EXECUTION_PROJECTION_DIGEST_FIELD", @@ -2763,7 +2751,7 @@ "mechanism": "none" }, { - "file": "tests/integration/t485-stage-stats.integration.test.ts", + "file": "tests/integration/t487-stage-stats.integration.test.ts", "mechanism": "cli" }, { @@ -4263,7 +4251,7 @@ }, { "unitClass": "function", - "unitId": "function:SUBAGENT_DISPATCH_TOOLS", + "unitId": "function:SUBAGENT_DISPATCH_TOOL", "minMechanism": "none", "coveredBy": [], "status": "UNCOVERED" @@ -5014,18 +5002,6 @@ ], "status": "covered" }, - { - "unitClass": "hook", - "unitId": "amadeus-subagent-model-guard", - "minMechanism": "none", - "coveredBy": [ - { - "file": "tests/integration/t485-subagent-model-guard.integration.test.ts", - "mechanism": "cli" - } - ], - "status": "covered" - }, { "unitClass": "hook", "unitId": "amadeus-sync-statusline", diff --git a/tests/integration/t485-stage-stats.integration.test.ts b/tests/integration/t487-stage-stats.integration.test.ts similarity index 99% rename from tests/integration/t485-stage-stats.integration.test.ts rename to tests/integration/t487-stage-stats.integration.test.ts index 046145d10..c188c6ed4 100644 --- a/tests/integration/t485-stage-stats.integration.test.ts +++ b/tests/integration/t487-stage-stats.integration.test.ts @@ -1,10 +1,10 @@ // covers: function:scanCorpus, function:collectReviewBlocks, function:main // size: medium // -// t485 — Issue #2405 (U1 stage-stats). Integration half of the twin: the real +// t487 — Issue #2405 (U1 stage-stats). Integration half of the twin: the real // filesystem scan, the CLI shell driven in-process (so bun --coverage measures // it), and the exit ladder measured by an actual spawn. The pure aggregation -// core is t484's job. +// core is t486's job. // // Every expectation is checked against an INDEPENDENT ORACLE — a walker // written here in the test that counts shards, lines and events on its own — diff --git a/tests/unit/t484-stage-stats.test.ts b/tests/unit/t486-stage-stats.test.ts similarity index 99% rename from tests/unit/t484-stage-stats.test.ts rename to tests/unit/t486-stage-stats.test.ts index 8438dd4ef..7c6baba15 100644 --- a/tests/unit/t484-stage-stats.test.ts +++ b/tests/unit/t486-stage-stats.test.ts @@ -1,9 +1,9 @@ // covers: function:nearestRankP95, function:composeStageStats // -// t484 — Issue #2405 (U1 stage-stats). Unit half of the twin: every pure +// t486 — Issue #2405 (U1 stage-stats). Unit half of the twin: every pure // function of amadeus-stage-stats.ts, driven in-process so bun --coverage // measures the whole aggregation contract (the FS scan and the CLI spawn live -// in t485's integration layer — cid:code-generation:fs-tests-integration-first). +// in t487's integration layer — cid:code-generation:fs-tests-integration-first). import { describe, expect, test } from "bun:test"; import { type AttributedRecord, From c5398862cc71b2a852cfbf3580d4cea9b0ae9838 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 08:21:16 +0900 Subject: [PATCH 3/8] test: restore registry entries dropped by pre-build regen; admit t487 as a cli spawner --- tests/.coverage-ratchet.json | 4 +-- tests/.coverage-registry.json | 36 +++++++++++++++---- .../t-coverage-mechanism-ratchet.test.ts | 1 + 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/.coverage-ratchet.json b/tests/.coverage-ratchet.json index 072b63ccc..bce1c151a 100644 --- a/tests/.coverage-ratchet.json +++ b/tests/.coverage-ratchet.json @@ -1,11 +1,11 @@ { "note": "Committed baseline: covered-unit count per class. The --check ratchet fails CI if any class's covered count DROPS below these numbers without a reviewed deferred entry. Monotonic anti-regression: you can cover more, never silently less. Regenerate with: bun tests/gen-coverage-registry.ts", "coveredByClass": { - "function": 180, + "function": 181, "audit": 44, "scope": 15, "stage": 8, - "hook": 14, + "hook": 15, "subcommand": 85, "render-surface": 7 } diff --git a/tests/.coverage-registry.json b/tests/.coverage-registry.json index 5098b4a92..5cbf098db 100644 --- a/tests/.coverage-registry.json +++ b/tests/.coverage-registry.json @@ -20,22 +20,22 @@ "render-surface": "tui" }, "counts": { - "total": 617, + "total": 619, "enumeratedByClass": { - "function": 349, + "function": 350, "audit": 90, "scope": 15, "stage": 32, - "hook": 14, + "hook": 15, "subcommand": 110, "render-surface": 7 }, "coveredByClass": { - "function": 180, + "function": 181, "audit": 44, "scope": 15, "stage": 8, - "hook": 14, + "hook": 15, "subcommand": 85, "render-surface": 7 } @@ -1935,6 +1935,18 @@ ], "status": "covered" }, + { + "unitClass": "function", + "unitId": "function:evaluateDispatchGuard", + "minMechanism": "none", + "coveredBy": [ + { + "file": "tests/integration/t485-subagent-model-guard.integration.test.ts", + "mechanism": "cli" + } + ], + "status": "covered" + }, { "unitClass": "function", "unitId": "function:EXECUTION_PROJECTION_DIGEST_FIELD", @@ -4251,7 +4263,7 @@ }, { "unitClass": "function", - "unitId": "function:SUBAGENT_DISPATCH_TOOL", + "unitId": "function:SUBAGENT_DISPATCH_TOOLS", "minMechanism": "none", "coveredBy": [], "status": "UNCOVERED" @@ -5002,6 +5014,18 @@ ], "status": "covered" }, + { + "unitClass": "hook", + "unitId": "amadeus-subagent-model-guard", + "minMechanism": "none", + "coveredBy": [ + { + "file": "tests/integration/t485-subagent-model-guard.integration.test.ts", + "mechanism": "cli" + } + ], + "status": "covered" + }, { "unitClass": "hook", "unitId": "amadeus-sync-statusline", diff --git a/tests/integration/t-coverage-mechanism-ratchet.test.ts b/tests/integration/t-coverage-mechanism-ratchet.test.ts index 13124559a..6dd3a4179 100644 --- a/tests/integration/t-coverage-mechanism-ratchet.test.ts +++ b/tests/integration/t-coverage-mechanism-ratchet.test.ts @@ -95,6 +95,7 @@ describe("repository-wide mechanism honesty ratchets", () => { // a new spawning test still cannot land without a human edit here. const EXPECTED_NONE_TO_CLI = [ "integration/t485-subagent-model-guard.integration.test.ts", + "integration/t487-stage-stats.integration.test.ts", "e2e/t-formal-verif-model-completeness-sensor.test.ts", "e2e/t237-election-walking-skeleton.test.ts", "e2e/t265-engine-boundary.test.ts", From e4b1962ce8520afc3aecee0d2989558e1a90edd7 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 08:33:11 +0900 Subject: [PATCH 4/8] fix(tools): attribute models on SUBAGENT_COMPLETED only to avoid double counting --- packages/framework/core/tools/amadeus-stage-stats.ts | 9 ++++++--- tests/unit/t486-stage-stats.test.ts | 11 ++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts index 668d7d4e1..91ff7f41c 100644 --- a/packages/framework/core/tools/amadeus-stage-stats.ts +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -395,8 +395,11 @@ export interface ModelAttribution { readonly totalCount: number; } -/** Attribute subagent rows to models. A row without a usable `Model` is the - * UNKNOWN bucket: the absence of the attribute is itself the record, so it is +/** Attribute subagent rows to models. Only SUBAGENT_COMPLETED rows join the + * population — a start and its completion describe one dispatch, and the + * sibling stats tool attributes on completion, so counting both would double + * every per-model total. A row without a usable `Model` is the UNKNOWN + * bucket: the absence of the attribute is itself the record, so it is * reported rather than dropped from the denominator. */ export function attributeModels(records: readonly AttributedRecord[]): ModelAttribution { const byModel = new Map(); @@ -407,7 +410,7 @@ export function attributeModels(records: readonly AttributedRecord[]): ModelAttr for (const { record } of records) { const event = eventOf(record); - if (event !== "SUBAGENT_COMPLETED" && event !== "SUBAGENT_STARTED") continue; + if (event !== "SUBAGENT_COMPLETED") continue; totalCount += 1; const model = attrOf(record, "Model"); if (model === null || model.trim() === "") { diff --git a/tests/unit/t486-stage-stats.test.ts b/tests/unit/t486-stage-stats.test.ts index 7c6baba15..588314202 100644 --- a/tests/unit/t486-stage-stats.test.ts +++ b/tests/unit/t486-stage-stats.test.ts @@ -352,14 +352,19 @@ describe("attributeModels — absence is recorded as absence", () => { v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:02Z", {}), v1("i1", "SUBAGENT_STARTED", "2026-01-01T00:00:03Z", { Model: "opus" }), ]); - expect(attribution.totalCount).toBe(4); - expect(attribution.attributableCount).toBe(3); + expect(attribution.totalCount).toBe(3); + expect(attribution.attributableCount).toBe(2); expect(attribution.unresolvedCount).toBe(1); expect(attribution.attributableCount + attribution.unresolvedCount).toBe(attribution.totalCount); - expect(attribution.byModel.get("opus")).toBe(2); + expect(attribution.byModel.get("opus")).toBe(1); expect(attribution.byModel.get("sonnet")).toBe(1); expect(attribution.byModelSource.get("pin")).toBe(1); }); + test("a start row never joins the population — one dispatch is one completion", () => { + const attribution = attributeModels([v1("i1", "SUBAGENT_STARTED", "2026-01-01T00:00:00Z", { Model: "opus" })]); + expect(attribution.totalCount).toBe(0); + expect(attribution.byModel.size).toBe(0); + }); test("a blank Model is unresolved, not a model named empty string", () => { const attribution = attributeModels([v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:00Z", { Model: " " })]); expect(attribution.unresolvedCount).toBe(1); From 35ddd5bb0c311c0cdba1669bc83641b451fe38d8 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 08:47:01 +0900 Subject: [PATCH 5/8] test: drive the resolution seams in-process for patch coverage; collapse the broken-line catch onto one measurable line --- .../core/tools/amadeus-stage-stats.ts | 5 +- .../t487-stage-stats.integration.test.ts | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts index 91ff7f41c..bf4d72224 100644 --- a/packages/framework/core/tools/amadeus-stage-stats.ts +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -815,10 +815,7 @@ function recordsFromShard(intent: string, body: string): { records: AttributedRe lines += 1; try { for (const record of readJournalRecords(line)) records.push({ intent, record }); - } catch { - broken += 1; - continue; - } + } catch { broken += 1; } } return { records, lines, broken }; } diff --git a/tests/integration/t487-stage-stats.integration.test.ts b/tests/integration/t487-stage-stats.integration.test.ts index c188c6ed4..912bb14f9 100644 --- a/tests/integration/t487-stage-stats.integration.test.ts +++ b/tests/integration/t487-stage-stats.integration.test.ts @@ -404,3 +404,51 @@ describe("read-only — the shipped source cannot write", () => { for (const [path, body] of snapshot) expect(readFileSync(path, "utf-8")).toBe(body); }); }); + +describe("resolution seams — driven in-process so bun --coverage measures them", () => { + test("a usage error returns 2 without touching the filesystem", () => { + const { code } = capturedMain(["--definitely-unknown-flag"]); + expect(code).toBe(2); + }); + + test("CLAUDE_PROJECT_DIR resolves the project dir when the flag is absent", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const saved = process.env.CLAUDE_PROJECT_DIR; + process.env.CLAUDE_PROJECT_DIR = projectDir; + try { + const { code, stdout } = capturedMain(["--space", "default"]); + expect(code).toBe(0); + expect(stdout).toContain("shards:"); + } finally { + if (saved === undefined) delete process.env.CLAUDE_PROJECT_DIR; + else process.env.CLAUDE_PROJECT_DIR = saved; + } + }); + + test("without env or flag the cwd rung resolves, and active-space names the space", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + writeFileSync(join(projectDir, "amadeus", "active-space"), "default\n"); + const savedEnv = process.env.CLAUDE_PROJECT_DIR; + const savedCwd = process.cwd(); + delete process.env.CLAUDE_PROJECT_DIR; + process.chdir(projectDir); + try { + const { code, stdout } = capturedMain([]); + expect(code).toBe(0); + expect(stdout).toContain('space "default"'); + } finally { + process.chdir(savedCwd); + if (savedEnv !== undefined) process.env.CLAUDE_PROJECT_DIR = savedEnv; + } + }); + + test("a missing active-space file falls back to the default space", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { code, stdout } = capturedMain(["--project-dir", projectDir]); + expect(code).toBe(0); + expect(stdout).toContain('space "default"'); + }); +}); From 1ac10dd5e8d610de79627788d9e4e4235d07ea98 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 08:48:39 +0900 Subject: [PATCH 6/8] fix(tools): keep the explicit continue terminal in the broken-line catch for the no-silent-drop gate --- packages/framework/core/tools/amadeus-stage-stats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts index bf4d72224..8283ddf22 100644 --- a/packages/framework/core/tools/amadeus-stage-stats.ts +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -815,7 +815,7 @@ function recordsFromShard(intent: string, body: string): { records: AttributedRe lines += 1; try { for (const record of readJournalRecords(line)) records.push({ intent, record }); - } catch { broken += 1; } + } catch { broken += 1; continue; } } return { records, lines, broken }; } From ca87682e623c86ad8dd208ebb79d7371d01d418c Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 09:14:22 +0900 Subject: [PATCH 7/8] fix(tools): collapse the active-space catch onto one measurable line for lcov attribution --- packages/framework/core/tools/amadeus-stage-stats.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts index 8283ddf22..3a53421d3 100644 --- a/packages/framework/core/tools/amadeus-stage-stats.ts +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -899,13 +899,11 @@ function resolveProjectDirLocal(explicitDir: string | undefined): string { } function activeSpaceLocal(projectDir: string): string { + let raw = ""; try { - const raw = readFileSync(join(projectDir, "amadeus", "active-space"), "utf-8").trim(); - if (SAFE_NAME.test(raw)) return raw; - } catch { - return "default"; - } - return "default"; + raw = readFileSync(join(projectDir, "amadeus", "active-space"), "utf-8").trim(); + } catch { return "default"; } + return SAFE_NAME.test(raw) ? raw : "default"; } const RENDERERS: Record string> = { From ee2e5b98a6b793cf5edb950fd071630e1b55fb82 Mon Sep 17 00:00:00 2001 From: Junichi Kato Date: Sat, 8 Aug 2026 09:35:13 +0900 Subject: [PATCH 8/8] =?UTF-8?q?fix(tools):=20address=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20invalid-timestamp=20bucket,=20prototype-safe=20s?= =?UTF-8?q?ensor=20lookup,=20symlink-safe=20record=20walk,=20spawn=20timeo?= =?UTF-8?q?uts,=20sanitize=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/tools/amadeus-stage-stats.ts | 64 +++++++++++++------ .../t487-stage-stats.integration.test.ts | 13 ++-- tests/unit/t486-stage-stats.test.ts | 43 ++++++++++++- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts index 3a53421d3..9fbbf005f 100644 --- a/packages/framework/core/tools/amadeus-stage-stats.ts +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -26,7 +26,7 @@ // the nearest-rank p95 is mirrored locally because the shipped surface must // not reach into tests. -import { readdirSync, readFileSync } from "node:fs"; +import { type Dirent, readdirSync, readFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { type JournalRecord, journalRecordField, readJournalRecords } from "./amadeus-journal.ts"; @@ -77,6 +77,7 @@ export interface ExclusionCounts { readonly orphanComplete: number; readonly unclosedIdle: number; readonly zeroSecond: number; + readonly invalidTimestamp: number; }; readonly review: { readonly unparseableReviewHeading: number }; } @@ -85,7 +86,7 @@ export interface ExclusionCounts { export function emptyExclusions(): ExclusionCounts { return { corpus: { brokenLine: 0, unreadableShard: 0 }, - windowing: { unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0 }, + windowing: { unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0, invalidTimestamp: 0 }, review: { unparseableReviewHeading: 0 }, }; } @@ -136,12 +137,17 @@ export function buildWindows(records: readonly AttributedRecord[]): { windows: S const windows: StageWindow[] = []; let unmatchedStart = 0; let orphanComplete = 0; + let invalidTimestamp = 0; for (const attributed of chronological(records)) { const event = eventOf(attributed.record); if (event !== "STAGE_STARTED" && event !== "STAGE_COMPLETED") continue; const stage = attrOf(attributed.record, "Stage"); if (stage === null || stage === "") continue; + if (Number.isNaN(epochSeconds(attributed.record.timestamp))) { + invalidTimestamp += 1; + continue; + } const key = `${attributed.intent}\u0000${stage}`; if (event === "STAGE_STARTED") { const queue = pending.get(key); @@ -166,7 +172,7 @@ export function buildWindows(records: readonly AttributedRecord[]): { windows: S for (const queue of pending.values()) unmatchedStart += queue.length; const zero = emptyExclusions(); - return { windows, buckets: { ...zero, windowing: { ...zero.windowing, unmatchedStart, orphanComplete } } }; + return { windows, buckets: { ...zero, windowing: { ...zero.windowing, unmatchedStart, orphanComplete, invalidTimestamp } } }; } // --- idle subtraction ------------------------------------------------------ @@ -178,10 +184,13 @@ export interface IdleInterval { readonly end: number; } -/** Per-intent idle spans plus the timestamps of openers that never closed. */ +/** Per-intent idle spans plus the timestamps of openers that never closed. + * Idle events whose timestamp does not parse are counted rather than allowed + * to poison the interval arithmetic with NaN. */ export interface IdleIndex { readonly intervals: ReadonlyMap; readonly unclosedOpeners: ReadonlyMap; + readonly invalidTimestampCount: number; } // The three idle shapes, as opener -> accepted closers. An opener that never @@ -224,12 +233,17 @@ export function indexIdle(records: readonly AttributedRecord[]): IdleIndex { const intervals = new Map(); const unclosedOpeners = new Map(); const openPerIntent: OpenerStacks = new Map(); + let invalidTimestampCount = 0; for (const attributed of chronological(records)) { const event = eventOf(attributed.record); const shape = event === null ? null : idleShapeFor(event); if (shape === null) continue; const at = epochSeconds(attributed.record.timestamp); + if (Number.isNaN(at)) { + invalidTimestampCount += 1; + continue; + } const stack = openerStack(openPerIntent, attributed.intent, shape.kind); if (event === shape.open) { stack.push(at); @@ -244,7 +258,7 @@ export function indexIdle(records: readonly AttributedRecord[]): IdleIndex { for (const stack of perKind.values()) leftovers.push(...stack); if (leftovers.length > 0) unclosedOpeners.set(intent, leftovers.sort((a, b) => a - b)); } - return { intervals, unclosedOpeners }; + return { intervals, unclosedOpeners, invalidTimestampCount }; } // Clip each span to the window, then merge the survivors so overlapping spans @@ -301,7 +315,10 @@ export function subtractIdle( } const zero = emptyExclusions(); - return { measured, buckets: { ...zero, windowing: { ...zero.windowing, unclosedIdle, zeroSecond } } }; + return { + measured, + buckets: { ...zero, windowing: { ...zero.windowing, unclosedIdle, zeroSecond, invalidTimestamp: index.invalidTimestampCount } }, + }; } // --- review iterations ----------------------------------------------------- @@ -358,11 +375,13 @@ export interface SensorTally { readonly failedRate: number; } -const SENSOR_EVENTS: Record = { - SENSOR_FIRED: "fired", - SENSOR_PASSED: "passed", - SENSOR_FAILED: "failed", -}; +// A Map, not an object literal: the corpus is outside the trust boundary, so an +// event name like "constructor" must miss instead of hitting Object.prototype. +const SENSOR_EVENTS: ReadonlyMap = new Map([ + ["SENSOR_FIRED", "fired"], + ["SENSOR_PASSED", "passed"], + ["SENSOR_FAILED", "failed"], +]); /** Tally sensor outcomes by the `Stage slug` attribute. The lifecycle events' * `Stage` attribute is a different key with different values and is never read @@ -371,7 +390,7 @@ export function tallySensors(records: readonly AttributedRecord[]): SensorTally[ const counters = new Map(); for (const { record } of records) { const event = eventOf(record); - const outcome = event === null ? undefined : SENSOR_EVENTS[event]; + const outcome = event === null ? undefined : SENSOR_EVENTS.get(event); if (outcome === undefined) continue; const slug = attrOf(record, "Stage slug"); if (slug === null || slug === "") continue; @@ -545,6 +564,7 @@ export function composeReport(input: { orphanComplete: built.buckets.windowing.orphanComplete, unclosedIdle: subtracted.buckets.windowing.unclosedIdle, zeroSecond: subtracted.buckets.windowing.zeroSecond, + invalidTimestamp: built.buckets.windowing.invalidTimestamp + subtracted.buckets.windowing.invalidTimestamp, }, review: { unparseableReviewHeading: input.unparseableReviewHeadingCount }, }, @@ -596,6 +616,7 @@ function measurementRefLines(report: StageStatsReport): string[] { `orphan-complete: ${windowing.orphanComplete}`, `unclosed-idle: ${windowing.unclosedIdle}`, `zero-second: ${windowing.zeroSecond}`, + `invalid-timestamp: ${windowing.invalidTimestamp}`, `unparseable-review-heading: ${review.unparseableReviewHeading}`, `constructed-windows: ${report.constructedWindowCount}`, `net-population: ${report.populationCount}`, @@ -850,14 +871,19 @@ export function scanCorpus(spaceRoot: string): ScannedCorpus { return { records, unreadableShardCount, brokenLineCount, shardCount, lineCount }; } -/** Markdown artefacts under one intent, as intent-relative paths. */ +/** Markdown artefacts under one intent, as intent-relative paths. Dirent-typed + * so only real directories recurse — a directory symlink pointing back at an + * ancestor would otherwise recurse forever. */ function markdownUnder(root: string, prefix: string, out: string[]): void { - for (const entry of listOrEmpty(root)) { - if (entry === "audit" || entry === "node_modules") continue; - const full = join(root, entry); - const relative = prefix === "" ? entry : `${prefix}/${entry}`; - if (entry.endsWith(".md")) out.push(relative); - else markdownUnder(full, relative, out); + let entries: Dirent[]; + try { + entries = readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + } catch { return; } + for (const entry of entries) { + if (entry.name === "audit" || entry.name === "node_modules") continue; + const relative = prefix === "" ? entry.name : `${prefix}/${entry.name}`; + if (entry.isFile() && entry.name.endsWith(".md")) out.push(relative); + else if (entry.isDirectory()) markdownUnder(join(root, entry.name), relative, out); } } diff --git a/tests/integration/t487-stage-stats.integration.test.ts b/tests/integration/t487-stage-stats.integration.test.ts index 912bb14f9..ab46a81ac 100644 --- a/tests/integration/t487-stage-stats.integration.test.ts +++ b/tests/integration/t487-stage-stats.integration.test.ts @@ -265,7 +265,7 @@ describe("main — the whole pipeline over a real corpus", () => { const { code, stdout } = capturedMain(["--project-dir", projectDir, "--space", "default"]); expect(code).toBe(0); const head = stdout.split("\n").slice(0, 16).join("\n"); - for (const key of ["shards:", "lines:", "broken-line:", "unreadable-shard:", "unmatched-start:", "orphan-complete:", "unclosed-idle:", "zero-second:", "unparseable-review-heading:"]) { + for (const key of ["shards:", "lines:", "broken-line:", "unreadable-shard:", "unmatched-start:", "orphan-complete:", "unclosed-idle:", "zero-second:", "invalid-timestamp:", "unparseable-review-heading:"]) { expect(head).toContain(key); } expect(stdout).toContain("unverified hypothesis"); @@ -337,10 +337,12 @@ describe("main — the whole pipeline over a real corpus", () => { // --- FR-7: exit ladder and the read-only invariant -------------------------- describe("exit ladder — measured by spawning the CLI", () => { + const RUN_OPTIONS = { encoding: "utf-8", env: process.env, timeout: 60_000, killSignal: "SIGKILL" } as const; + test("a healthy corpus exits 0", () => { const projectDir = scratch(); buildCorpus(projectDir); - const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], { encoding: "utf-8", env: process.env }); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], RUN_OPTIONS); expect(run.status).toBe(0); expect(run.stdout).toContain("hypothesisNotice"); }); @@ -349,13 +351,13 @@ describe("exit ladder — measured by spawning the CLI", () => { const projectDir = scratch(); const spaceRoot = buildCorpus(projectDir); symlinkSync(join(spaceRoot, "intents", "alpha", "audit", "nowhere.jsonl"), join(spaceRoot, "intents", "alpha", "audit", "dangling.jsonl")); - const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], { encoding: "utf-8", env: process.env }); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], RUN_OPTIONS); expect(run.status).toBe(1); expect(run.stdout).toContain('"unreadableShard": 1'); }); test("an unknown flag exits 2 and prints usage", () => { - const run = spawnSync("bun", [TOOL, "--nope"], { encoding: "utf-8", env: process.env }); + const run = spawnSync("bun", [TOOL, "--nope"], RUN_OPTIONS); expect(run.status).toBe(2); expect(run.stderr).toContain("Unknown argument"); }); @@ -383,7 +385,8 @@ describe("read-only — the shipped source cannot write", () => { const imports = [...source.matchAll(/import\s*\{([^}]*)\}\s*from\s*["']node:fs["']/g)].flatMap((m) => (m[1] ?? "").split(",").map((s) => s.trim()).filter((s) => s !== ""), ); - expect(imports.sort()).toEqual(["readFileSync", "readdirSync"]); + const valueImports = imports.filter((s) => !s.startsWith("type ")); + expect(valueImports.sort()).toEqual(["readFileSync", "readdirSync"]); for (const api of WRITE_APIS) expect(source.includes(api)).toBe(false); expect(source.includes("node:fs/promises")).toBe(false); }); diff --git a/tests/unit/t486-stage-stats.test.ts b/tests/unit/t486-stage-stats.test.ts index 588314202..e9cf8ce06 100644 --- a/tests/unit/t486-stage-stats.test.ts +++ b/tests/unit/t486-stage-stats.test.ts @@ -10,6 +10,7 @@ import { buildWindows, composeStageStats, emptyExclusions, + indexIdle, type MeasuredWindow, nearestRankP95, parseReviewHeadings, @@ -145,9 +146,32 @@ describe("buildWindows — pairing START with COMPLETED per intent x stage", () test("emptyExclusions starts every bucket at zero", () => { const zero = emptyExclusions(); expect(zero.corpus).toEqual({ brokenLine: 0, unreadableShard: 0 }); - expect(zero.windowing).toEqual({ unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0 }); + expect(zero.windowing).toEqual({ unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0, invalidTimestamp: 0 }); expect(zero.review).toEqual({ unparseableReviewHeading: 0 }); }); + + test("an unparseable timestamp leaves the window population and is counted, not NaN-poisoned", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "not-a-timestamp", { Stage: "design" }), + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:30Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:01:00Z", { Stage: "build" }), + v1("i1", "STAGE_COMPLETED", "also-garbage", { Stage: "build" }), + ]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.invalidTimestamp).toBe(2); + expect(buckets.windowing.orphanComplete).toBe(1); + expect(buckets.windowing.unmatchedStart).toBe(1); + }); + + test("an idle event with an unparseable timestamp is counted instead of poisoning the intervals", () => { + const index = indexIdle([ + v1("i1", "STAGE_AWAITING_APPROVAL", "garbage-time"), + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:05Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:10Z"), + ]); + expect(index.invalidTimestampCount).toBe(1); + expect(index.intervals.get("i1")).toHaveLength(1); + }); }); // A stage window built straight from literals, so the idle tests do not depend @@ -463,10 +487,27 @@ describe("renderers — deterministic, header-first, hypothesis stated", () => { expect(head).toContain("orphan-complete: 0"); expect(head).toContain("unclosed-idle: 0"); expect(head).toContain("zero-second: 1"); + expect(head).toContain("invalid-timestamp: 0"); expect(head).toContain("unparseable-review-heading: 1"); expect(text).toContain(HYPOTHESIS_NOTICE); }); + test("a stage name with control bytes and a newline is reduced at the render point", () => { + const report = sampleReport(); + const hostile = { ...report, stages: [{ stage: "de\u0007sign\ninjected", n: 1, rawMedian: 1, netMean: 1, netMedian: 1, netP95: 1 }] }; + const markdown = renderMarkdown(hostile); + expect(markdown).toContain("| design |"); + expect(markdown).not.toContain("injected"); + expect(markdown).not.toContain("\u0007"); + }); + + test("a csv cell with a comma and a quote cannot forge an extra column", () => { + const report = sampleReport(); + const hostile = { ...report, stages: [{ stage: 'a,"b', n: 1, rawMedian: 1, netMean: 1, netMedian: 1, netP95: 1 }] }; + const row = renderCsv(hostile).split("\n").find((line) => line.includes('a,""b')); + expect(row).toBe('"a,""b",1,1,1,1,1'); + }); + test("csv also carries the measurement ref and the hypothesis before the rows", () => { const text = renderCsv(sampleReport()); const lines = text.split("\n");