diff --git a/packages/framework/core/tools/amadeus-stage-stats.ts b/packages/framework/core/tools/amadeus-stage-stats.ts new file mode 100644 index 000000000..9fbbf005f --- /dev/null +++ b/packages/framework/core/tools/amadeus-stage-stats.ts @@ -0,0 +1,968 @@ +// Stage performance statistics — the read-only aggregation CLI (#2405). +// +// Usage: bun amadeus-stage-stats.ts [--project-dir ] [--space ] +// [--format markdown|csv|json] [--json] +// +// One command derives the per-stage duration baseline from the audit shards +// (amadeus/spaces//intents/* /audit/*.jsonl) plus the §12a review +// iteration distribution from the record artefacts. Stage windows are +// STAGE_STARTED -> STAGE_COMPLETED pairs; the reported net duration subtracts +// the idle spans (approval waits, parks, session gaps) that make raw +// wall-clock unusable as a performance signal. +// +// net is an ESTIMATE. The header says so on every run: subtracting idle is a +// hypothesis about working time, not a measurement of it. +// +// Reading contract mirrored from the sibling amadeus-subagent-stats.ts: +// measurement ref first, absence recorded as absence (a row without Model is +// the UNKNOWN bucket, never dropped), and a shard that exists but cannot be +// read is fail-loud at the exit code. Nothing here writes. +// +// The two-generation journal normalization is NOT reimplemented here: schema +// v1 (event/fields) and v2 (eventName/attributes) both arrive through +// amadeus-journal.ts's exported reader and field accessor. This module +// deliberately does not import amadeus-lib.ts (the same dependency-direction +// ruling the sibling tool records), and it does not reference the test tree — +// the nearest-rank p95 is mirrored locally because the shipped surface must +// not reach into tests. + +import { type Dirent, readdirSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { type JournalRecord, journalRecordField, readJournalRecords } from "./amadeus-journal.ts"; + +// --- domain types ---------------------------------------------------------- + +/** One journal record together with the intent its shard path places it in. */ +export interface AttributedRecord { + readonly intent: string; + readonly record: JournalRecord; +} + +/** The scan phase's output: what was read, and what could not be. */ +export interface ScannedCorpus { + readonly records: readonly AttributedRecord[]; + readonly unreadableShardCount: number; + readonly brokenLineCount: number; + readonly shardCount: number; + readonly lineCount: number; +} + +/** A STAGE_STARTED -> STAGE_COMPLETED pair. Timestamps are second-grained. */ +export interface StageWindow { + readonly intent: string; + readonly stage: string; + readonly startedAt: string; + readonly completedAt: string; + readonly rawSeconds: number; +} + +/** A window with its idle span subtracted. Invariant: netSeconds >= 0. */ +export interface MeasuredWindow extends StageWindow { + readonly idleSeconds: number; + readonly netSeconds: number; +} + +// The closed set of exclusion buckets, layered by the three DISJOINT +// populations they count in. A flat record would invite the reader to add the +// seven numbers together, which is meaningless: corpus counts lines and shards +// that never became a window, review counts headings in a different data +// source entirely, and only two of the windowing buckets take part in the +// window identity. Every bucket is reported regardless (silence is not an +// option), and adding a bucket breaks the renderer at the type boundary. +export interface ExclusionCounts { + readonly corpus: { readonly brokenLine: number; readonly unreadableShard: number }; + readonly windowing: { + readonly unmatchedStart: number; + readonly orphanComplete: number; + readonly unclosedIdle: number; + readonly zeroSecond: number; + readonly invalidTimestamp: number; + }; + readonly review: { readonly unparseableReviewHeading: number }; +} + +/** All buckets at zero — the identity element the folds start from. */ +export function emptyExclusions(): ExclusionCounts { + return { + corpus: { brokenLine: 0, unreadableShard: 0 }, + windowing: { unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0, invalidTimestamp: 0 }, + review: { unparseableReviewHeading: 0 }, + }; +} + +/** One row of the per-stage duration table. */ +export interface StageStat { + readonly stage: string; + readonly n: number; + readonly rawMedian: number; + readonly netMean: number; + readonly netMedian: number; + readonly netP95: number; +} + +// --- window construction --------------------------------------------------- + +/** The audit event name of a record, under either schema generation. */ +export function eventOf(record: JournalRecord): string | null { + return journalRecordField(record, "Event"); +} + +/** A record's payload attribute, under either schema generation. */ +export function attrOf(record: JournalRecord, key: string): string | null { + return journalRecordField(record, key); +} + +// Second-grained epoch. The audit writer strips milliseconds, so window +// arithmetic is integer seconds and sub-second work collapses to zero — the +// zero-second bucket exists to say so rather than to hide it. +function epochSeconds(timestamp: string): number { + return Math.floor(Date.parse(timestamp) / 1000); +} + +// Timestamp-ascending, with the original index as the tie-break so records +// stamped in the same second keep their shard order (START before COMPLETED). +function chronological(records: readonly AttributedRecord[]): AttributedRecord[] { + return records + .map((entry, index) => ({ entry, index })) + .sort((a, b) => a.entry.record.timestamp.localeCompare(b.entry.record.timestamp) || a.index - b.index) + .map((wrapped) => wrapped.entry); +} + +/** Pair STAGE_STARTED with the next STAGE_COMPLETED inside the same + * intent x stage. Unpaired events on either side are counted as events, not + * as windows — they never enter the window population. */ +export function buildWindows(records: readonly AttributedRecord[]): { windows: StageWindow[]; buckets: ExclusionCounts } { + const pending = new Map(); + const windows: StageWindow[] = []; + let unmatchedStart = 0; + let orphanComplete = 0; + let invalidTimestamp = 0; + + for (const attributed of chronological(records)) { + const event = eventOf(attributed.record); + if (event !== "STAGE_STARTED" && event !== "STAGE_COMPLETED") continue; + const stage = attrOf(attributed.record, "Stage"); + if (stage === null || stage === "") continue; + if (Number.isNaN(epochSeconds(attributed.record.timestamp))) { + invalidTimestamp += 1; + continue; + } + const key = `${attributed.intent}\u0000${stage}`; + if (event === "STAGE_STARTED") { + const queue = pending.get(key); + if (queue === undefined) pending.set(key, [attributed]); + else queue.push(attributed); + continue; + } + const queue = pending.get(key); + const started = queue?.shift(); + if (started === undefined) { + orphanComplete += 1; + continue; + } + windows.push({ + intent: attributed.intent, + stage, + startedAt: started.record.timestamp, + completedAt: attributed.record.timestamp, + rawSeconds: epochSeconds(attributed.record.timestamp) - epochSeconds(started.record.timestamp), + }); + } + for (const queue of pending.values()) unmatchedStart += queue.length; + + const zero = emptyExclusions(); + return { windows, buckets: { ...zero, windowing: { ...zero.windowing, unmatchedStart, orphanComplete, invalidTimestamp } } }; +} + +// --- idle subtraction ------------------------------------------------------ + +/** One idle span, already closed by its terminating event. */ +export interface IdleInterval { + readonly kind: "awaiting" | "parked" | "session-gap"; + readonly start: number; + readonly end: number; +} + +/** Per-intent idle spans plus the timestamps of openers that never closed. + * Idle events whose timestamp does not parse are counted rather than allowed + * to poison the interval arithmetic with NaN. */ +export interface IdleIndex { + readonly intervals: ReadonlyMap; + readonly unclosedOpeners: ReadonlyMap; + readonly invalidTimestampCount: number; +} + +// The three idle shapes, as opener -> accepted closers. An opener that never +// meets its closer inside the intent is NOT given an implied terminal: guessing +// one would silently invent working time, so the affected window leaves the +// population instead. +const IDLE_SHAPES: readonly { kind: IdleInterval["kind"]; open: string; close: readonly string[] }[] = [ + { kind: "awaiting", open: "STAGE_AWAITING_APPROVAL", close: ["GATE_APPROVED", "GATE_REJECTED"] }, + { kind: "parked", open: "WORKFLOW_PARKED", close: ["WORKFLOW_UNPARKED"] }, + { kind: "session-gap", open: "SESSION_ENDED", close: ["SESSION_STARTED", "SESSION_RESUMED"] }, +]; + +// The open-opener stacks, keyed by intent and idle kind. Kept as its own type +// so the fold below reads as three small steps instead of one nested sweep. +type OpenerStacks = Map>; + +/** The stack of unmatched openers for one intent x kind, created on demand. */ +function openerStack(stacks: OpenerStacks, intent: string, kind: IdleInterval["kind"]): number[] { + const perKind = stacks.get(intent) ?? new Map(); + stacks.set(intent, perKind); + const stack = perKind.get(kind) ?? []; + perKind.set(kind, stack); + return stack; +} + +/** Append one closed span to an intent's interval list. */ +function pushInterval(intervals: Map, intent: string, span: IdleInterval): void { + const bucket = intervals.get(intent) ?? []; + intervals.set(intent, bucket); + bucket.push(span); +} + +/** The idle shape this event participates in, or null when it is not idle. */ +function idleShapeFor(event: string): (typeof IDLE_SHAPES)[number] | null { + return IDLE_SHAPES.find((shape) => event === shape.open || shape.close.includes(event)) ?? null; +} + +/** Build the per-intent idle index from the corpus. Pure over the records. */ +export function indexIdle(records: readonly AttributedRecord[]): IdleIndex { + const intervals = new Map(); + const unclosedOpeners = new Map(); + const openPerIntent: OpenerStacks = new Map(); + let invalidTimestampCount = 0; + + for (const attributed of chronological(records)) { + const event = eventOf(attributed.record); + const shape = event === null ? null : idleShapeFor(event); + if (shape === null) continue; + const at = epochSeconds(attributed.record.timestamp); + if (Number.isNaN(at)) { + invalidTimestampCount += 1; + continue; + } + const stack = openerStack(openPerIntent, attributed.intent, shape.kind); + if (event === shape.open) { + stack.push(at); + continue; + } + const start = stack.shift(); + if (start !== undefined) pushInterval(intervals, attributed.intent, { kind: shape.kind, start, end: at }); + } + + for (const [intent, perKind] of openPerIntent) { + const leftovers: number[] = []; + for (const stack of perKind.values()) leftovers.push(...stack); + if (leftovers.length > 0) unclosedOpeners.set(intent, leftovers.sort((a, b) => a - b)); + } + return { intervals, unclosedOpeners, invalidTimestampCount }; +} + +// Clip each span to the window, then merge the survivors so overlapping spans +// are subtracted once. Clipping is what keeps net non-negative: the merged +// length can never exceed the window it was clipped into. +function clippedIdleSeconds(window: StageWindow, spans: readonly IdleInterval[]): number { + const from = epochSeconds(window.startedAt); + const to = epochSeconds(window.completedAt); + const clipped = spans + .map((span) => ({ start: Math.max(span.start, from), end: Math.min(span.end, to) })) + .filter((span) => span.end > span.start) + .sort((a, b) => a.start - b.start); + + let total = 0; + let cursor = from; + for (const span of clipped) { + const start = Math.max(span.start, cursor); + if (span.end > start) { + total += span.end - start; + cursor = span.end; + } + } + return total; +} + +/** Subtract idle from every window. A window is dropped from the population + * when an unclosed idle opener falls inside it (its idle is unknowable) or + * when it collapsed to zero seconds. The two are decided in that order, so a + * window that satisfies both is counted exactly once. */ +export function subtractIdle( + windows: readonly StageWindow[], + records: readonly AttributedRecord[], +): { measured: MeasuredWindow[]; buckets: ExclusionCounts } { + const index = indexIdle(records); + const measured: MeasuredWindow[] = []; + let unclosedIdle = 0; + let zeroSecond = 0; + + for (const window of windows) { + const from = epochSeconds(window.startedAt); + const to = epochSeconds(window.completedAt); + const openers = index.unclosedOpeners.get(window.intent) ?? []; + const shadowed = openers.some((at) => at >= from && at <= to); + if (shadowed) { + unclosedIdle += 1; + continue; + } + if (window.rawSeconds === 0) { + zeroSecond += 1; + continue; + } + const idleSeconds = clippedIdleSeconds(window, index.intervals.get(window.intent) ?? []); + measured.push({ ...window, idleSeconds, netSeconds: window.rawSeconds - idleSeconds }); + } + + const zero = emptyExclusions(); + return { + measured, + buckets: { ...zero, windowing: { ...zero.windowing, unclosedIdle, zeroSecond, invalidTimestamp: index.invalidTimestampCount } }, + }; +} + +// --- review iterations ----------------------------------------------------- + +/** One §12a review block located in a record artefact. */ +export interface ReviewBlock { + readonly intent: string; + readonly stagePath: string; + readonly unit: string | null; + readonly iteration: number; +} + +// The reviewer runtime finds its own blocks with a permissive H2 scan followed +// by an exact marker comparison. The same two stages are mirrored here so a +// heading the writer would not recognize is surfaced as unparseable rather +// than quietly matched by a looser reader. +const REVIEW_HEADING = /^## Review(?:[ \t].*)?$/gm; +const REVIEW_MARKER = /^## Review — Iteration (\d+)$/; + +/** Iterations found in one artefact's text, plus headings the writer's exact + * marker would not accept (suffixed variants are real and stay visible). */ +export function parseReviewHeadings(content: string): { iterations: number[]; unparseable: number } { + const iterations: number[] = []; + let unparseable = 0; + for (const heading of content.matchAll(REVIEW_HEADING)) { + const marker = REVIEW_MARKER.exec(heading[0].trim()); + if (marker === null) { + unparseable += 1; + continue; + } + iterations.push(Number(marker[1])); + } + return { iterations, unparseable }; +} + +/** Stage path and unit for an artefact, taken from its intent-relative path. + * A literal `{unit-name}` directory is a real thing on disk; it is displayed + * verbatim rather than repaired into an invented unit name. */ +export function reviewAttribution(relativePath: string): { stagePath: string; unit: string | null } { + const segments = relativePath.split("/"); + const stagePath = segments.slice(0, -1).join("/"); + const unit = segments[0] === "construction" && segments.length >= 4 ? (segments[1] ?? null) : null; + return { stagePath, unit }; +} + +// --- sensors --------------------------------------------------------------- + +/** Sensor outcomes for one stage slug. */ +export interface SensorTally { + readonly stageSlug: string; + readonly fired: number; + readonly passed: number; + readonly failed: number; + readonly failedRate: number; +} + +// A Map, not an object literal: the corpus is outside the trust boundary, so an +// event name like "constructor" must miss instead of hitting Object.prototype. +const SENSOR_EVENTS: ReadonlyMap = new Map([ + ["SENSOR_FIRED", "fired"], + ["SENSOR_PASSED", "passed"], + ["SENSOR_FAILED", "failed"], +]); + +/** Tally sensor outcomes by the `Stage slug` attribute. The lifecycle events' + * `Stage` attribute is a different key with different values and is never read + * here — mixing them would merge unrelated rows. */ +export function tallySensors(records: readonly AttributedRecord[]): SensorTally[] { + const counters = new Map(); + for (const { record } of records) { + const event = eventOf(record); + const outcome = event === null ? undefined : SENSOR_EVENTS.get(event); + if (outcome === undefined) continue; + const slug = attrOf(record, "Stage slug"); + if (slug === null || slug === "") continue; + const counter = counters.get(slug) ?? { fired: 0, passed: 0, failed: 0 }; + counters.set(slug, counter); + counter[outcome] += 1; + } + return [...counters.entries()] + .map(([stageSlug, c]) => ({ stageSlug, ...c, failedRate: c.fired === 0 ? Number.NaN : c.failed / c.fired })) + .sort((a, b) => b.fired - a.fired || a.stageSlug.localeCompare(b.stageSlug)); +} + +// --- model attribution ----------------------------------------------------- + +/** Model breakdown of the subagent rows, with the unattributable ones kept. */ +export interface ModelAttribution { + readonly byModel: ReadonlyMap; + readonly byModelSource: ReadonlyMap; + readonly unresolvedCount: number; + readonly attributableCount: number; + readonly totalCount: number; +} + +/** Attribute subagent rows to models. Only SUBAGENT_COMPLETED rows join the + * population — a start and its completion describe one dispatch, and the + * sibling stats tool attributes on completion, so counting both would double + * every per-model total. A row without a usable `Model` is the UNKNOWN + * bucket: the absence of the attribute is itself the record, so it is + * reported rather than dropped from the denominator. */ +export function attributeModels(records: readonly AttributedRecord[]): ModelAttribution { + const byModel = new Map(); + const byModelSource = new Map(); + let unresolvedCount = 0; + let attributableCount = 0; + let totalCount = 0; + + for (const { record } of records) { + const event = eventOf(record); + if (event !== "SUBAGENT_COMPLETED") continue; + totalCount += 1; + const model = attrOf(record, "Model"); + if (model === null || model.trim() === "") { + unresolvedCount += 1; + continue; + } + attributableCount += 1; + byModel.set(model, (byModel.get(model) ?? 0) + 1); + const source = attrOf(record, "Model Source"); + if (source !== null && source.trim() !== "") byModelSource.set(source, (byModelSource.get(source) ?? 0) + 1); + } + return { byModel, byModelSource, unresolvedCount, attributableCount, totalCount }; +} + +// --- statistics ------------------------------------------------------------ + +// Nearest rank, no interpolation: the smallest sample whose rank covers 95% of +// the set. An empty set yields NaN rather than 0 so a hole in the measurement +// stays visibly non-finite instead of reading as a real zero. +export function nearestRankP95(values: readonly number[]): number { + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.ceil(sorted.length * 0.95) - 1] ?? Number.NaN; +} + +/** Arithmetic mean; NaN on an empty population (no 0 fallback). */ +export function mean(values: readonly number[]): number { + if (values.length === 0) return Number.NaN; + return values.reduce((a, b) => a + b, 0) / values.length; +} + +/** Median (mean of the two middle samples on an even count); NaN when empty. */ +export function median(values: readonly number[]): number { + if (values.length === 0) return Number.NaN; + const sorted = [...values].sort((left, right) => left - right); + const mid = sorted.length >> 1; + return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +/** Fold measured windows into the per-stage table. Ordering: count-desc, then + * stage-asc, so the same population always renders in the same order. */ +export function composeStageStats(measured: readonly MeasuredWindow[]): StageStat[] { + const byStage = new Map(); + for (const window of measured) { + const bucket = byStage.get(window.stage); + if (bucket === undefined) byStage.set(window.stage, [window]); + else bucket.push(window); + } + const rows: StageStat[] = []; + for (const [stage, windows] of byStage) { + const net = windows.map((w) => w.netSeconds); + rows.push({ + stage, + n: windows.length, + rawMedian: median(windows.map((w) => w.rawSeconds)), + netMean: mean(net), + netMedian: median(net), + netP95: nearestRankP95(net), + }); + } + return rows.sort((a, b) => b.n - a.n || a.stage.localeCompare(b.stage)); +} + +// --- report assembly ------------------------------------------------------- + +// The one sentence this tool must never ship without. net is derived by +// subtracting idle spans; nobody has verified that the remainder equals the +// time someone actually spent working, so the report says so where the reader +// cannot miss it. +export const HYPOTHESIS_NOTICE = + "net (working time) is an estimate produced by subtracting idle spans; its agreement with real working time is an unverified hypothesis."; + +/** Review blocks folded per stage path. */ +export interface ReviewBucket { + readonly stagePath: string; + readonly unit: string | null; + readonly blocks: number; + readonly maxIteration: number; +} + +/** Everything the renderers are allowed to draw from. The measurement ref and + * the hypothesis notice are required fields, so an output that omits them is + * not expressible. */ +export interface StageStatsReport { + readonly scanScope: string; + readonly hypothesisNotice: string; + readonly shardCount: number; + readonly lineCount: number; + readonly exclusions: ExclusionCounts; + readonly constructedWindowCount: number; + readonly populationCount: number; + readonly stages: readonly StageStat[]; + readonly sensors: readonly SensorTally[]; + readonly models: ModelAttribution; + readonly reviewBuckets: readonly ReviewBucket[]; +} + +/** Fold review blocks into per-stage-path buckets, count-desc then path-asc. */ +export function bucketReviews(blocks: readonly ReviewBlock[]): ReviewBucket[] { + const buckets = new Map(); + for (const block of blocks) { + const bucket = buckets.get(block.stagePath) ?? { unit: block.unit, blocks: 0, maxIteration: 0 }; + buckets.set(block.stagePath, { + unit: bucket.unit, + blocks: bucket.blocks + 1, + maxIteration: Math.max(bucket.maxIteration, block.iteration), + }); + } + return [...buckets.entries()] + .map(([stagePath, b]) => ({ stagePath, ...b })) + .sort((a, b) => b.blocks - a.blocks || a.stagePath.localeCompare(b.stagePath)); +} + +/** Assemble the whole report from a scanned corpus and the review blocks. + * Pure over its input: the same corpus always produces the same report. */ +export function composeReport(input: { + scanScope: string; + corpus: ScannedCorpus; + reviewBlocks: readonly ReviewBlock[]; + unparseableReviewHeadingCount: number; +}): StageStatsReport { + const built = buildWindows(input.corpus.records); + const subtracted = subtractIdle(built.windows, input.corpus.records); + return { + scanScope: input.scanScope, + hypothesisNotice: HYPOTHESIS_NOTICE, + shardCount: input.corpus.shardCount, + lineCount: input.corpus.lineCount, + exclusions: { + corpus: { brokenLine: input.corpus.brokenLineCount, unreadableShard: input.corpus.unreadableShardCount }, + windowing: { + unmatchedStart: built.buckets.windowing.unmatchedStart, + orphanComplete: built.buckets.windowing.orphanComplete, + unclosedIdle: subtracted.buckets.windowing.unclosedIdle, + zeroSecond: subtracted.buckets.windowing.zeroSecond, + invalidTimestamp: built.buckets.windowing.invalidTimestamp + subtracted.buckets.windowing.invalidTimestamp, + }, + review: { unparseableReviewHeading: input.unparseableReviewHeadingCount }, + }, + constructedWindowCount: built.windows.length, + populationCount: subtracted.measured.length, + stages: composeStageStats(subtracted.measured), + sensors: tallySensors(input.corpus.records), + models: attributeModels(input.corpus.records), + reviewBuckets: bucketReviews(input.reviewBlocks), + }; +} + +// --- rendering ------------------------------------------------------------- + +// Values from the corpus reach a terminal here. The corpus is outside the +// trust boundary, so a stage slug or model name is reduced to its first line +// and stripped of control bytes at the render point only — the aggregation +// keys stay verbatim, and the JSON form keeps the raw value because its +// encoding already escapes control bytes and machine consumers need it whole. +const FIRST_PRINTABLE = 0x20; +const DELETE_CODE = 0x7f; + +function safe(value: string): string { + const firstLine = value.split("\n", 1)[0] ?? ""; + let out = ""; + for (const char of firstLine) { + const code = char.codePointAt(0) ?? 0; + if (code >= FIRST_PRINTABLE && code !== DELETE_CODE) out += char; + } + return out; +} + +// A number the reader must be able to tell apart from a real zero. +function num(value: number): string { + return Number.isNaN(value) ? "n/a" : `${Math.round(value * 100) / 100}`; +} + +/** The measurement ref lines, shared by every output form: what was scanned, + * and every exclusion bucket, whatever its value. */ +function measurementRefLines(report: StageStatsReport): string[] { + const { corpus, windowing, review } = report.exclusions; + return [ + `scan scope: ${safe(report.scanScope)}`, + `shards: ${report.shardCount}`, + `lines: ${report.lineCount}`, + `broken-line: ${corpus.brokenLine}`, + `unreadable-shard: ${corpus.unreadableShard}`, + `unmatched-start: ${windowing.unmatchedStart}`, + `orphan-complete: ${windowing.orphanComplete}`, + `unclosed-idle: ${windowing.unclosedIdle}`, + `zero-second: ${windowing.zeroSecond}`, + `invalid-timestamp: ${windowing.invalidTimestamp}`, + `unparseable-review-heading: ${review.unparseableReviewHeading}`, + `constructed-windows: ${report.constructedWindowCount}`, + `net-population: ${report.populationCount}`, + ]; +} + +/** Sorted entries of a tally map: count-desc, then key-asc. */ +function sortedEntries(tally: ReadonlyMap): [string, number][] { + return [...tally.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); +} + +/** The human-readable form. */ +export function renderMarkdown(report: StageStatsReport): string { + const lines: string[] = ["# stage-stats — measurement ref", ""]; + for (const line of measurementRefLines(report)) lines.push(`- ${line}`); + lines.push( + "", + `> ${HYPOTHESIS_NOTICE}`, + "", + `Identity W: ${report.constructedWindowCount} constructed = ${report.populationCount} population + ${report.exclusions.windowing.unclosedIdle} unclosed-idle + ${report.exclusions.windowing.zeroSecond} zero-second`, + `Identity M: ${report.models.totalCount} total = ${report.models.attributableCount} attributable + ${report.models.unresolvedCount} unresolved`, + "", + "## Stage durations (seconds)", + "", + "| stage | n | raw median | net mean | net median | net p95 |", + "| --- | --- | --- | --- | --- | --- |", + ); + if (report.stages.length === 0) lines.push("| (none) | 0 | n/a | n/a | n/a | n/a |"); + for (const stat of report.stages) { + lines.push(`| ${safe(stat.stage)} | ${stat.n} | ${num(stat.rawMedian)} | ${num(stat.netMean)} | ${num(stat.netMedian)} | ${num(stat.netP95)} |`); + } + + lines.push("", "## Sensor outcomes", "", "| stage slug | fired | passed | failed | failed rate |", "| --- | --- | --- | --- | --- |"); + if (report.sensors.length === 0) lines.push("| (none) | 0 | 0 | 0 | n/a |"); + for (const row of report.sensors) { + lines.push(`| ${safe(row.stageSlug)} | ${row.fired} | ${row.passed} | ${row.failed} | ${num(row.failedRate)} |`); + } + + lines.push("", "## Model attribution", "", `- attributable: ${report.models.attributableCount} / ${report.models.totalCount}`, `- unresolved (UNKNOWN): ${report.models.unresolvedCount}`); + for (const [model, count] of sortedEntries(report.models.byModel)) lines.push(`- model ${safe(model)}: ${count}`); + for (const [source, count] of sortedEntries(report.models.byModelSource)) lines.push(`- model-source ${safe(source)}: ${count}`); + + lines.push("", "## Review iterations", "", "| stage path | unit | blocks | max iteration |", "| --- | --- | --- | --- |"); + if (report.reviewBuckets.length === 0) lines.push("| (none) | | 0 | 0 |"); + for (const bucket of report.reviewBuckets) { + lines.push(`| ${safe(bucket.stagePath)} | ${safe(bucket.unit ?? "")} | ${bucket.blocks} | ${bucket.maxIteration} |`); + } + return `${lines.join("\n")}\n`; +} + +// CSV quoting: double the quotes and wrap, so a value containing a comma or a +// quote cannot forge an extra column. +function csvCell(value: string): string { + return `"${safe(value).replace(/"/g, '""')}"`; +} + +/** The spreadsheet form: the same measurement ref, then one section per axis. */ +export function renderCsv(report: StageStatsReport): string { + const lines: string[] = ["section,key,value"]; + lines.push(`meta,${csvCell("stage-stats")},${csvCell(report.scanScope)}`); + lines.push(`meta,${csvCell("hypothesis")},${csvCell(HYPOTHESIS_NOTICE)}`); + for (const line of measurementRefLines(report)) { + const split = line.indexOf(":"); + lines.push(`ref,${line.slice(0, split)},${csvCell(line.slice(split + 2))}`); + } + lines.push("", "stage,n,raw_median,net_mean,net_median,net_p95"); + for (const stat of report.stages) { + lines.push(`${csvCell(stat.stage)},${stat.n},${num(stat.rawMedian)},${num(stat.netMean)},${num(stat.netMedian)},${num(stat.netP95)}`); + } + lines.push("", "sensor_stage_slug,fired,passed,failed,failed_rate"); + for (const row of report.sensors) lines.push(`${csvCell(row.stageSlug)},${row.fired},${row.passed},${row.failed},${num(row.failedRate)}`); + lines.push("", "model,count"); + for (const [model, count] of sortedEntries(report.models.byModel)) lines.push(`${csvCell(model)},${count}`); + lines.push(`${csvCell("(unresolved)")},${report.models.unresolvedCount}`); + lines.push("", "review_stage_path,unit,blocks,max_iteration"); + for (const bucket of report.reviewBuckets) { + lines.push(`${csvCell(bucket.stagePath)},${csvCell(bucket.unit ?? "")},${bucket.blocks},${bucket.maxIteration}`); + } + return `${lines.join("\n")}\n`; +} + +/** The machine form. Maps become arrays in the same fixed order the text forms + * use, so a consumer sees one ordering contract, not two. */ +export function serializeJson(report: StageStatsReport): Record { + return { + scanScope: report.scanScope, + hypothesisNotice: report.hypothesisNotice, + shardCount: report.shardCount, + lineCount: report.lineCount, + exclusions: report.exclusions, + constructedWindowCount: report.constructedWindowCount, + populationCount: report.populationCount, + stages: report.stages, + sensors: report.sensors, + models: { + byModel: sortedEntries(report.models.byModel).map(([model, count]) => ({ model, count })), + byModelSource: sortedEntries(report.models.byModelSource).map(([source, count]) => ({ source, count })), + unresolvedCount: report.models.unresolvedCount, + attributableCount: report.models.attributableCount, + totalCount: report.models.totalCount, + }, + reviewBuckets: report.reviewBuckets, + }; +} + +// --- argv ------------------------------------------------------------------ + +/** Command line options, already validated into their narrow shapes. */ +export interface CliOptions { + readonly projectDir?: string; + readonly space?: string; + readonly format: "markdown" | "csv" | "json"; +} + +/** The one thing a usage error carries. */ +export interface UsageError { + readonly message: string; +} + +/** A parse outcome; the shell is the only place it becomes an exit code. */ +export type Result = { readonly ok: true; readonly value: T } | { readonly ok: false; readonly error: E }; + +const USAGE = + "Usage: bun amadeus-stage-stats.ts [--project-dir ] [--space ] [--format markdown|csv|json] [--json]"; + +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const FORMATS: readonly CliOptions["format"][] = ["markdown", "csv", "json"]; + +function isFormat(value: string): value is CliOptions["format"] { + return (FORMATS as readonly string[]).includes(value); +} + +const VALUE_FLAGS = ["--project-dir", "--space", "--format"] as const; +type ValueFlag = (typeof VALUE_FLAGS)[number]; + +function isValueFlag(arg: string): arg is ValueFlag { + return (VALUE_FLAGS as readonly string[]).includes(arg); +} + +function usageError(message: string): { ok: false; error: UsageError } { + return { ok: false, error: { message: `${message}\n${USAGE}` } }; +} + +/** Apply one already-paired flag/value to the accumulating options, or refuse + * the value. Keeping the per-flag rules here leaves the loop a plain walk. */ +function applyValueFlag(flag: ValueFlag, value: string, into: { projectDir?: string; space?: string; format: CliOptions["format"] }): UsageError | null { + if (flag === "--project-dir") { + into.projectDir = value; + return null; + } + if (flag === "--space") { + if (!SAFE_NAME.test(value)) return { message: `Invalid --space value: ${value}\n${USAGE}` }; + into.space = value; + return null; + } + if (!isFormat(value)) return { message: `Invalid --format value: ${value}\n${USAGE}` }; + into.format = value; + return null; +} + +/** Parse, do not validate: an unknown flag, a missing value, an unsupported + * format, or an unsafe space name never reaches the rest of the program. */ +export function parseArgs(argv: readonly string[]): Result { + const options: { projectDir?: string; space?: string; format: CliOptions["format"] } = { format: "markdown" }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] as string; + if (arg === "--json") { + options.format = "json"; + continue; + } + if (!isValueFlag(arg)) return usageError(`Unknown argument: ${arg}`); + const value = argv[i + 1]; + if (value === undefined) return usageError(`${arg} requires a value`); + i += 1; + const rejected = applyValueFlag(arg, value, options); + if (rejected !== null) return { ok: false, error: rejected }; + } + return { ok: true, value: options }; +} + +// --------------------------------------------------------------------------- +// Filesystem layer + CLI shell — the only process-boundary code in this module. +// --------------------------------------------------------------------------- + +/** Directory entries, or none when the directory is absent or unreadable. An + * absent tree and a tree that vanishes mid-scan are the same thing here: an + * empty corpus. The fail-loud case is a shard that EXISTS but cannot be read. */ +function listOrEmpty(dir: string): string[] { + try { + return readdirSync(dir).sort(); + } catch { + return []; + } +} + +/** One shard's bytes, or null when it exists but cannot be read. */ +function readOrNull(path: string): string | null { + try { + return readFileSync(path, "utf-8"); + } catch (e) { + const reason = e instanceof Error ? e.message : String(e); + const detail = safe(`${path}: ${reason}`); + process.stderr.write(`stage-stats: unreadable: ${detail}\n`); + return null; + } +} + +// Records in one shard, counting lines that do not decode. The journal reader +// is strict per buffer, so it is fed one line at a time: a single torn row then +// costs that row and nothing else, and the count of torn rows is reported. +function recordsFromShard(intent: string, body: string): { records: AttributedRecord[]; lines: number; broken: number } { + const records: AttributedRecord[] = []; + let lines = 0; + let broken = 0; + for (const line of body.split("\n")) { + if (line.trim() === "") continue; + lines += 1; + try { + for (const record of readJournalRecords(line)) records.push({ intent, record }); + } catch { broken += 1; continue; } + } + return { records, lines, broken }; +} + +/** Read every audit shard under the space's intents tree. The intent an event + * belongs to comes from the shard's PATH, never from the recorded intentId — + * most historical v1 rows carry a degenerate value there. */ +export function scanCorpus(spaceRoot: string): ScannedCorpus { + const records: AttributedRecord[] = []; + let unreadableShardCount = 0; + let brokenLineCount = 0; + let shardCount = 0; + let lineCount = 0; + + const intentsRoot = join(spaceRoot, "intents"); + for (const intent of listOrEmpty(intentsRoot)) { + const auditDir = join(intentsRoot, intent, "audit"); + for (const shard of listOrEmpty(auditDir)) { + if (!shard.endsWith(".jsonl")) continue; + const body = readOrNull(join(auditDir, shard)); + if (body === null) { + unreadableShardCount += 1; + continue; + } + shardCount += 1; + const read = recordsFromShard(intent, body); + records.push(...read.records); + lineCount += read.lines; + brokenLineCount += read.broken; + } + } + return { records, unreadableShardCount, brokenLineCount, shardCount, lineCount }; +} + +/** Markdown artefacts under one intent, as intent-relative paths. Dirent-typed + * so only real directories recurse — a directory symlink pointing back at an + * ancestor would otherwise recurse forever. */ +function markdownUnder(root: string, prefix: string, out: string[]): void { + let entries: Dirent[]; + try { + entries = readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + } catch { return; } + for (const entry of entries) { + if (entry.name === "audit" || entry.name === "node_modules") continue; + const relative = prefix === "" ? entry.name : `${prefix}/${entry.name}`; + if (entry.isFile() && entry.name.endsWith(".md")) out.push(relative); + else if (entry.isDirectory()) markdownUnder(join(root, entry.name), relative, out); + } +} + +/** Collect every §12a review block from the record artefacts. This is the + * record tree, not the audit tree: iteration counts live in the artefacts the + * reviewer appended to, and nowhere else. */ +export function collectReviewBlocks(intentsRoot: string): { blocks: ReviewBlock[]; unparseableHeadingCount: number } { + const blocks: ReviewBlock[] = []; + let unparseableHeadingCount = 0; + for (const intent of listOrEmpty(intentsRoot)) { + const paths: string[] = []; + markdownUnder(join(intentsRoot, intent), "", paths); + for (const relative of paths) { + const body = readOrNull(join(intentsRoot, intent, relative)); + if (body === null) continue; + const parsed = parseReviewHeadings(body); + unparseableHeadingCount += parsed.unparseable; + const attribution = reviewAttribution(relative); + for (const iteration of parsed.iterations) { + blocks.push({ intent, stagePath: attribution.stagePath, unit: attribution.unit, iteration }); + } + } + } + return { blocks, unparseableHeadingCount }; +} + +// The two path idioms below mirror the sibling stats tool, reimplemented small +// because this module must not import amadeus-lib.ts. + +function resolveProjectDirLocal(explicitDir: string | undefined): string { + if (explicitDir) return explicitDir; + if (process.env.CLAUDE_PROJECT_DIR) return process.env.CLAUDE_PROJECT_DIR; + const scriptDir = dirname(fileURLToPath(import.meta.url)); + if (basename(scriptDir) === "tools") { + const leaf = basename(dirname(scriptDir)); + if (/^\.[a-z0-9][a-z0-9._-]*$/i.test(leaf)) return dirname(dirname(scriptDir)); + } + return process.cwd(); +} + +function activeSpaceLocal(projectDir: string): string { + let raw = ""; + try { + raw = readFileSync(join(projectDir, "amadeus", "active-space"), "utf-8").trim(); + } catch { return "default"; } + return SAFE_NAME.test(raw) ? raw : "default"; +} + +const RENDERERS: Record string> = { + markdown: renderMarkdown, + csv: renderCsv, + json: (report) => JSON.stringify(serializeJson(report), null, 2), +}; + +/** 0 normal, 1 the corpus had a hole, 2 the caller made a usage error. */ +export function main(argv: readonly string[]): number { + const parsed = parseArgs(argv); + if (!parsed.ok) { + process.stderr.write(`${parsed.error.message}\n`); + return 2; + } + const projectDir = resolveProjectDirLocal(parsed.value.projectDir); + const space = parsed.value.space ?? activeSpaceLocal(projectDir); + const spaceRoot = join(projectDir, "amadeus", "spaces", space); + + const corpus = scanCorpus(spaceRoot); + const reviews = collectReviewBlocks(join(spaceRoot, "intents")); + const report = composeReport({ + scanScope: `space "${space}" — amadeus/spaces/${space}/intents/*/audit/*.jsonl + record *.md`, + corpus, + reviewBlocks: reviews.blocks, + unparseableReviewHeadingCount: reviews.unparseableHeadingCount, + }); + process.stdout.write(`${RENDERERS[parsed.value.format](report)}`); + + // A shard that existed but could not be read means the report covers less + // than the observable corpus — say so in the exit code so a caller does not + // mistake a partial sweep for a complete one. + return corpus.unreadableShardCount > 0 ? 1 : 0; +} + +if (import.meta.main) process.exit(main(process.argv.slice(2))); diff --git a/tests/.coverage-registry.json b/tests/.coverage-registry.json index 1cb47b9dd..fed386a3c 100644 --- a/tests/.coverage-registry.json +++ b/tests/.coverage-registry.json @@ -2769,6 +2769,10 @@ "file": "tests/integration/t420-unchecked-cast-guard-cli.test.ts", "mechanism": "none" }, + { + "file": "tests/integration/t487-stage-stats.integration.test.ts", + "mechanism": "cli" + }, { "file": "tests/unit/t-graph-dispatch-seam.test.ts", "mechanism": "cli" diff --git a/tests/integration/t-coverage-mechanism-ratchet.test.ts b/tests/integration/t-coverage-mechanism-ratchet.test.ts index a7d62b184..38941ca07 100644 --- a/tests/integration/t-coverage-mechanism-ratchet.test.ts +++ b/tests/integration/t-coverage-mechanism-ratchet.test.ts @@ -98,6 +98,7 @@ describe("repository-wide mechanism honesty ratchets", () => { "integration/t481-autonomy-canonical-state-write.integration.test.ts", "integration/t482-autonomy-refusal-event.integration.test.ts", "integration/t483-preview-non-auto-kinds.integration.test.ts", + "integration/t487-stage-stats.integration.test.ts", "e2e/t-formal-verif-model-completeness-sensor.test.ts", "e2e/t237-election-walking-skeleton.test.ts", "e2e/t265-engine-boundary.test.ts", diff --git a/tests/integration/t487-stage-stats.integration.test.ts b/tests/integration/t487-stage-stats.integration.test.ts new file mode 100644 index 000000000..ab46a81ac --- /dev/null +++ b/tests/integration/t487-stage-stats.integration.test.ts @@ -0,0 +1,457 @@ +// covers: function:scanCorpus, function:collectReviewBlocks, function:main +// size: medium +// +// t487 — Issue #2405 (U1 stage-stats). Integration half of the twin: the real +// filesystem scan, the CLI shell driven in-process (so bun --coverage measures +// it), and the exit ladder measured by an actual spawn. The pure aggregation +// core is t486's job. +// +// Every expectation is checked against an INDEPENDENT ORACLE — a walker +// written here in the test that counts shards, lines and events on its own — +// so a defect in the scanner cannot cancel out against the expectation. +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { collectReviewBlocks, composeReport, main, scanCorpus } from "../../packages/framework/core/tools/amadeus-stage-stats.ts"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const TOOL = join(REPO_ROOT, "packages", "framework", "core", "tools", "amadeus-stage-stats.ts"); + +const scratchDirs: string[] = []; +afterEach(() => { + while (scratchDirs.length > 0) rmSync(scratchDirs.pop()!, { recursive: true, force: true }); +}); + +function scratch(): string { + const dir = mkdtempSync(join(tmpdir(), "amadeus-stage-stats-")); + scratchDirs.push(dir); + return dir; +} + +// --- fixture builders ------------------------------------------------------- + +function v1Line(seq: number, event: string, timestamp: string, fields: Record = {}): string { + return JSON.stringify({ schemaVersion: 1, seq, cloneId: "c1", intentId: "intents", timestamp, heading: event, event, fields }); +} + +function v2Line(seq: number, event: string, timestamp: string, attrs: Record = {}): string { + return JSON.stringify({ + schemaVersion: 2, + eventId: `e${seq}`, + seq, + timestamp, + eventName: "amadeus.stage", + attributes: { Event: event, ...attrs }, + intentId: "intents", + space: "default", + cloneId: "c2", + traceId: null, + spanId: null, + traceFlags: 0, + idempotencyKey: `k${seq}`, + canonical: true, + }); +} + +function writeShard(spaceRoot: string, intent: string, name: string, lines: readonly string[]): string { + const dir = join(spaceRoot, "intents", intent, "audit"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, `${lines.join("\n")}\n`); + return path; +} + +function writeArtefact(spaceRoot: string, intent: string, relative: string, body: string): void { + const path = join(spaceRoot, "intents", intent, relative); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, body); +} + +// A corpus mixing both journal generations, a torn line, a degenerate +// intentId, an unmatched start, a zero-second window, an unclosed park, and +// review headings both canonical and suffixed. +function buildCorpus(projectDir: string): string { + const spaceRoot = join(projectDir, "amadeus", "spaces", "default"); + writeShard(spaceRoot, "alpha", "shard-a.jsonl", [ + v1Line(1, "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1Line(2, "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1Line(3, "GATE_APPROVED", "2026-01-01T00:00:50Z"), + v2Line(4, "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + "{ this line is not json", + v2Line(5, "SENSOR_FIRED", "2026-01-01T00:00:05Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2Line(6, "SENSOR_FAILED", "2026-01-01T00:00:06Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2Line(7, "SUBAGENT_COMPLETED", "2026-01-01T00:00:07Z", { Model: "opus", "Model Source": "pin" }), + v1Line(8, "SUBAGENT_COMPLETED", "2026-01-01T00:00:08Z"), + ]); + writeShard(spaceRoot, "beta", "shard-b.jsonl", [ + v1Line(1, "STAGE_STARTED", "2026-01-01T01:00:00Z", { Stage: "build" }), + v1Line(2, "STAGE_COMPLETED", "2026-01-01T01:00:00Z", { Stage: "build" }), + v1Line(3, "STAGE_STARTED", "2026-01-01T02:00:00Z", { Stage: "never-finished" }), + v1Line(4, "STAGE_STARTED", "2026-01-01T03:00:00Z", { Stage: "parked-forever" }), + v1Line(5, "WORKFLOW_PARKED", "2026-01-01T03:00:10Z"), + v1Line(6, "STAGE_COMPLETED", "2026-01-01T03:01:00Z", { Stage: "parked-forever" }), + ]); + writeArtefact(spaceRoot, "alpha", "construction/unit-x/functional-design/business-rules.md", "## Review — Iteration 1\nbody\n## Review — Iteration 2\n"); + writeArtefact(spaceRoot, "alpha", "construction/{unit-name}/code-generation/code-summary.md", "## Review — Iteration 1\n"); + writeArtefact(spaceRoot, "beta", "inception/requirements-analysis/requirements.md", "## Review — Iteration 1 (follow-up)\n"); + return spaceRoot; +} + +// --- independent oracles ---------------------------------------------------- + +// A walker written from scratch: it counts shards, non-empty lines, and torn +// lines without calling any function under test. +function oracleEventName(line: string): string { + const parsed = JSON.parse(line) as Record; + return parsed.schemaVersion === 2 ? String((parsed.attributes as Record).Event) : String(parsed.event); +} + +function oracleShardPaths(spaceRoot: string): string[] { + const intentsRoot = join(spaceRoot, "intents"); + const paths: string[] = []; + for (const intent of readdirSync(intentsRoot).sort()) { + const auditDir = join(intentsRoot, intent, "audit"); + let entries: string[] = []; + try { + entries = readdirSync(auditDir).sort(); + } catch { + entries = []; + } + for (const shard of entries) if (shard.endsWith(".jsonl")) paths.push(join(auditDir, shard)); + } + return paths; +} + +function oracleScan(spaceRoot: string): { shards: number; lines: number; broken: number; events: Record } { + let shards = 0; + let lines = 0; + let broken = 0; + const events: Record = {}; + for (const path of oracleShardPaths(spaceRoot)) { + let body: string; + try { + body = readFileSync(path, "utf-8"); + } catch { + continue; + } + shards += 1; + for (const line of body.split("\n")) { + if (line.trim() === "") continue; + lines += 1; + try { + const name = oracleEventName(line); + events[name] = (events[name] ?? 0) + 1; + } catch { + broken += 1; + } + } + } + return { shards, lines, broken, events }; +} + +// A second, independent heading counter: split on lines and compare the exact +// canonical marker, without touching the module's parser. +function oracleMarkdownPaths(dir: string, out: string[]): void { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === "audit") continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) oracleMarkdownPaths(full, out); + else if (entry.name.endsWith(".md")) out.push(full); + } +} + +function oracleReviewCount(spaceRoot: string): { canonical: number; suffixed: number } { + let canonical = 0; + let suffixed = 0; + const paths: string[] = []; + oracleMarkdownPaths(join(spaceRoot, "intents"), paths); + for (const path of paths) { + for (const line of readFileSync(path, "utf-8").split("\n")) { + // The permissive scan the writer uses: "## Review" followed by nothing, + // a space, or a tab. Anything else is a different heading entirely. + if (!/^## Review(?:[ \t].*)?$/.test(line)) continue; + if (/^## Review — Iteration \d+$/.test(line)) canonical += 1; + else suffixed += 1; + } + } + return { canonical, suffixed }; +} + +function capturedMain(argv: readonly string[]): { code: number; stdout: string } { + const chunks: string[] = []; + const original = process.stdout.write.bind(process.stdout); + (process.stdout as unknown as { write: (s: string) => boolean }).write = (s: string) => { + chunks.push(s); + return true; + }; + try { + return { code: main(argv), stdout: chunks.join("") }; + } finally { + (process.stdout as unknown as { write: typeof original }).write = original; + } +} + +// --- FR-1: corpus scan and attribution -------------------------------------- + +describe("scanCorpus — both journal generations, path-derived attribution", () => { + test("rows of both schema versions reach the aggregation, and torn lines are counted", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const oracle = oracleScan(spaceRoot); + const corpus = scanCorpus(spaceRoot); + + expect(corpus.shardCount).toBe(oracle.shards); + expect(corpus.lineCount).toBe(oracle.lines); + expect(corpus.brokenLineCount).toBe(oracle.broken); + expect(corpus.brokenLineCount).toBe(1); + // 15 written lines minus the one that does not decode. + expect(corpus.records).toHaveLength(oracle.lines - oracle.broken); + const versions = new Set(corpus.records.map((r) => r.record.schemaVersion)); + expect([...versions].sort()).toEqual([1, 2]); + }); + + test("a degenerate recorded intentId is overridden by the shard path", () => { + const projectDir = scratch(); + const corpus = scanCorpus(buildCorpus(projectDir)); + expect(corpus.records.every((r) => r.record.intentId === "intents")).toBe(true); + expect([...new Set(corpus.records.map((r) => r.intent))].sort()).toEqual(["alpha", "beta"]); + }); + + test("an unreadable shard is counted, and the rest of the corpus still scans", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const before = scanCorpus(spaceRoot); + // A dangling symlink is the portable way to make a directory entry that + // exists for readdir but cannot be read (a directory would raise EISDIR on + // macOS and return "" on Linux). + symlinkSync(join(spaceRoot, "intents", "alpha", "audit", "nowhere.jsonl"), join(spaceRoot, "intents", "alpha", "audit", "dangling.jsonl")); + const after = scanCorpus(spaceRoot); + expect(after.unreadableShardCount).toBe(1); + expect(after.records.length).toBe(before.records.length); + }); + + test("an absent space tree is an empty corpus, not a crash", () => { + const corpus = scanCorpus(join(scratch(), "no", "such", "space")); + expect(corpus).toEqual({ records: [], unreadableShardCount: 0, brokenLineCount: 0, shardCount: 0, lineCount: 0 }); + }); +}); + +// --- FR-3: review iterations from the record tree --------------------------- + +describe("collectReviewBlocks — record artefacts, literal unit dirs kept", () => { + test("canonical headings become blocks and suffixed headings are counted", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const oracle = oracleReviewCount(spaceRoot); + const collected = collectReviewBlocks(join(spaceRoot, "intents")); + + expect(collected.blocks).toHaveLength(oracle.canonical); + expect(collected.unparseableHeadingCount).toBe(oracle.suffixed); + expect(collected.unparseableHeadingCount).toBe(1); + expect(collected.blocks.map((b) => b.unit)).toContain("{unit-name}"); + expect(collected.blocks.map((b) => b.unit)).toContain("unit-x"); + }); +}); + +// --- FR-2 / FR-4 / FR-5 / FR-6 through the assembled report ----------------- + +describe("main — the whole pipeline over a real corpus", () => { + test("the report opens with the measurement ref and states the hypothesis", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { code, stdout } = capturedMain(["--project-dir", projectDir, "--space", "default"]); + expect(code).toBe(0); + const head = stdout.split("\n").slice(0, 16).join("\n"); + for (const key of ["shards:", "lines:", "broken-line:", "unreadable-shard:", "unmatched-start:", "orphan-complete:", "unclosed-idle:", "zero-second:", "invalid-timestamp:", "unparseable-review-heading:"]) { + expect(head).toContain(key); + } + expect(stdout).toContain("unverified hypothesis"); + }); + + test("json carries the layered identities and the exclusion buckets", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { stdout } = capturedMain(["--project-dir", projectDir, "--space", "default", "--json"]); + const report = JSON.parse(stdout) as { + constructedWindowCount: number; + populationCount: number; + exclusions: { corpus: Record; windowing: Record; review: Record }; + models: { attributableCount: number; unresolvedCount: number; totalCount: number }; + stages: { stage: string; rawMedian: number; netMedian: number }[]; + sensors: { stageSlug: string; fired: number; failed: number }[]; + }; + // Identity W over the real corpus. + expect(report.populationCount + report.exclusions.windowing.unclosedIdle + report.exclusions.windowing.zeroSecond).toBe(report.constructedWindowCount); + // Identity M. + expect(report.models.attributableCount + report.models.unresolvedCount).toBe(report.models.totalCount); + expect(report.models.unresolvedCount).toBe(1); + // The fixture's shapes, each counted exactly once. + expect(report.exclusions.corpus.brokenLine).toBe(1); + expect(report.exclusions.windowing.unmatchedStart).toBe(1); + expect(report.exclusions.windowing.zeroSecond).toBe(1); + expect(report.exclusions.windowing.unclosedIdle).toBe(1); + expect(report.exclusions.review.unparseableReviewHeading).toBe(1); + // Idle subtraction: the design window is 100s raw, 30s of it approval wait. + const design = report.stages.find((s) => s.stage === "design"); + expect(design?.rawMedian).toBe(100); + expect(design?.netMedian).toBe(70); + // Sensors bucket on `Stage slug`, whose value differs from `Stage` here. + expect(report.sensors.map((s) => s.stageSlug)).toEqual(["design"]); + }); + + test("two runs over the same corpus produce byte-identical output in all three forms", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + for (const form of [[], ["--format", "csv"], ["--json"]]) { + const args = ["--project-dir", projectDir, "--space", "default", ...form]; + expect(capturedMain(args).stdout).toBe(capturedMain(args).stdout); + } + }); + + test("composeReport over the scanned corpus matches what main rendered", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const reviews = collectReviewBlocks(join(spaceRoot, "intents")); + const report = composeReport({ + scanScope: "x", + corpus: scanCorpus(spaceRoot), + reviewBlocks: reviews.blocks, + unparseableReviewHeadingCount: reviews.unparseableHeadingCount, + }); + const { stdout } = capturedMain(["--project-dir", projectDir, "--space", "default", "--json"]); + expect((JSON.parse(stdout) as { populationCount: number }).populationCount).toBe(report.populationCount); + }); + + test("scanning the real workspace stays well inside the sixty-second ceiling", () => { + const started = Date.now(); + const { code } = capturedMain(["--project-dir", REPO_ROOT, "--space", "default", "--json"]); + const elapsed = (Date.now() - started) / 1000; + expect(code === 0 || code === 1).toBe(true); + expect(elapsed).toBeLessThan(60); + }); +}); + +// --- FR-7: exit ladder and the read-only invariant -------------------------- + +describe("exit ladder — measured by spawning the CLI", () => { + const RUN_OPTIONS = { encoding: "utf-8", env: process.env, timeout: 60_000, killSignal: "SIGKILL" } as const; + + test("a healthy corpus exits 0", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], RUN_OPTIONS); + expect(run.status).toBe(0); + expect(run.stdout).toContain("hypothesisNotice"); + }); + + test("a corpus with an unreadable shard exits 1 while still printing the report", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + symlinkSync(join(spaceRoot, "intents", "alpha", "audit", "nowhere.jsonl"), join(spaceRoot, "intents", "alpha", "audit", "dangling.jsonl")); + const run = spawnSync("bun", [TOOL, "--project-dir", projectDir, "--space", "default", "--json"], RUN_OPTIONS); + expect(run.status).toBe(1); + expect(run.stdout).toContain('"unreadableShard": 1'); + }); + + test("an unknown flag exits 2 and prints usage", () => { + const run = spawnSync("bun", [TOOL, "--nope"], RUN_OPTIONS); + expect(run.status).toBe(2); + expect(run.stderr).toContain("Unknown argument"); + }); +}); + +describe("read-only — the shipped source cannot write", () => { + const WRITE_APIS = [ + "writeFileSync", + "appendFileSync", + "mkdirSync", + "rmSync", + "unlinkSync", + "renameSync", + "cpSync", + "copyFileSync", + "mkdtempSync", + "chmodSync", + "createWriteStream", + "writeFile", + "appendFile", + ]; + + test("no filesystem write API is imported or referenced by the tool", () => { + const source = readFileSync(TOOL, "utf-8"); + const imports = [...source.matchAll(/import\s*\{([^}]*)\}\s*from\s*["']node:fs["']/g)].flatMap((m) => + (m[1] ?? "").split(",").map((s) => s.trim()).filter((s) => s !== ""), + ); + const valueImports = imports.filter((s) => !s.startsWith("type ")); + expect(valueImports.sort()).toEqual(["readFileSync", "readdirSync"]); + for (const api of WRITE_APIS) expect(source.includes(api)).toBe(false); + expect(source.includes("node:fs/promises")).toBe(false); + }); + + test("a run leaves the corpus byte-identical", () => { + const projectDir = scratch(); + const spaceRoot = buildCorpus(projectDir); + const snapshot = new Map(); + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else snapshot.set(full, readFileSync(full, "utf-8")); + } + }; + walk(spaceRoot); + capturedMain(["--project-dir", projectDir, "--space", "default"]); + for (const [path, body] of snapshot) expect(readFileSync(path, "utf-8")).toBe(body); + }); +}); + +describe("resolution seams — driven in-process so bun --coverage measures them", () => { + test("a usage error returns 2 without touching the filesystem", () => { + const { code } = capturedMain(["--definitely-unknown-flag"]); + expect(code).toBe(2); + }); + + test("CLAUDE_PROJECT_DIR resolves the project dir when the flag is absent", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const saved = process.env.CLAUDE_PROJECT_DIR; + process.env.CLAUDE_PROJECT_DIR = projectDir; + try { + const { code, stdout } = capturedMain(["--space", "default"]); + expect(code).toBe(0); + expect(stdout).toContain("shards:"); + } finally { + if (saved === undefined) delete process.env.CLAUDE_PROJECT_DIR; + else process.env.CLAUDE_PROJECT_DIR = saved; + } + }); + + test("without env or flag the cwd rung resolves, and active-space names the space", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + writeFileSync(join(projectDir, "amadeus", "active-space"), "default\n"); + const savedEnv = process.env.CLAUDE_PROJECT_DIR; + const savedCwd = process.cwd(); + delete process.env.CLAUDE_PROJECT_DIR; + process.chdir(projectDir); + try { + const { code, stdout } = capturedMain([]); + expect(code).toBe(0); + expect(stdout).toContain('space "default"'); + } finally { + process.chdir(savedCwd); + if (savedEnv !== undefined) process.env.CLAUDE_PROJECT_DIR = savedEnv; + } + }); + + test("a missing active-space file falls back to the default space", () => { + const projectDir = scratch(); + buildCorpus(projectDir); + const { code, stdout } = capturedMain(["--project-dir", projectDir]); + expect(code).toBe(0); + expect(stdout).toContain('space "default"'); + }); +}); diff --git a/tests/unit/t486-stage-stats.test.ts b/tests/unit/t486-stage-stats.test.ts new file mode 100644 index 000000000..e9cf8ce06 --- /dev/null +++ b/tests/unit/t486-stage-stats.test.ts @@ -0,0 +1,569 @@ +// covers: function:nearestRankP95, function:composeStageStats +// +// t486 — Issue #2405 (U1 stage-stats). Unit half of the twin: every pure +// function of amadeus-stage-stats.ts, driven in-process so bun --coverage +// measures the whole aggregation contract (the FS scan and the CLI spawn live +// in t487's integration layer — cid:code-generation:fs-tests-integration-first). +import { describe, expect, test } from "bun:test"; +import { + type AttributedRecord, + buildWindows, + composeStageStats, + emptyExclusions, + indexIdle, + type MeasuredWindow, + nearestRankP95, + parseReviewHeadings, + reviewAttribution, + type StageWindow, + subtractIdle, + attributeModels, + composeReport, + HYPOTHESIS_NOTICE, + parseArgs, + renderCsv, + renderMarkdown, + serializeJson, + tallySensors, +} from "../../packages/framework/core/tools/amadeus-stage-stats.ts"; + +// Independent oracle for the corpus shape: the tests hand-build v1 and v2 +// journal records rather than routing through the module's own scanner, so a +// scanner defect cannot cancel against the expectation. +function v1(intent: string, event: string, timestamp: string, fields: Record = {}): AttributedRecord { + return { intent, record: { schemaVersion: 1, seq: 1, cloneId: "c", intentId: "intents", timestamp, heading: "h", event, fields } }; +} + +function v2(intent: string, event: string, timestamp: string, attrs: Record = {}): AttributedRecord { + return { + intent, + record: { + schemaVersion: 2, + eventId: "e", + seq: 1, + timestamp, + eventName: "amadeus.test", + attributes: { Event: event, ...attrs }, + intentId: "intents", + space: "default", + cloneId: "c", + traceId: null, + spanId: null, + traceFlags: 0, + idempotencyKey: `${timestamp}:${event}`, + canonical: true, + }, + }; +} + +// An independent oracle for the window shape: the tests build MeasuredWindow +// literals directly rather than calling the module's own constructor, so a +// constructor defect cannot cancel out against the expectation (BR: no +// self-referential comparison). +function win(stage: string, raw: number, net: number): MeasuredWindow { + return { intent: "i", stage, startedAt: "2026-01-01T00:00:00Z", completedAt: "2026-01-01T00:00:01Z", rawSeconds: raw, idleSeconds: raw - net, netSeconds: net }; +} + +describe("nearestRankP95 — nearest rank, NaN on empty", () => { + test("empty input propagates NaN instead of collapsing to 0", () => { + expect(nearestRankP95([])).toBeNaN(); + }); + test("nearest rank picks sorted[ceil(0.95n) - 1]", () => { + const values = Array.from({ length: 100 }, (_, i) => i + 1); + expect(nearestRankP95(values)).toBe(95); + }); + test("a single sample is its own p95", () => { + expect(nearestRankP95([42])).toBe(42); + }); + test("input order does not change the result", () => { + expect(nearestRankP95([9, 1, 5, 3, 7])).toBe(nearestRankP95([1, 3, 5, 7, 9])); + }); +}); + +describe("composeStageStats — per-stage mean/median/p95", () => { + test("groups by stage and reports n, raw median, and net statistics", () => { + const stats = composeStageStats([win("a", 10, 5), win("a", 30, 15), win("b", 100, 100)]); + expect(stats.map((s) => s.stage)).toEqual(["a", "b"]); + const a = stats[0]!; + expect(a.n).toBe(2); + expect(a.rawMedian).toBe(20); + expect(a.netMean).toBe(10); + expect(a.netMedian).toBe(10); + expect(a.netP95).toBe(15); + }); + test("empty population yields an empty table, never a zero row", () => { + expect(composeStageStats([])).toEqual([]); + }); + test("ordering is count-desc then key-asc", () => { + const stats = composeStageStats([win("zz", 1, 1), win("aa", 1, 1), win("aa", 1, 1)]); + expect(stats.map((s) => s.stage)).toEqual(["aa", "zz"]); + }); +}); + +describe("buildWindows — pairing START with COMPLETED per intent x stage", () => { + test("pairs both journal generations and computes raw seconds", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v2("i1", "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(1); + expect(windows[0]!.rawSeconds).toBe(100); + expect(windows[0]!.stage).toBe("design"); + expect(buckets.windowing.unmatchedStart).toBe(0); + expect(buckets.windowing.orphanComplete).toBe(0); + }); + + test("a START with no COMPLETED is the unmatched-start bucket, not a window", () => { + const { windows, buckets } = buildWindows([v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" })]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.unmatchedStart).toBe(1); + }); + + test("a COMPLETED with no START is the orphan-complete bucket", () => { + const { buckets } = buildWindows([v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:00Z", { Stage: "design" })]); + expect(buckets.windowing.orphanComplete).toBe(1); + }); + + test("the same stage in two intents does not cross-pair", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1("i2", "STAGE_COMPLETED", "2026-01-01T00:00:10Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.unmatchedStart).toBe(1); + expect(buckets.windowing.orphanComplete).toBe(1); + }); + + test("records are ordered by timestamp before pairing, not by input order", () => { + const { windows } = buildWindows([ + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:30Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + ]); + expect(windows).toHaveLength(1); + expect(windows[0]!.rawSeconds).toBe(30); + }); + + test("emptyExclusions starts every bucket at zero", () => { + const zero = emptyExclusions(); + expect(zero.corpus).toEqual({ brokenLine: 0, unreadableShard: 0 }); + expect(zero.windowing).toEqual({ unmatchedStart: 0, orphanComplete: 0, unclosedIdle: 0, zeroSecond: 0, invalidTimestamp: 0 }); + expect(zero.review).toEqual({ unparseableReviewHeading: 0 }); + }); + + test("an unparseable timestamp leaves the window population and is counted, not NaN-poisoned", () => { + const { windows, buckets } = buildWindows([ + v1("i1", "STAGE_STARTED", "not-a-timestamp", { Stage: "design" }), + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:00:30Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:01:00Z", { Stage: "build" }), + v1("i1", "STAGE_COMPLETED", "also-garbage", { Stage: "build" }), + ]); + expect(windows).toHaveLength(0); + expect(buckets.windowing.invalidTimestamp).toBe(2); + expect(buckets.windowing.orphanComplete).toBe(1); + expect(buckets.windowing.unmatchedStart).toBe(1); + }); + + test("an idle event with an unparseable timestamp is counted instead of poisoning the intervals", () => { + const index = indexIdle([ + v1("i1", "STAGE_AWAITING_APPROVAL", "garbage-time"), + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:05Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:10Z"), + ]); + expect(index.invalidTimestampCount).toBe(1); + expect(index.intervals.get("i1")).toHaveLength(1); + }); +}); + +// A stage window built straight from literals, so the idle tests do not depend +// on buildWindows being correct (independent oracle). +function stageWindow(startedAt: string, completedAt: string, rawSeconds: number): StageWindow { + return { intent: "i1", stage: "design", startedAt, completedAt, rawSeconds }; +} + +describe("subtractIdle — clip, merge, and exclude", () => { + test("an approval wait inside the window makes net strictly less than raw", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + ], + ); + expect(measured).toHaveLength(1); + expect(measured[0]!.idleSeconds).toBe(30); + expect(measured[0]!.netSeconds).toBe(70); + expect(measured[0]!.netSeconds).toBeLessThan(measured[0]!.rawSeconds); + }); + + test("a park span and a session gap both subtract", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:10Z"), + v1("i1", "WORKFLOW_UNPARKED", "2026-01-01T00:00:20Z"), + v2("i1", "SESSION_ENDED", "2026-01-01T00:00:30Z"), + v2("i1", "SESSION_RESUMED", "2026-01-01T00:00:45Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(25); + expect(measured[0]!.netSeconds).toBe(75); + }); + + test("overlapping idle spans merge — no double subtraction", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:10Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:20Z"), + v1("i1", "WORKFLOW_UNPARKED", "2026-01-01T00:00:40Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(40); + expect(measured[0]!.netSeconds).toBe(60); + }); + + test("idle extending past both window edges clips to the window, never negative", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:10Z", "2026-01-01T00:00:20Z", 10)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:00Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:01:00Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(10); + expect(measured[0]!.netSeconds).toBe(0); + expect(measured[0]!.netSeconds).toBeGreaterThanOrEqual(0); + }); + + test("idle abutting the window end (same-try-block gate approval) yields no negative net", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:30Z", 30)], + [ + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:30Z"), + ], + ); + expect(measured[0]!.netSeconds).toBe(20); + }); + + test("an idle span in a different intent does not reach this window", () => { + const { measured } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [ + v1("i2", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i2", "GATE_APPROVED", "2026-01-01T00:00:50Z"), + ], + ); + expect(measured[0]!.idleSeconds).toBe(0); + expect(measured[0]!.netSeconds).toBe(100); + }); + + test("an unclosed idle opener drops the window from the population and is counted", () => { + const { measured, buckets } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:01:40Z", 100)], + [v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z")], + ); + expect(measured).toHaveLength(0); + expect(buckets.windowing.unclosedIdle).toBe(1); + expect(buckets.windowing.zeroSecond).toBe(0); + }); + + test("a zero-second window is excluded as below the timestamp resolution", () => { + const { measured, buckets } = subtractIdle([stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0)], []); + expect(measured).toHaveLength(0); + expect(buckets.windowing.zeroSecond).toBe(1); + }); + + test("zeroSecond and unclosedIdle are mutually exclusive — a window satisfying both counts once", () => { + const { measured, buckets } = subtractIdle( + [stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0)], + [v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:00:00Z")], + ); + expect(measured).toHaveLength(0); + expect(buckets.windowing.unclosedIdle + buckets.windowing.zeroSecond).toBe(1); + expect(buckets.windowing.unclosedIdle).toBe(1); + }); + + test("identity W holds: constructed windows = population + unclosedIdle + zeroSecond", () => { + const windows = [ + stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:10Z", 10), + stageWindow("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", 0), + stageWindow("2026-01-01T00:01:00Z", "2026-01-01T00:02:00Z", 60), + ]; + const { measured, buckets } = subtractIdle(windows, [v1("i1", "WORKFLOW_PARKED", "2026-01-01T00:01:10Z")]); + expect(measured.length + buckets.windowing.unclosedIdle + buckets.windowing.zeroSecond).toBe(windows.length); + }); +}); + +describe("parseReviewHeadings — two-stage match, suffixes are reported", () => { + test("a canonical heading yields its iteration", () => { + const { iterations, unparseable } = parseReviewHeadings("intro\n\n## Review — Iteration 2\n\nbody\n"); + expect(iterations).toEqual([2]); + expect(unparseable).toBe(0); + }); + test("a suffixed heading is counted as unparseable, never dropped in silence", () => { + const { iterations, unparseable } = parseReviewHeadings("## Review — Iteration 1 (follow-up)\n"); + expect(iterations).toEqual([]); + expect(unparseable).toBe(1); + }); + test("multiple canonical headings all count", () => { + const { iterations } = parseReviewHeadings("## Review — Iteration 1\nx\n## Review — Iteration 3\n"); + expect(iterations).toEqual([1, 3]); + }); + test("a line that only mentions the marker mid-text is not a heading", () => { + const { iterations, unparseable } = parseReviewHeadings("see ## Review — Iteration 1 above\n"); + expect(iterations).toEqual([]); + expect(unparseable).toBe(0); + }); + test("an unrelated H2 is not a review heading", () => { + expect(parseReviewHeadings("## Findings\n").unparseable).toBe(0); + }); +}); + +describe("reviewAttribution — stage and unit come from the path", () => { + test("a construction unit path keeps its unit segment", () => { + expect(reviewAttribution("construction/stage-stats-cli/functional-design/business-rules.md")).toEqual({ + stagePath: "construction/stage-stats-cli/functional-design", + unit: "stage-stats-cli", + }); + }); + test("a literal {unit-name} directory is displayed as-is, not corrected", () => { + expect(reviewAttribution("construction/{unit-name}/code-generation/code-summary.md").unit).toBe("{unit-name}"); + }); + test("a non-construction path has no unit", () => { + expect(reviewAttribution("inception/requirements-analysis/requirements.md")).toEqual({ + stagePath: "inception/requirements-analysis", + unit: null, + }); + }); +}); + +describe("tallySensors — bucketed by `Stage slug`, never by `Stage`", () => { + test("the `Stage slug` attribute wins over a differing `Stage`", () => { + const rows = tallySensors([ + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:00Z", { "Stage slug": "design", Stage: "Design Stage" }), + v1("i1", "SENSOR_PASSED", "2026-01-01T00:00:01Z", { "Stage slug": "design", Stage: "Design Stage" }), + v2("i1", "SENSOR_FIRED", "2026-01-01T00:00:02Z", { "Stage slug": "design", Stage: "other" }), + v2("i1", "SENSOR_FAILED", "2026-01-01T00:00:03Z", { "Stage slug": "design", Stage: "other" }), + ]); + expect(rows).toHaveLength(1); + expect(rows[0]!.stageSlug).toBe("design"); + expect(rows[0]!.fired).toBe(2); + expect(rows[0]!.passed).toBe(1); + expect(rows[0]!.failed).toBe(1); + expect(rows[0]!.failedRate).toBe(0.5); + }); + test("a slug that never fired reports NaN, not a 0% pass", () => { + const rows = tallySensors([v1("i1", "SENSOR_FAILED", "2026-01-01T00:00:00Z", { "Stage slug": "x" })]); + expect(rows[0]!.failedRate).toBeNaN(); + }); + test("ordering is fired-desc then slug-asc", () => { + const rows = tallySensors([ + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:00Z", { "Stage slug": "zz" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:01Z", { "Stage slug": "aa" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:02Z", { "Stage slug": "aa" }), + ]); + expect(rows.map((r) => r.stageSlug)).toEqual(["aa", "zz"]); + }); +}); + +describe("attributeModels — absence is recorded as absence", () => { + test("attributable + unresolved equals the total (identity M)", () => { + const attribution = attributeModels([ + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:00Z", { Model: "opus", "Model Source": "pin" }), + v2("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:01Z", { Model: "sonnet", "Model Source": "default" }), + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:02Z", {}), + v1("i1", "SUBAGENT_STARTED", "2026-01-01T00:00:03Z", { Model: "opus" }), + ]); + expect(attribution.totalCount).toBe(3); + expect(attribution.attributableCount).toBe(2); + expect(attribution.unresolvedCount).toBe(1); + expect(attribution.attributableCount + attribution.unresolvedCount).toBe(attribution.totalCount); + expect(attribution.byModel.get("opus")).toBe(1); + expect(attribution.byModel.get("sonnet")).toBe(1); + expect(attribution.byModelSource.get("pin")).toBe(1); + }); + test("a start row never joins the population — one dispatch is one completion", () => { + const attribution = attributeModels([v1("i1", "SUBAGENT_STARTED", "2026-01-01T00:00:00Z", { Model: "opus" })]); + expect(attribution.totalCount).toBe(0); + expect(attribution.byModel.size).toBe(0); + }); + test("a blank Model is unresolved, not a model named empty string", () => { + const attribution = attributeModels([v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:00Z", { Model: " " })]); + expect(attribution.unresolvedCount).toBe(1); + expect(attribution.byModel.size).toBe(0); + }); + test("non-subagent events are outside the model population entirely", () => { + expect(attributeModels([v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z")]).totalCount).toBe(0); + }); +}); + +// A corpus exercising every reported axis at once: two paired windows, an idle +// span, an unmatched start, a zero-second window, sensors on two slugs, and one +// attributable plus one unattributable subagent row. +function mixedCorpus(): AttributedRecord[] { + return [ + v1("i1", "STAGE_STARTED", "2026-01-01T00:00:00Z", { Stage: "design" }), + v1("i1", "STAGE_AWAITING_APPROVAL", "2026-01-01T00:00:20Z"), + v1("i1", "GATE_APPROVED", "2026-01-01T00:00:40Z"), + v2("i1", "STAGE_COMPLETED", "2026-01-01T00:01:40Z", { Stage: "design" }), + v1("i1", "STAGE_STARTED", "2026-01-01T00:02:00Z", { Stage: "build" }), + v1("i1", "STAGE_COMPLETED", "2026-01-01T00:02:30Z", { Stage: "build" }), + v1("i2", "STAGE_STARTED", "2026-01-01T00:03:00Z", { Stage: "design" }), + v1("i2", "STAGE_COMPLETED", "2026-01-01T00:03:00Z", { Stage: "design" }), + v1("i2", "STAGE_STARTED", "2026-01-01T00:04:00Z", { Stage: "orphan" }), + v1("i1", "SENSOR_FIRED", "2026-01-01T00:00:05Z", { "Stage slug": "design", Stage: "Design" }), + v1("i1", "SENSOR_FAILED", "2026-01-01T00:00:06Z", { "Stage slug": "design", Stage: "Design" }), + v2("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:07Z", { Model: "opus", "Model Source": "pin" }), + v1("i1", "SUBAGENT_COMPLETED", "2026-01-01T00:00:08Z", {}), + ]; +} + +function sampleReport() { + return composeReport({ + scanScope: "space \"default\"", + corpus: { records: mixedCorpus(), unreadableShardCount: 0, brokenLineCount: 2, shardCount: 3, lineCount: 13 }, + reviewBlocks: [ + { intent: "i1", stagePath: "construction/u/functional-design", unit: "u", iteration: 1 }, + { intent: "i1", stagePath: "construction/u/functional-design", unit: "u", iteration: 2 }, + { intent: "i1", stagePath: "construction/{unit-name}/code-generation", unit: "{unit-name}", iteration: 1 }, + ], + unparseableReviewHeadingCount: 1, + }); +} + +describe("composeReport — measurement ref, buckets, and the layered identity", () => { + test("carries every corpus counter into the report", () => { + const report = sampleReport(); + expect(report.shardCount).toBe(3); + expect(report.lineCount).toBe(13); + expect(report.exclusions.corpus.brokenLine).toBe(2); + expect(report.exclusions.corpus.unreadableShard).toBe(0); + expect(report.exclusions.review.unparseableReviewHeading).toBe(1); + }); + + test("identity W holds on the assembled report", () => { + const report = sampleReport(); + const w = report.exclusions.windowing; + expect(report.populationCount + w.unclosedIdle + w.zeroSecond).toBe(report.constructedWindowCount); + }); + + test("identity M holds on the assembled report", () => { + const m = sampleReport().models; + expect(m.attributableCount + m.unresolvedCount).toBe(m.totalCount); + }); + + test("the unmatched start is counted but is not part of identity W", () => { + const report = sampleReport(); + expect(report.exclusions.windowing.unmatchedStart).toBe(1); + expect(report.constructedWindowCount).toBe(3); + }); + + test("idle subtraction shows up as net below raw for the design stage", () => { + const design = sampleReport().stages.find((s) => s.stage === "design"); + expect(design?.netMedian).toBe(80); + expect(design?.rawMedian).toBe(100); + }); + + test("review iterations are bucketed by stage path, literal unit dirs kept", () => { + const buckets = sampleReport().reviewBuckets; + expect(buckets.map((b) => b.stagePath)).toContain("construction/{unit-name}/code-generation"); + const fd = buckets.find((b) => b.stagePath === "construction/u/functional-design"); + expect(fd?.blocks).toBe(2); + expect(fd?.maxIteration).toBe(2); + }); +}); + +describe("renderers — deterministic, header-first, hypothesis stated", () => { + test("markdown opens with the measurement ref and states the hypothesis", () => { + const text = renderMarkdown(sampleReport()); + const head = text.split("\n").slice(0, 20).join("\n"); + expect(head).toContain("shards: 3"); + expect(head).toContain("lines: 13"); + expect(head).toContain("broken-line: 2"); + expect(head).toContain("unreadable-shard: 0"); + expect(head).toContain("unmatched-start: 1"); + expect(head).toContain("orphan-complete: 0"); + expect(head).toContain("unclosed-idle: 0"); + expect(head).toContain("zero-second: 1"); + expect(head).toContain("invalid-timestamp: 0"); + expect(head).toContain("unparseable-review-heading: 1"); + expect(text).toContain(HYPOTHESIS_NOTICE); + }); + + test("a stage name with control bytes and a newline is reduced at the render point", () => { + const report = sampleReport(); + const hostile = { ...report, stages: [{ stage: "de\u0007sign\ninjected", n: 1, rawMedian: 1, netMean: 1, netMedian: 1, netP95: 1 }] }; + const markdown = renderMarkdown(hostile); + expect(markdown).toContain("| design |"); + expect(markdown).not.toContain("injected"); + expect(markdown).not.toContain("\u0007"); + }); + + test("a csv cell with a comma and a quote cannot forge an extra column", () => { + const report = sampleReport(); + const hostile = { ...report, stages: [{ stage: 'a,"b', n: 1, rawMedian: 1, netMean: 1, netMedian: 1, netP95: 1 }] }; + const row = renderCsv(hostile).split("\n").find((line) => line.includes('a,""b')); + expect(row).toBe('"a,""b",1,1,1,1,1'); + }); + + test("csv also carries the measurement ref and the hypothesis before the rows", () => { + const text = renderCsv(sampleReport()); + const lines = text.split("\n"); + expect(text).toContain(HYPOTHESIS_NOTICE); + expect(lines[0]).toBe("section,key,value"); + expect(lines[1]).toContain("stage-stats"); + expect(text).toContain("zero-second,"); + // Measurement ref precedes every data section, not merely present somewhere. + expect(text.indexOf("unparseable-review-heading")).toBeLessThan(text.indexOf("stage,n,raw_median")); + }); + + test("all three forms are byte-identical across two runs of the same input", () => { + expect(renderMarkdown(sampleReport())).toBe(renderMarkdown(sampleReport())); + expect(renderCsv(sampleReport())).toBe(renderCsv(sampleReport())); + expect(JSON.stringify(serializeJson(sampleReport()))).toBe(JSON.stringify(serializeJson(sampleReport()))); + }); + + test("json turns the maps into fixed-order arrays and keeps the notice", () => { + const json = serializeJson(sampleReport()) as Record; + expect(json.hypothesisNotice).toBe(HYPOTHESIS_NOTICE); + expect(Array.isArray((json.models as Record).byModel)).toBe(true); + expect(json.constructedWindowCount).toBe(3); + }); +}); + +describe("parseArgs — parse, do not validate", () => { + test("no arguments default to markdown", () => { + const parsed = parseArgs([]); + expect(parsed.ok).toBe(true); + if (parsed.ok) expect(parsed.value.format).toBe("markdown"); + }); + test("--json selects the json form", () => { + const parsed = parseArgs(["--json"]); + expect(parsed.ok && parsed.value.format).toBe("json"); + }); + test("--format csv selects the csv form", () => { + const parsed = parseArgs(["--format", "csv"]); + expect(parsed.ok && parsed.value.format).toBe("csv"); + }); + test("an unknown flag is a usage error, not a silently ignored argument", () => { + const parsed = parseArgs(["--nope"]); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.error.message).toContain("--nope"); + }); + test("an option missing its value is a usage error", () => { + expect(parseArgs(["--space"]).ok).toBe(false); + }); + test("an unsupported --format value is a usage error", () => { + expect(parseArgs(["--format", "yaml"]).ok).toBe(false); + }); + test("an unsafe --space value is a usage error", () => { + expect(parseArgs(["--space", "../etc"]).ok).toBe(false); + }); + test("--project-dir and --space carry through when valid", () => { + const parsed = parseArgs(["--project-dir", "/tmp/x", "--space", "team-a"]); + expect(parsed.ok && parsed.value.projectDir).toBe("/tmp/x"); + expect(parsed.ok && parsed.value.space).toBe("team-a"); + }); +});