fix(report): expose per-case execution metrics

CHANGELOG.md

@@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Fixed
+- HTML and Markdown reports now identify the top-level duration as evaluation
+  wall time and show per-case tested-agent execution time plus input, output,
+  and total token usage. Benchmark cases include compact with-Skill,
+  without-Skill, and delta annotations beside the case heading. Agent-judge
+  time and tokens are reported separately and included in an explicit overall
+  token total.
 - Codex JSONL parsing now accepts records up to a configurable 16 MiB default
   while rejecting oversized records explicitly. The stdout stream is written to
   a bounded-download artifact instead of being buffered without limit in host

internal/report/README.md

@@ -67,6 +67,8 @@ package "internal/report" {
     EndTime : time.Time
     CaseResults : []CaseResult
     TotalTokens : int
+    JudgeTokens : int
+    OverallTokens : int
     Benchmark : *BenchmarkResult
     +TotalDuration() time.Duration
     +OverallPassRate() float64
@@ -78,6 +80,11 @@ package "internal/report" {
     Status : judge.Status
     DurationMs : int64
     Turns : int
+    InputTokens : int
+    OutputTokens : int
+    JudgeDurationMs : int64
+    JudgeInputTokens : int
+    JudgeOutputTokens : int
     Error : string
     Grading : *judge.Result
   }
@@ -160,9 +167,25 @@ end note
 
 - Rendered with the standard library's `html/template`; the template is loaded via `go:embed` from `templates/report.html`
 - Bundles responsive CSS styles
-- Displays: skill name, engine, model, start time, execution time, pass rate
+- Displays: skill name, engine, model, start time, evaluation wall time, pass rate, and separated tested-agent / judge / overall token totals
 - Summary cards: Total / Passed / Failed / Skipped / Errors / Pass Rate
-- Per-case detail table: status icons, assertion results, evidence
+- Per-case details: compact tested-agent and optional agent-judge metrics beside the case heading, plus status icons, assertion results, and evidence; input/output token counts remain available as hover details
+- Benchmark cases show compact with-Skill, without-Skill, and delta metrics beside the case heading so execution cost remains secondary to the response and grading content
+
+### Metric semantics
+
+- `Input.TotalDuration()` is **evaluation wall time** (`EndTime - StartTime`). It
+  includes orchestration, tested-agent execution, judging, and other framework
+  overhead, so it is not expected to equal the sum of visible case execution
+  times, especially when cases run concurrently.
+- `CaseResult.DurationMs` is **tested-agent execution time** for that case and
+  configuration. It is the primary duration for comparing Skill behavior.
+- `CaseResult.InputTokens` and `OutputTokens` are tested-agent token usage.
+  `Input.TotalTokens` retains its existing JSON name for compatibility and is
+  the sum of those tested-agent tokens across all configurations.
+- `JudgeDurationMs`, `JudgeInputTokens`, and `JudgeOutputTokens` are populated
+  when the judge runs a separate agent session. `Input.JudgeTokens` aggregates
+  those tokens, and `Input.OverallTokens` is tested-agent plus judge tokens.
 
 ### JUnitReporter (`junit.go`)
 

internal/report/html.go

@@ -58,15 +58,17 @@ type htmlReportData struct {
 // -- Embedded JSON types for JavaScript consumption --
 
 type embeddedReportData struct {
-	SkillName   string           `json:"skill_name"`
-	EngineName  string           `json:"engine_name"`
-	ModelName   string           `json:"model_name"`
-	StartTime   string           `json:"start_time"`
-	Duration    string           `json:"duration"`
-	TotalTokens int              `json:"total_tokens"`
-	Summary     embeddedSummary  `json:"summary"`
-	Cases       []embeddedCase   `json:"cases"`
-	Benchmark   *BenchmarkResult `json:"benchmark,omitempty"`
+	SkillName          string           `json:"skill_name"`
+	EngineName         string           `json:"engine_name"`
+	ModelName          string           `json:"model_name"`
+	StartTime          string           `json:"start_time"`
+	EvaluationWallTime string           `json:"evaluation_wall_time"`
+	AgentTokens        int              `json:"agent_tokens"`
+	JudgeTokens        int              `json:"judge_tokens"`
+	OverallTokens      int              `json:"overall_tokens"`
+	Summary            embeddedSummary  `json:"summary"`
+	Cases              []embeddedCase   `json:"cases"`
+	Benchmark          *BenchmarkResult `json:"benchmark,omitempty"`
 }
 
 type embeddedSummary struct {
@@ -79,20 +81,28 @@ type embeddedSummary struct {
 }
 
 type embeddedCase struct {
-	ID            string            `json:"id"`
-	Title         string            `json:"title,omitempty"`
-	Status        string            `json:"status"`
-	DurationMs    int64             `json:"duration_ms"`
-	Duration      string            `json:"duration"`
-	Turns         int               `json:"turns"`
-	Error         string            `json:"error,omitempty"`
-	Grading       *embeddedGrading  `json:"grading,omitempty"`
-	Configuration string            `json:"configuration,omitempty"`
-	Prompt        string            `json:"prompt,omitempty"`
-	Response      string            `json:"response,omitempty"`
-	Baseline      *embeddedCase     `json:"baseline,omitempty"`
-	TurnResults   []embeddedTurn    `json:"turn_results,omitempty"`
-	JudgeSkills   []judge.SkillInfo `json:"judge_skills,omitempty"`
+	ID                string            `json:"id"`
+	Title             string            `json:"title,omitempty"`
+	Status            string            `json:"status"`
+	AgentDurationMs   int64             `json:"agent_duration_ms"`
+	AgentDuration     string            `json:"agent_duration"`
+	InputTokens       int               `json:"input_tokens"`
+	OutputTokens      int               `json:"output_tokens"`
+	AgentTokens       int               `json:"agent_tokens"`
+	JudgeDurationMs   int64             `json:"judge_duration_ms"`
+	JudgeDuration     string            `json:"judge_duration"`
+	JudgeInputTokens  int               `json:"judge_input_tokens"`
+	JudgeOutputTokens int               `json:"judge_output_tokens"`
+	JudgeTokens       int               `json:"judge_tokens"`
+	Turns             int               `json:"turns"`
+	Error             string            `json:"error,omitempty"`
+	Grading           *embeddedGrading  `json:"grading,omitempty"`
+	Configuration     string            `json:"configuration,omitempty"`
+	Prompt            string            `json:"prompt,omitempty"`
+	Response          string            `json:"response,omitempty"`
+	Baseline          *embeddedCase     `json:"baseline,omitempty"`
+	TurnResults       []embeddedTurn    `json:"turn_results,omitempty"`
+	JudgeSkills       []judge.SkillInfo `json:"judge_skills,omitempty"`
 }
 
 // embeddedTurn holds per-turn data for the HTML report JavaScript.
@@ -136,18 +146,26 @@ type caseStatusCounts struct {
 
 func caseResultToEmbeddedCase(cr CaseResult) embeddedCase {
 	ec := embeddedCase{
-		ID:            cr.CaseID,
-		Title:         cr.Title,
-		Status:        string(cr.Status),
-		DurationMs:    cr.DurationMs,
-		Duration:      fmt.Sprintf("%.1fs", float64(cr.DurationMs)/1000.0),
-		Turns:         cr.Turns,
-		Error:         cr.Error,
-		Configuration: cr.Configuration,
-		Prompt:        cr.Prompt,
-		Response:      cr.Response,
-		TurnResults:   caseTurnResultsToEmbedded(cr.TurnResults),
-		JudgeSkills:   cr.JudgeSkills,
+		ID:                cr.CaseID,
+		Title:             cr.Title,
+		Status:            string(cr.Status),
+		AgentDurationMs:   cr.DurationMs,
+		AgentDuration:     fmt.Sprintf("%.1fs", float64(cr.DurationMs)/1000.0),
+		InputTokens:       cr.InputTokens,
+		OutputTokens:      cr.OutputTokens,
+		AgentTokens:       cr.InputTokens + cr.OutputTokens,
+		JudgeDurationMs:   cr.JudgeDurationMs,
+		JudgeDuration:     fmt.Sprintf("%.1fs", float64(cr.JudgeDurationMs)/1000.0),
+		JudgeInputTokens:  cr.JudgeInputTokens,
+		JudgeOutputTokens: cr.JudgeOutputTokens,
+		JudgeTokens:       cr.JudgeInputTokens + cr.JudgeOutputTokens,
+		Turns:             cr.Turns,
+		Error:             cr.Error,
+		Configuration:     cr.Configuration,
+		Prompt:            cr.Prompt,
+		Response:          cr.Response,
+		TurnResults:       caseTurnResultsToEmbedded(cr.TurnResults),
+		JudgeSkills:       cr.JudgeSkills,
 	}
 	if cr.Grading != nil {
 		eg := &embeddedGrading{
@@ -252,12 +270,14 @@ func (r *HTMLReporter) buildTemplateData(in Input) (htmlReportData, error) {
 	cases := buildEmbeddedCases(grouped, orderedIDs)
 
 	ed := embeddedReportData{
-		SkillName:   in.SkillName,
-		EngineName:  in.EngineName,
-		ModelName:   in.ModelName,
-		StartTime:   in.StartTime.Format(time.RFC3339),
-		Duration:    fmt.Sprintf("%.1fs", in.TotalDuration().Seconds()),
-		TotalTokens: in.TotalTokens,
+		SkillName:          in.SkillName,
+		EngineName:         in.EngineName,
+		ModelName:          in.ModelName,
+		StartTime:          in.StartTime.Format(time.RFC3339),
+		EvaluationWallTime: fmt.Sprintf("%.1fs", in.TotalDuration().Seconds()),
+		AgentTokens:        in.TotalTokens,
+		JudgeTokens:        in.JudgeTokens,
+		OverallTokens:      in.TotalTokens + in.JudgeTokens,
 		Summary: embeddedSummary{
 			Total:    len(cases),
 			Passed:   counts.passed,

internal/report/json.go

@@ -17,6 +17,9 @@ type JSONReporter struct {
 
 // Write implements the Reporter interface.
 func (r *JSONReporter) Write(_ context.Context, in Input) error {
+	// OverallTokens is derived so reports regenerated from an older result.json
+	// also receive a consistent explicit total.
+	in.OverallTokens = in.TotalTokens + in.JudgeTokens
 	data, err := json.MarshalIndent(in, "", "  ")
 	if err != nil {
 		return fmt.Errorf("json marshal: %w", err)

internal/report/markdown.go

@@ -60,26 +60,66 @@ func writeMarkdownSummary(sb *strings.Builder, in Input) {
 	fmt.Fprintf(sb, "| Errors | %d |\n", markdownCountByStatus(in, judge.StatusError))
 	fmt.Fprintf(sb, "| Skipped | %d |\n", markdownCountByStatus(in, judge.StatusSkip))
 	fmt.Fprintf(sb, "| Pass Rate | %.1f%% |\n", in.OverallPassRate()*100)
-	fmt.Fprintf(sb, "| Duration | %s |\n", markdownDuration(in.TotalDuration().Milliseconds()))
-	fmt.Fprintf(sb, "| Total Tokens | %d |\n\n", in.TotalTokens)
+	fmt.Fprintf(sb, "| Evaluation Wall Time | %s |\n", markdownDuration(in.TotalDuration().Milliseconds()))
+	fmt.Fprintf(sb, "| Tested Agent Tokens | %d |\n", in.TotalTokens)
+	fmt.Fprintf(sb, "| Judge Tokens | %d |\n", in.JudgeTokens)
+	fmt.Fprintf(sb, "| Overall Tokens | %d |\n\n", in.TotalTokens+in.JudgeTokens)
 }
 
 func writeMarkdownCases(sb *strings.Builder, in Input) {
 	sb.WriteString("## Cases\n\n")
-	sb.WriteString("| Case | Title | Status | Duration | Turns |\n")
-	sb.WriteString("|---|---|---|---:|---:|\n")
+	if markdownHasJudgeMetrics(in) {
+		writeMarkdownCasesWithJudgeMetrics(sb, in)
+		return
+	}
+	sb.WriteString("| Case | Title | Configuration | Status | Agent Time | Input Tokens | Output Tokens | Agent Tokens | Turns |\n")
+	sb.WriteString("|---|---|---|---|---:|---:|---:|---:|---:|\n")
+	for _, cr := range in.CaseResults {
+		fmt.Fprintf(sb, "| %s | %s | %s | %s | %s | %d | %d | %d | %d |\n",
+			markdownTableCell(cr.CaseID),
+			markdownTableCell(cr.Title),
+			markdownTableCell(cr.Configuration),
+			markdownTableCell(string(cr.Status)),
+			markdownDuration(cr.DurationMs),
+			cr.InputTokens,
+			cr.OutputTokens,
+			cr.InputTokens+cr.OutputTokens,
+			cr.Turns,
+		)
+	}
+	sb.WriteString("\n")
+}
+
+func writeMarkdownCasesWithJudgeMetrics(sb *strings.Builder, in Input) {
+	sb.WriteString("| Case | Title | Configuration | Status | Agent Time | Input Tokens | Output Tokens | Agent Tokens | Judge Time | Judge Tokens | Turns |\n")
+	sb.WriteString("|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|\n")
 	for _, cr := range in.CaseResults {
-		fmt.Fprintf(sb, "| %s | %s | %s | %s | %d |\n",
+		fmt.Fprintf(sb, "| %s | %s | %s | %s | %s | %d | %d | %d | %s | %d | %d |\n",
 			markdownTableCell(cr.CaseID),
 			markdownTableCell(cr.Title),
+			markdownTableCell(cr.Configuration),
 			markdownTableCell(string(cr.Status)),
 			markdownDuration(cr.DurationMs),
+			cr.InputTokens,
+			cr.OutputTokens,
+			cr.InputTokens+cr.OutputTokens,
+			markdownDuration(cr.JudgeDurationMs),
+			cr.JudgeInputTokens+cr.JudgeOutputTokens,
 			cr.Turns,
 		)
 	}
 	sb.WriteString("\n")
 }
 
+func markdownHasJudgeMetrics(in Input) bool {
+	for _, cr := range in.CaseResults {
+		if cr.JudgeDurationMs != 0 || cr.JudgeInputTokens != 0 || cr.JudgeOutputTokens != 0 {
+			return true
+		}
+	}
+	return false
+}
+
 func writeMarkdownFailureDetails(sb *strings.Builder, in Input) {
 	var details strings.Builder
 	for _, cr := range in.CaseResults {

internal/report/reporter.go

@@ -22,7 +22,9 @@ type Input struct {
 	StartTime     time.Time        `json:"start_time"`
 	EndTime       time.Time        `json:"end_time"`
 	CaseResults   []CaseResult     `json:"case_results"`
-	TotalTokens   int              `json:"total_tokens"`
+	TotalTokens   int              `json:"total_tokens"` // tested-agent tokens across all configurations
+	JudgeTokens   int              `json:"judge_tokens"`
+	OverallTokens int              `json:"overall_tokens"`
 	Benchmark     *BenchmarkResult `json:"benchmark,omitempty"`
 }
 
@@ -99,20 +101,23 @@ func (in Input) PrimaryCaseResults() []CaseResult {
 
 // CaseResult represents the result of a single case execution.
 type CaseResult struct {
-	CaseID        string            `json:"case_id"`
-	Title         string            `json:"title"`
-	Status        judge.Status      `json:"status"`
-	DurationMs    int64             `json:"duration_ms"`
-	Turns         int               `json:"turns"`
-	InputTokens   int               `json:"input_tokens"`
-	OutputTokens  int               `json:"output_tokens"`
-	Error         string            `json:"error,omitempty"`
-	Grading       *judge.Result     `json:"grading"`
-	JudgeSkills   []judge.SkillInfo `json:"judge_skills,omitempty"`
-	Configuration string            `json:"configuration,omitempty"` // "with_skill" or "without_skill"
-	Prompt        string            `json:"prompt,omitempty"`        // input prompt sent to the agent
-	Response      string            `json:"response,omitempty"`      // agent final message
-	TurnResults   []CaseTurnResult  `json:"turn_results,omitempty"`  // per-turn outcomes; nil for single-turn
+	CaseID            string            `json:"case_id"`
+	Title             string            `json:"title"`
+	Status            judge.Status      `json:"status"`
+	DurationMs        int64             `json:"duration_ms"`
+	Turns             int               `json:"turns"`
+	InputTokens       int               `json:"input_tokens"`
+	OutputTokens      int               `json:"output_tokens"`
+	JudgeDurationMs   int64             `json:"judge_duration_ms,omitempty"`
+	JudgeInputTokens  int               `json:"judge_input_tokens,omitempty"`
+	JudgeOutputTokens int               `json:"judge_output_tokens,omitempty"`
+	Error             string            `json:"error,omitempty"`
+	Grading           *judge.Result     `json:"grading"`
+	JudgeSkills       []judge.SkillInfo `json:"judge_skills,omitempty"`
+	Configuration     string            `json:"configuration,omitempty"` // "with_skill" or "without_skill"
+	Prompt            string            `json:"prompt,omitempty"`        // input prompt sent to the agent
+	Response          string            `json:"response,omitempty"`      // agent final message
+	TurnResults       []CaseTurnResult  `json:"turn_results,omitempty"`  // per-turn outcomes; nil for single-turn
 }
 
 // CaseTurnResult holds the outcome of a single turn for reporting purposes.