Add storyboard contact sheet flow and runtime prereq surfacing

README.md

@@ -100,6 +100,7 @@ Each session produces a timestamped folder in `./proofshot-artifacts/`:
 | `viewer.html` | Standalone interactive viewer with scrub bar, timeline, and Console/Server log tabs |
 | `SUMMARY.md` | Markdown report with errors, screenshots, and video |
 | `step-*.png` | Screenshots captured at key moments |
+| `storyboard.png` / `storyboard-scenes.json` | Optional FFmpeg storyboard contact sheet plus scene metadata |
 | `session-log.json` | Action timeline with timestamps and element data |
 | `server.log` | Dev server stdout/stderr (when using `--run`) |
 | `console-output.log` | Browser console output |
@@ -165,8 +166,12 @@ Stop recording, collect errors, generate proof artifacts.
 ```bash
 proofshot stop              # Stop session and close browser
 proofshot stop --no-close   # Stop but keep browser open
+proofshot stop --storyboard # Also generate a storyboard contact sheet
 ```
 
+Storyboard generation is opt-in. It uses FFmpeg scene detection, writes `storyboard.png` plus `storyboard-scenes.json`, leaves `session-log.json` unchanged, and records fallback mode explicitly when no scenes are found.
+Use `proofshot storyboard --input ./proofshot-artifacts/<session-dir>` to regenerate a storyboard from an existing session.
+
 ### `proofshot exec`
 
 Pass-through to agent-browser with automatic session logging. Captures timestamps, element data, and resolves screenshot paths.
@@ -188,7 +193,7 @@ proofshot diff --baseline ./previous-artifacts
 
 ### `proofshot pr`
 
-Upload session artifacts to GitHub and post a verification comment on the PR. Finds all sessions recorded on the current branch, uploads screenshots and video, and posts a formatted comment with embedded screenshots.
+Upload session artifacts to GitHub and post a verification comment on the PR. Finds all sessions recorded on the current branch, uploads screenshots, video, and storyboard contact sheets when present, and posts a formatted comment with embedded proof artifacts.
 
 ```bash
 proofshot pr              # Auto-detect PR from current branch

src/artifacts/pr-format.ts

@@ -100,18 +100,29 @@ export function formatPRComment(data: PRCommentData): string {
   }
 
   // Screenshots
-  if (data.screenshots.size > 0) {
+  const screenshotEntries = [...data.screenshots.entries()];
+  const storyboardEntry = screenshotEntries.find(([filename]) => isStoryboardFile(filename));
+  const regularScreenshots = screenshotEntries.filter(([filename]) => !isStoryboardFile(filename));
+
+  if (storyboardEntry) {
+    const [filename, url] = storyboardEntry;
+    const label = storyboardLabel(filename);
+    md += `### Storyboard\n\n`;
+    md += `![${label}](${url})\n\n`;
+  }
+
+  if (regularScreenshots.length > 0) {
     md += `### Screenshots\n\n`;
 
-    if (data.screenshots.size <= 3) {
-      for (const [filename, url] of data.screenshots) {
+    if (regularScreenshots.length <= 3) {
+      for (const [filename, url] of regularScreenshots) {
         const label = filename.replace(/\.png$/, '').replace(/^step-/, '');
         md += `**${label}**\n\n`;
         md += `![${label}](${url})\n\n`;
       }
     } else {
-      md += `<details>\n<summary>View ${data.screenshots.size} screenshots</summary>\n\n`;
-      for (const [filename, url] of data.screenshots) {
+      md += `<details>\n<summary>View ${regularScreenshots.length} screenshots</summary>\n\n`;
+      for (const [filename, url] of regularScreenshots) {
         const label = filename.replace(/\.png$/, '').replace(/^step-/, '');
         md += `**${label}**\n\n![${label}](${url})\n\n`;
       }
@@ -132,3 +143,11 @@ export function formatPRComment(data: PRCommentData): string {
 
   return md;
 }
+
+function isStoryboardFile(filename: string): boolean {
+  return path.basename(filename).toLowerCase().startsWith('storyboard');
+}
+
+function storyboardLabel(filename: string): string {
+  return path.basename(filename).replace(/\.png$/i, '');
+}

src/artifacts/storyboard.test.ts

@@ -0,0 +1,115 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { findExecutablePathMock, spawnSyncMock } = vi.hoisted(() => ({
+  findExecutablePathMock: vi.fn(),
+  spawnSyncMock: vi.fn(),
+}));
+
+vi.mock('../utils/process.js', () => ({
+  findExecutablePath: findExecutablePathMock,
+}));
+
+vi.mock('child_process', () => ({
+  spawnSync: spawnSyncMock,
+}));
+
+import { generateStoryboardArtifact } from './storyboard.js';
+
+describe('generateStoryboardArtifact', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+    findExecutablePathMock.mockImplementation((command: string) =>
+      command === 'ffprobe' ? '/usr/bin/ffprobe' : '/usr/bin/ffmpeg',
+    );
+    spawnSyncMock.mockReset();
+  });
+
+  function makeSessionDir(name: string): string {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), `proofshot-${name}-`));
+    const sessionDir = path.join(root, 'session');
+    fs.mkdirSync(sessionDir, { recursive: true });
+    fs.writeFileSync(path.join(sessionDir, 'session.webm'), '');
+    return sessionDir;
+  }
+
+  it('writes real scene timestamps when ffmpeg reports cuts', () => {
+    const sessionDir = makeSessionDir('storyboard-scene');
+    const outputPath = path.join(sessionDir, 'storyboard.png');
+
+    spawnSyncMock.mockImplementation((file: string, args: string[]) => {
+      if (args.some((arg) => arg.includes('showinfo'))) {
+        return {
+          stdout: '',
+          stderr:
+            '[Parsed_showinfo_0 @ 0x0] n: 1 pts: 100 pts_time:1.0 pos:0\n' +
+            '[Parsed_showinfo_0 @ 0x0] n: 2 pts: 234 pts_time:2.3 pos:0\n' +
+            '[Parsed_showinfo_0 @ 0x0] n: 3 pts: 345 pts_time:3.4 pos:0\n' +
+            '[Parsed_showinfo_0 @ 0x0] n: 4 pts: 456 pts_time:4.6 pos:0\n',
+          status: 0,
+          error: undefined,
+        } as never;
+      }
+
+      fs.writeFileSync(outputPath, 'png');
+      return { stdout: '', stderr: '', status: 0, error: undefined } as never;
+    });
+
+    const result = generateStoryboardArtifact({ inputDir: sessionDir, outputPath });
+    const storyboard = JSON.parse(fs.readFileSync(path.join(sessionDir, 'storyboard-scenes.json'), 'utf8'));
+
+    expect(result.imagePath).toBe(outputPath);
+    expect(storyboard.mode).toBe('scene');
+    expect(storyboard.scenes).toEqual([
+      { label: 'scene-001', timeSec: 1 },
+      { label: 'scene-002', timeSec: 2.3 },
+      { label: 'scene-003', timeSec: 3.4 },
+      { label: 'scene-004', timeSec: 4.6 },
+    ]);
+    expect(spawnSyncMock.mock.calls[0][1].join(' ')).toContain('showinfo');
+    expect(spawnSyncMock.mock.calls[1][1].join(' ')).toContain("select='gt(scene");
+  });
+
+  it('falls back immediately when no scenes are detected', () => {
+    const sessionDir = makeSessionDir('storyboard-fallback');
+    const outputPath = path.join(sessionDir, 'storyboard.png');
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+
+    spawnSyncMock.mockImplementation((file: string, args: string[]) => {
+      if (args.some((arg) => arg.includes('showinfo'))) {
+        return { stdout: '', stderr: '', status: 0, error: undefined } as never;
+      }
+
+      if (args.includes('-show_entries')) {
+        return { stdout: '40.0\n', stderr: '', status: 0, error: undefined } as never;
+      }
+
+      fs.writeFileSync(outputPath, 'png');
+      return { stdout: '', stderr: '', status: 0, error: undefined } as never;
+    });
+
+    generateStoryboardArtifact({ inputDir: sessionDir, outputPath });
+    const storyboard = JSON.parse(fs.readFileSync(path.join(sessionDir, 'storyboard-scenes.json'), 'utf8'));
+
+    expect(storyboard.mode).toBe('fallback');
+    expect(storyboard.scenes).toHaveLength(20);
+    expect(storyboard.scenes[0]).toEqual({ label: 'sample-001', timeSec: 1 });
+    expect(storyboard.scenes[19]).toEqual({ label: 'sample-020', timeSec: 39 });
+    expect(spawnSyncMock.mock.calls).toHaveLength(3);
+    expect(spawnSyncMock.mock.calls[2][1].join(' ')).toContain('fps=0.5');
+    expect(spawnSyncMock.mock.calls[2][1].join(' ')).not.toContain('select=gt(scene');
+    expect(logSpy.mock.calls.flat().join('\n')).toContain('Storyboard fallback');
+  });
+
+  it('skips cleanly when ffmpeg is unavailable', () => {
+    findExecutablePathMock.mockReturnValue(null);
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+
+    const result = generateStoryboardArtifact({ inputDir: '/tmp/does-not-matter' });
+
+    expect(result).toEqual({ imagePath: null, jsonPath: null });
+    expect(logSpy.mock.calls.flat().join('\n')).toContain('Storyboard unavailable');
+  });
+});

src/artifacts/storyboard.ts

@@ -0,0 +1,171 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { spawnSync } from 'child_process';
+import chalk from 'chalk';
+import { findExecutablePath } from '../utils/process.js';
+
+interface StoryboardOptions {
+  inputDir: string;
+  outputPath?: string;
+  threshold?: number;
+  grid?: string;
+  width?: number;
+}
+
+const DEFAULT_THRESHOLD = 0.3;
+const DEFAULT_GRID = '4x5';
+const DEFAULT_WIDTH = 1600;
+const MIN_SCENES_FOR_SCENE_MODE = 4;
+const MAX_SCENES = 20;
+
+function parseSceneTimes(text: string): number[] {
+  const seen = new Set<string>();
+  const times: number[] = [];
+  for (const match of text.matchAll(/pts_time:([0-9]+(?:\.[0-9]+)?)/g)) {
+    const value = Number(match[1]);
+    if (!Number.isFinite(value) || value < 0) continue;
+    const time = Math.round(value * 10) / 10;
+    const key = time.toFixed(1);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    times.push(time);
+    if (times.length >= MAX_SCENES) break;
+  }
+  return times;
+}
+
+function scenesFromTimes(times: number[]) {
+  return times.map((timeSec, i) => ({ label: `scene-${String(i + 1).padStart(3, '0')}`, timeSec }));
+}
+
+function sampledFramesFromDuration(durationSec: number): { label: string; timeSec: number }[] {
+  if (!Number.isFinite(durationSec) || durationSec <= 0) return [];
+
+  return Array.from({ length: MAX_SCENES }, (_, i) => ({
+    label: `sample-${String(i + 1).padStart(3, '0')}`,
+    timeSec: Number(((durationSec * (i + 0.5)) / MAX_SCENES).toFixed(1)),
+  }));
+}
+
+function runFfmpeg(ffmpeg: string, args: string[]): string {
+  const result = spawnSync(ffmpeg, args, {
+    encoding: 'utf-8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    env: process.env,
+  });
+  const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim();
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error(output || `Command failed: ${ffmpeg}`);
+  return output;
+}
+
+function renderFilter(
+  threshold: number,
+  grid: string,
+  width: number,
+  options: { useSceneSelect: boolean; fps?: number | null },
+): string {
+  const cols = Number.parseInt(grid, 10) || 4;
+  const thumbWidth = Math.max(1, Math.floor(width / cols));
+  const fpsPrefix = options.fps ? `fps=${options.fps},` : '';
+  const prefix = options.useSceneSelect ? `select='gt(scene\\,${threshold})',` : fpsPrefix;
+  return `${prefix}scale=${thumbWidth}:-1:flags=lanczos,tile=${grid}`;
+}
+
+function readVideoDuration(ffprobe: string, videoPath: string): number | null {
+  try {
+    const output = runFfmpeg(ffprobe, [
+      '-v',
+      'error',
+      '-show_entries',
+      'format=duration',
+      '-of',
+      'default=noprint_wrappers=1:nokey=1',
+      videoPath,
+    ]).trim();
+    const duration = Number(output.split(/\r?\n/)[0]);
+    return Number.isFinite(duration) && duration > 0 ? duration : null;
+  } catch {
+    return null;
+  }
+}
+
+export function generateStoryboardArtifact(options: StoryboardOptions): {
+  imagePath: string | null;
+  jsonPath: string | null;
+} {
+  const ffmpeg = findExecutablePath('ffmpeg');
+  if (!ffmpeg) {
+    console.log(chalk.dim('Storyboard unavailable: install ffmpeg to generate storyboard artifacts.'));
+    return { imagePath: null, jsonPath: null };
+  }
+  const ffprobe = findExecutablePath('ffprobe');
+
+  const inputDir = path.resolve(options.inputDir);
+  const videoPath = ['session.webm', 'session.mp4', 'session.mov']
+    .map((name) => path.join(inputDir, name))
+    .find((candidate) => fs.existsSync(candidate));
+
+  if (!videoPath) throw new Error(`No session video found in ${inputDir}`);
+
+  const outputPath = path.resolve(options.outputPath || path.join(inputDir, 'storyboard.png'));
+  const threshold = options.threshold ?? DEFAULT_THRESHOLD;
+  const grid = options.grid ?? DEFAULT_GRID;
+  const width = options.width ?? DEFAULT_WIDTH;
+  const sceneOutput = runFfmpeg(ffmpeg, ['-hide_banner', '-i', videoPath, '-vf', `select='gt(scene\\,${threshold})',showinfo`, '-an', '-f', 'null', '-']);
+  let scenes = scenesFromTimes(parseSceneTimes(sceneOutput));
+  let mode: 'scene' | 'fallback' = scenes.length >= MIN_SCENES_FOR_SCENE_MODE ? 'scene' : 'fallback';
+  const durationSec = mode === 'fallback' && ffprobe ? readVideoDuration(ffprobe, videoPath) : null;
+  if (mode === 'fallback' && durationSec) {
+    scenes = sampledFramesFromDuration(durationSec);
+  }
+
+  const artifactPath = path.join(inputDir, 'storyboard-scenes.json');
+  const writeArtifact = (): void => {
+    fs.writeFileSync(
+      artifactPath,
+      JSON.stringify(
+        {
+          videoPath: path.basename(videoPath),
+          mode,
+          scenes,
+          threshold,
+          grid,
+          width,
+          source: 'ffmpeg',
+        },
+        null,
+        2,
+      ) + '\n',
+    );
+  };
+
+  const render = (useSceneSelect: boolean): void => {
+    runFfmpeg(ffmpeg, [
+      '-hide_banner',
+      '-i',
+      videoPath,
+      '-vf',
+      renderFilter(threshold, grid, width, {
+        useSceneSelect,
+        fps: !useSceneSelect && durationSec ? Number((MAX_SCENES / durationSec).toFixed(3)) : null,
+      }),
+      '-frames:v',
+      '1',
+      outputPath,
+    ]);
+  };
+
+  if (mode === 'scene') {
+    render(true);
+  } else {
+    render(false);
+  }
+
+  if (mode === 'fallback') {
+    console.log(chalk.dim('Storyboard fallback: no scene cuts detected; using a plain tile sheet.'));
+  }
+
+  writeArtifact();
+  return { imagePath: outputPath, jsonPath: artifactPath };
+}