From 73546d592295449c238f5d2bff40b452e2cceb82 Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 04:21:08 +0000 Subject: [PATCH 01/32] refactor: overhaul progress tracking with cumulative stats and simplified stream status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify stream lifecycle to just started/complete β€” errors are now orthogonal (a stream can be complete with errors). Replace interval-based progress emission with event-driven emission on status transitions. Add cumulative tracking across runs (record count, request count, elapsed time) persisted via engine state. Rename TraceProgress β†’ TraceGlobalProgress and rows_per_second β†’ records_per_second for consistency. Emit catalog as first message so the UI knows all streams upfront. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/dashboard/src/pages/PipelineDetail.tsx | 9 +- apps/engine/src/api/app.ts | 139 +++++++------ apps/engine/src/lib/engine.test.ts | 4 +- apps/engine/src/lib/engine.ts | 2 + apps/engine/src/lib/pipeline.test.ts | 4 +- apps/engine/src/lib/progress.test.ts | 183 ++++++++++++++++-- apps/engine/src/lib/progress.ts | 141 +++++++++----- .../src/temporal/activities/_shared.ts | 9 +- .../temporal/workflows/pipeline-workflow.ts | 3 +- packages/protocol/src/helpers.ts | 11 +- packages/protocol/src/index.ts | 1 + packages/protocol/src/protocol.ts | 107 +++++----- packages/source-stripe/src/transport.test.ts | 1 + 13 files changed, 412 insertions(+), 202 deletions(-) diff --git a/apps/dashboard/src/pages/PipelineDetail.tsx b/apps/dashboard/src/pages/PipelineDetail.tsx index 1aed546a..281deaa8 100644 --- a/apps/dashboard/src/pages/PipelineDetail.tsx +++ b/apps/dashboard/src/pages/PipelineDetail.tsx @@ -13,15 +13,14 @@ interface StreamProgress { status: string cumulative_record_count: number run_record_count: number - records_per_second?: number errors?: Array<{ message: string; failure_type?: string }> } interface GlobalProgress { elapsed_ms: number run_record_count: number - rows_per_second: number - window_rows_per_second: number + records_per_second: number + window_records_per_second: number state_checkpoint_count: number } @@ -201,11 +200,11 @@ export function PipelineDetail({ id, onBack }: PipelineDetailProps) { diff --git a/apps/engine/src/api/app.ts b/apps/engine/src/api/app.ts index 79221dd0..586e805c 100644 --- a/apps/engine/src/api/app.ts +++ b/apps/engine/src/api/app.ts @@ -131,89 +131,116 @@ const REASON_EMOJI: Record = { aborted: 'πŸ›‘', } -const STATUS_EMOJI: Record = { - complete: 'βœ…', - started: 'πŸ”„', - running: 'πŸ”„', +const ERROR_EMOJI: Record = { transient_error: '⚠️', system_error: '❌', - config_error: '❌', + config_error: 'βš™οΈ', auth_error: 'πŸ”’', } +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms` + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s` + const mins = Math.floor(ms / 60_000) + const secs = Math.round((ms % 60_000) / 1000) + if (mins < 60) return secs > 0 ? `${mins}m ${secs}s` : `${mins}m` + const hrs = Math.floor(mins / 60) + const remainMins = mins % 60 + return remainMins > 0 ? `${hrs}h ${remainMins}m` : `${hrs}h` +} + +function formatNumber(n: number): string { + return n.toLocaleString('en-US') +} + function formatEof(eof: EofPayload): string { const emoji = REASON_EMOJI[eof.reason] ?? '❓' - const elapsed = eof.global_progress?.elapsed_ms - ? `${(eof.global_progress.elapsed_ms / 1000).toFixed(1)}s` - : '' - const totalRows = eof.global_progress?.run_record_count ?? 0 - const rps = eof.global_progress?.rows_per_second?.toFixed(1) ?? '0' - const checkpoints = eof.global_progress?.state_checkpoint_count ?? 0 + const gp = eof.global_progress + const runRecords = gp?.run_record_count ?? 0 + const cumulativeRecords = gp?.cumulative_record_count ?? runRecords + const rps = gp?.records_per_second?.toFixed(1) ?? '0' + const runElapsed = gp?.elapsed_ms ?? 0 + const cumulativeElapsed = gp?.cumulative_elapsed_ms ?? runElapsed + const cumulativeRequests = gp?.cumulative_request_count ?? 0 + const runRequests = gp?.request_count ?? 0 const lines: string[] = [] + lines.push(`${emoji} Sync ${eof.reason}`) + lines.push( + ` Total: ${formatNumber(cumulativeRecords)} records | ${formatNumber(cumulativeRequests)} requests | ${formatDuration(cumulativeElapsed)}` + ) lines.push( - `${emoji} Sync ${eof.reason}${elapsed ? ` (${elapsed}` : ''}${totalRows ? ` | ${totalRows} rows, ${rps} rows/s` : ''}${checkpoints ? `, ${checkpoints} checkpoints` : ''}${elapsed ? ')' : ''}` + ` This run: +${formatNumber(runRecords)} records | ${formatNumber(runRequests)} requests | ${formatDuration(runElapsed)} | ${rps} records/s` ) const sp = eof.stream_progress if (sp) { - let complete = 0 - let inProgress = 0 - let errored = 0 - let pending = 0 - const errorStreams: string[] = [] - const activeStreams: { name: string; rows: number; rps: string }[] = [] + type StreamEntry = { name: string; cumulative: number; run: number; errors: StreamError[] } + type StreamError = { message: string; failure_type?: string } + const completeStreams: StreamEntry[] = [] + const startedStreams: StreamEntry[] = [] + let errorsCount = 0 for (const [name, s] of Object.entries(sp)) { + const entry: StreamEntry = { + name, + cumulative: s.cumulative_record_count, + run: s.run_record_count, + errors: s.errors ?? [], + } + if (entry.errors.length > 0) errorsCount++ if (s.status === 'complete') { - complete++ - if (s.run_record_count > 0) { - activeStreams.push({ - name, - rows: s.run_record_count, - rps: s.records_per_second?.toFixed(1) ?? '0', - }) - } - } else if (s.status === 'started' || s.status === 'running') { - inProgress++ - if (s.run_record_count > 0) { - activeStreams.push({ - name, - rows: s.run_record_count, - rps: s.records_per_second?.toFixed(1) ?? '0', - }) - } - } else if ( - s.status === 'transient_error' || - s.status === 'system_error' || - s.status === 'config_error' || - s.status === 'auth_error' - ) { - errored++ - const errMsg = s.errors?.[0]?.message ?? s.status - errorStreams.push(`${STATUS_EMOJI[s.status]} ${name}: ${errMsg}`) + completeStreams.push(entry) } else { - pending++ + startedStreams.push(entry) } } - // Show streams that synced rows this run - for (const s of activeStreams.sort((a, b) => b.rows - a.rows)) { - lines.push(` βœ… ${s.name}: ${s.rows} rows @ ${s.rps} rows/s`) + // Sort by cumulative record count descending + completeStreams.sort((a, b) => b.cumulative - a.cumulative) + startedStreams.sort((a, b) => b.cumulative - a.cumulative) + + const maxNameLen = Math.max(...Object.keys(sp).map((n) => n.length), 10) + + function formatStreamLine(entry: StreamEntry): string[] { + const result: string[] = [] + const countStr = + entry.cumulative > 0 + ? `${formatNumber(entry.cumulative).padStart(10)}${entry.run > 0 ? ` (+${formatNumber(entry.run)})` : ''}` + : '' + result.push(` ${entry.name.padEnd(maxNameLen)} ${countStr}`) + for (const err of entry.errors) { + const errEmoji = ERROR_EMOJI[err.failure_type ?? 'system_error'] ?? '❌' + result.push( + ` ${errEmoji} ${err.message}${err.failure_type ? ` (${err.failure_type})` : ''}` + ) + } + return result } - // Show errored streams - for (const e of errorStreams) { - lines.push(` ${e}`) + if (completeStreams.length > 0) { + lines.push('') + lines.push(` Complete (${completeStreams.length}):`) + for (const s of completeStreams) { + lines.push(...formatStreamLine(s)) + } + } + + if (startedStreams.length > 0) { + lines.push('') + lines.push(` Started (${startedStreams.length}):`) + for (const s of startedStreams) { + lines.push(...formatStreamLine(s)) + } } // Summary line + lines.push('') const parts: string[] = [] - if (complete) parts.push(`${complete} complete`) - if (inProgress) parts.push(`${inProgress} in progress`) - if (errored) parts.push(`${errored} errored`) - if (pending) parts.push(`${pending} pending`) - parts.push(`${totalRows} total rows this run`) + if (completeStreams.length) parts.push(`${completeStreams.length} complete`) + if (startedStreams.length) parts.push(`${startedStreams.length} started`) + if (errorsCount) parts.push(`${errorsCount} streams with errors`) + parts.push(`+${formatNumber(runRecords)} records this run`) lines.push(` πŸ“Š ${parts.join(', ')}`) } diff --git a/apps/engine/src/lib/engine.test.ts b/apps/engine/src/lib/engine.test.ts index 0f46285f..b50a2cde 100644 --- a/apps/engine/src/lib/engine.test.ts +++ b/apps/engine/src/lib/engine.test.ts @@ -252,7 +252,7 @@ describe('protocol schemas', () => { trace_type: 'stream_status', stream_status: { stream: 'customers', - status: 'running', + status: 'started', }, }, }) @@ -1072,7 +1072,7 @@ describe('engine.pipeline_sync() pipeline', () => { trace_type: 'stream_status' as const, stream_status: { stream: 'customers', - status: 'running' as const, + status: 'started' as const, }, }, } diff --git a/apps/engine/src/lib/engine.ts b/apps/engine/src/lib/engine.ts index 91a8bd9d..9f742590 100644 --- a/apps/engine/src/lib/engine.ts +++ b/apps/engine/src/lib/engine.ts @@ -14,6 +14,7 @@ import { SyncState, RecordMessage, SourceStateMessage, + EofMessage, coerceSyncState, collectFirst, split, @@ -578,6 +579,7 @@ export async function createEngine(resolver: ConnectorResolver): Promise yield* trackProgress({ initial_state: normalizedState, + catalog: filteredCatalog, recordCounter, })(limited) })() diff --git a/apps/engine/src/lib/pipeline.test.ts b/apps/engine/src/lib/pipeline.test.ts index 752827e1..d5a43887 100644 --- a/apps/engine/src/lib/pipeline.test.ts +++ b/apps/engine/src/lib/pipeline.test.ts @@ -304,13 +304,13 @@ describe('log()', () => { type: 'trace', trace: { trace_type: 'stream_status', - stream_status: { stream: 'orders', status: 'running' }, + stream_status: { stream: 'orders', status: 'started' }, }, }, ] await drain(log(toAsync(msgs))) expect(logger.info).toHaveBeenCalledWith( - { stream: 'orders', status: 'running' }, + { stream: 'orders', status: 'started' }, 'stream_status' ) }) diff --git a/apps/engine/src/lib/progress.test.ts b/apps/engine/src/lib/progress.test.ts index 896b8401..2212ab07 100644 --- a/apps/engine/src/lib/progress.test.ts +++ b/apps/engine/src/lib/progress.test.ts @@ -72,8 +72,14 @@ describe('trackProgress', () => { const outputs = await collect( trackProgress({ - interval_ms: 0, - initial_cumulative_counts: { customers: 5 }, + initial_state: { + source: { streams: {}, global: {} }, + destination: { streams: {}, global: {} }, + engine: { + streams: { customers: { cumulative_record_count: 5 } }, + global: {}, + }, + }, recordCounter: counter, })( toAsync([ @@ -100,10 +106,10 @@ describe('trackProgress', () => { ) ) - const progressTraces = outputs.filter( - (m) => m.type === 'trace' && m.trace.trace_type === 'progress' + const globalProgressTraces = outputs.filter( + (m) => m.type === 'trace' && m.trace.trace_type === 'global_progress' ) - expect(progressTraces.length).toBeGreaterThan(0) + expect(globalProgressTraces.length).toBeGreaterThan(0) const eof = outputs.find((m) => m.type === 'eof') expect(eof).toBeDefined() @@ -118,7 +124,7 @@ describe('trackProgress', () => { }, destination: { streams: {}, global: {} }, engine: { - streams: { customers: { cumulative_record_count: 7 } }, + streams: { customers: { cumulative_record_count: 7, status: 'complete' } }, global: {}, }, }, @@ -138,6 +144,137 @@ describe('trackProgress', () => { }) }) + it('emits stream_status only on transitions, not periodically', async () => { + const counter = createRecordCounter() + const outputs = await collect( + trackProgress({ + recordCounter: counter, + })( + toAsync([ + { + type: 'source_state', + source_state: { state_type: 'stream', stream: 'customers', data: { cursor: '1' } }, + }, + // Second source_state for same stream should NOT emit another stream_status + { + type: 'source_state', + source_state: { state_type: 'stream', stream: 'customers', data: { cursor: '2' } }, + }, + { + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }, + { type: 'eof', eof: { reason: 'complete' } }, + ]) + ) + ) + + const streamStatusTraces = outputs.filter( + (m) => + m.type === 'trace' && m.trace.trace_type === 'stream_status' && m._emitted_by === 'engine' + ) + // First source_state β†’ started transition + complete transition + final on EOF = 3 + // The second source_state should NOT trigger another (already started) + const statusValues = streamStatusTraces.map((m) => (m as any).trace.stream_status.status) + // started (from first source_state), complete (from stream_status trace), complete (final on EOF) + expect(statusValues).toEqual(['started', 'complete', 'complete']) + }) + + it('co-emits global_progress with every stream_status', async () => { + const counter = createRecordCounter() + const outputs = await collect( + trackProgress({ + recordCounter: counter, + })( + toAsync([ + { + type: 'source_state', + source_state: { state_type: 'stream', stream: 'customers', data: { cursor: '1' } }, + }, + { type: 'eof', eof: { reason: 'complete' } }, + ]) + ) + ) + + // Every stream_status from engine should be followed by a global_progress + const engineTraces = outputs.filter((m) => m.type === 'trace' && m._emitted_by === 'engine') + for (let i = 0; i < engineTraces.length - 1; i++) { + const current = engineTraces[i] as any + const next = engineTraces[i + 1] as any + if (current.trace.trace_type === 'stream_status') { + expect(next.trace.trace_type).toBe('global_progress') + } + } + }) + + it('emits catalog as first message when provided', async () => { + const counter = createRecordCounter() + const outputs = await collect( + trackProgress({ + recordCounter: counter, + catalog: { + streams: [ + { + stream: { name: 'customers', primary_key: [['id']] }, + sync_mode: 'incremental', + destination_sync_mode: 'append', + }, + ], + }, + })(toAsync([{ type: 'eof', eof: { reason: 'complete' } }])) + ) + + expect(outputs[0]).toMatchObject({ + type: 'catalog', + catalog: { streams: [{ name: 'customers', primary_key: [['id']] }] }, + }) + }) + + it('errors are orthogonal to lifecycle status', async () => { + const counter = createRecordCounter() + const outputs = await collect( + trackProgress({ + recordCounter: counter, + })( + toAsync([ + { + type: 'source_state', + source_state: { state_type: 'stream', stream: 'customers', data: { cursor: '1' } }, + }, + { + type: 'trace', + trace: { + trace_type: 'error', + error: { + message: 'rate limited', + failure_type: 'transient_error', + stream: 'customers', + }, + }, + }, + { + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }, + { type: 'eof', eof: { reason: 'complete' } }, + ]) + ) + ) + + const eof = outputs.find((m) => m.type === 'eof') as any + // Stream is complete AND has errors β€” they're orthogonal + expect(eof.eof.stream_progress.customers.status).toBe('complete') + expect(eof.eof.stream_progress.customers.errors).toEqual([ + { message: 'rate limited', failure_type: 'transient_error' }, + ]) + }) + it('aggregates multiple stream states and global state into EOF', async () => { const counter = createRecordCounter() await collect( @@ -165,7 +302,6 @@ describe('trackProgress', () => { const outputs = await collect( trackProgress({ - interval_ms: 0, recordCounter: counter, })( toAsync([ @@ -210,8 +346,8 @@ describe('trackProgress', () => { destination: { streams: {}, global: {} }, engine: { streams: { - customers: { cumulative_record_count: 1 }, - invoices: { cumulative_record_count: 1 }, + customers: { cumulative_record_count: 1, status: 'started' }, + invoices: { cumulative_record_count: 1, status: 'started' }, }, global: {}, }, @@ -239,7 +375,6 @@ describe('trackProgress', () => { const outputs = await collect( trackProgress({ - interval_ms: 0, initial_state: { source: { streams: { @@ -294,10 +429,9 @@ describe('trackProgress', () => { }, engine: { streams: { - customers: { cumulative_record_count: 6, note: 'keep-me' }, + customers: { cumulative_record_count: 6, note: 'keep-me', status: 'started' }, invoices: { cumulative_record_count: 2, untouched: true }, }, - global: { sync_id: 'prev' }, }, }, }, @@ -322,30 +456,43 @@ describe('trackProgress', () => { const outputs = await collect( trackProgress({ - interval_ms: 0, initial_state: initialState, recordCounter: createRecordCounter(), })(toAsync([{ type: 'eof', eof: { reason: 'complete' } }])) ) const eof = outputs.find((m) => m.type === 'eof') + // Engine global is enriched with cumulative totals, so partial match expect(eof).toMatchObject({ type: 'eof', - eof: { state: initialState }, + eof: { + state: { + source: initialState.source, + destination: initialState.destination, + engine: { + streams: initialState.engine.streams, + }, + }, + }, }) }) - it('omits state from EOF when no source_state messages were emitted', async () => { + it('includes engine global cumulative stats even when no source_state messages were emitted', async () => { const counter = createRecordCounter() const outputs = await collect( trackProgress({ - interval_ms: 0, recordCounter: counter, })(toAsync([{ type: 'eof', eof: { reason: 'complete' } }])) ) - const eof = outputs.find((m) => m.type === 'eof') + const eof = outputs.find((m) => m.type === 'eof') as any expect(eof).toBeDefined() - expect((eof as any).eof.state).toBeUndefined() + // Engine global always has cumulative stats (zeroed out for fresh runs) + expect(eof.eof.state.engine.global).toMatchObject({ + cumulative_record_count: 0, + cumulative_request_count: 0, + }) + // No source or destination state since no messages were emitted + expect(Object.keys(eof.eof.state.source.streams)).toHaveLength(0) }) }) diff --git a/apps/engine/src/lib/progress.ts b/apps/engine/src/lib/progress.ts index b316323c..aad24c0a 100644 --- a/apps/engine/src/lib/progress.ts +++ b/apps/engine/src/lib/progress.ts @@ -3,9 +3,10 @@ import type { SyncState, SyncOutput, TraceStreamStatus, - TraceProgress, + TraceGlobalProgress, EofPayload, EofStreamProgress, + ConfiguredCatalog, } from '@stripe/sync-protocol' import { emptySyncState } from '@stripe/sync-protocol' @@ -37,15 +38,14 @@ export function createRecordCounter() { } export function trackProgress(opts: { - interval_ms?: number initial_state?: SyncState - initial_cumulative_counts?: Record + /** Configured catalog β€” emitted as the first message so the UI knows all streams upfront. */ + catalog?: ConfiguredCatalog /** Shared counter fed by createRecordCounter().tap() on the data path. */ recordCounter?: ReturnType }): (msgs: AsyncIterable) => AsyncIterable { - const intervalMs = opts.interval_ms ?? 2000 - return async function* (messages) { + // Initialize cumulative counts from engine state const initialCumulativeCounts = opts.initial_state?.engine?.streams ? Object.fromEntries( Object.entries(opts.initial_state.engine.streams) @@ -55,33 +55,59 @@ export function trackProgress(opts: { ]) .filter(([, v]) => typeof v === 'number' && v >= 0) ) - : (opts.initial_cumulative_counts ?? {}) + : {} const cumulativeRecordCount = new Map(Object.entries(initialCumulativeCounts)) - const prevSnapshotCounts = new Map() + + // Initialize cumulative global stats from engine state + const engineGlobal = (opts.initial_state?.engine?.global ?? {}) as Record + let cumulativeGlobalRecordCount = (engineGlobal.cumulative_record_count as number) ?? 0 + let cumulativeRequestCount = (engineGlobal.cumulative_request_count as number) ?? 0 + let cumulativeElapsedMs = (engineGlobal.cumulative_elapsed_ms as number) ?? 0 + let stateCheckpointCount = 0 const streamStatus = new Map() + const lastEmittedStatus = new Map() // Restore stream statuses: engine state first, then source state overrides // (source state is authoritative β€” streams the source skips emit no messages) if (opts.initial_state?.engine?.streams) { for (const [stream, data] of Object.entries(opts.initial_state.engine.streams)) { const status = (data as { status?: Status })?.status - if (status) streamStatus.set(stream, status) + if (status === 'started' || status === 'complete') { + streamStatus.set(stream, status) + } } } if (opts.initial_state?.source?.streams) { for (const [stream, data] of Object.entries(opts.initial_state.source.streams)) { - const status = (data as { status?: string })?.status - if (status) streamStatus.set(stream, status as Status) + const srcStatus = (data as { status?: string })?.status + // Map source error statuses to lifecycle status for the engine + if (srcStatus === 'complete') { + streamStatus.set(stream, 'complete') + } else if ( + srcStatus === 'pending' || + srcStatus === 'transient_error' || + srcStatus === 'system_error' || + srcStatus === 'config_error' || + srcStatus === 'auth_error' + ) { + // Source hasn't completed β€” keep as started (or don't set if not started yet) + if (streamStatus.has(stream)) { + // Already has a status from engine state, keep it unless it was complete + // and source says otherwise + } else if (srcStatus !== 'pending') { + streamStatus.set(stream, 'started') + } + } } } + const streamErrors = new Map() const hadInitialState = opts.initial_state != null const finalState: SyncState = structuredClone(opts.initial_state ?? emptySyncState()) const startedAt = Date.now() let lastWindowAt = startedAt - let lastEmitAt = startedAt let prevWindowTotal = 0 function elapsedMs() { @@ -103,10 +129,6 @@ export function trackProgress(opts: { return sum } - function windowRecordCount(stream: string): number { - return runRecordCount(stream) - (prevSnapshotCounts.get(stream) ?? 0) - } - function totalWindowRecords(): number { return totalRunRecords() - prevWindowTotal } @@ -122,12 +144,8 @@ export function trackProgress(opts: { } function snapshotWindow() { - if (opts.recordCounter) { - for (const [k, v] of opts.recordCounter.counts) prevSnapshotCounts.set(k, v) - } prevWindowTotal = totalRunRecords() lastWindowAt = Date.now() - lastEmitAt = Date.now() } function buildStreamStatus(stream: string): SyncOutput | undefined { @@ -144,8 +162,6 @@ export function trackProgress(opts: { status, cumulative_record_count: cumulative, run_record_count: run, - window_record_count: windowRecordCount(stream), - records_per_second: run / elapsedSec(), }, }, _emitted_by: 'engine', @@ -155,21 +171,38 @@ export function trackProgress(opts: { function buildGlobalProgress(): SyncOutput { const windowDuration = Math.max((Date.now() - lastWindowAt) / 1000, 0.001) - const progress: TraceProgress = { + const runRecords = totalRunRecords() + const globalProgress: TraceGlobalProgress = { elapsed_ms: elapsedMs(), - run_record_count: totalRunRecords(), - rows_per_second: totalRunRecords() / elapsedSec(), - window_rows_per_second: totalWindowRecords() / windowDuration, + run_record_count: runRecords, + cumulative_record_count: cumulativeGlobalRecordCount + runRecords, + records_per_second: runRecords / elapsedSec(), + window_records_per_second: totalWindowRecords() / windowDuration, state_checkpoint_count: stateCheckpointCount, + cumulative_request_count: cumulativeRequestCount, + cumulative_elapsed_ms: cumulativeElapsedMs + elapsedMs(), } return { type: 'trace', - trace: { trace_type: 'progress' as const, progress }, + trace: { trace_type: 'global_progress' as const, global_progress: globalProgress }, _emitted_by: 'engine', _ts: new Date().toISOString(), } as SyncOutput } + /** Emit stream_status + global_progress pair if status changed. */ + function* emitIfStatusChanged(stream: string): Iterable { + const current = streamStatus.get(stream) + if (!current) return + if (lastEmittedStatus.get(stream) === current) return + + lastEmittedStatus.set(stream, current) + const ss = buildStreamStatus(stream) + if (ss) yield ss + yield buildGlobalProgress() + snapshotWindow() + } + function buildStreamProgress(stream: string): EofStreamProgress | undefined { const status = streamStatus.get(stream) if (!status) return undefined @@ -179,7 +212,6 @@ export function trackProgress(opts: { status, cumulative_record_count: cumulative, run_record_count: run, - records_per_second: run / elapsedSec(), errors: streamErrors.has(stream) ? streamErrors.get(stream) : undefined, } } @@ -199,6 +231,15 @@ export function trackProgress(opts: { } } + // Update engine global state with cumulative totals + const runRecords = totalRunRecords() + finalState.engine.global = { + ...finalState.engine.global, + cumulative_record_count: cumulativeGlobalRecordCount + runRecords, + cumulative_request_count: cumulativeRequestCount, + cumulative_elapsed_ms: cumulativeElapsedMs + elapsedMs(), + } + const hasAnyState = Object.keys(finalState.source.streams).length > 0 || Object.keys(finalState.source.global).length > 0 || @@ -218,15 +259,19 @@ export function trackProgress(opts: { const sp = buildStreamProgress(s) if (sp) streamProgressMap[s] = sp } + const runRecords = totalRunRecords() const eof: EofPayload = { reason, state: buildAccumulatedState(), global_progress: { elapsed_ms: elapsedMs(), - run_record_count: totalRunRecords(), - rows_per_second: totalRunRecords() / elapsedSec(), - window_rows_per_second: totalWindowRecords() / windowDuration, + run_record_count: runRecords, + cumulative_record_count: cumulativeGlobalRecordCount + runRecords, + records_per_second: runRecords / elapsedSec(), + window_records_per_second: totalWindowRecords() / windowDuration, state_checkpoint_count: stateCheckpointCount, + cumulative_request_count: cumulativeRequestCount, + cumulative_elapsed_ms: cumulativeElapsedMs + elapsedMs(), }, stream_progress: Object.keys(streamProgressMap).length > 0 ? streamProgressMap : undefined, } @@ -238,16 +283,14 @@ export function trackProgress(opts: { } as SyncOutput } - function* maybeEmitProgress(): Iterable { - const now = Date.now() - if (now - lastEmitAt < intervalMs) return - - for (const stream of allStreams()) { - const ss = buildStreamStatus(stream) - if (ss) yield ss - } - yield buildGlobalProgress() - snapshotWindow() + // Emit catalog as first message so the UI knows all streams upfront + if (opts.catalog) { + yield { + type: 'catalog', + catalog: { streams: opts.catalog.streams.map((cs) => cs.stream) }, + _emitted_by: 'engine', + _ts: new Date().toISOString(), + } as SyncOutput } for await (const msg of messages) { @@ -256,28 +299,35 @@ export function trackProgress(opts: { if (msg.source_state.state_type === 'stream') { const stream = msg.source_state.stream finalState.source.streams[stream] = msg.source_state.data - if (!streamStatus.has(stream)) streamStatus.set(stream, 'started') + if (!streamStatus.has(stream)) { + streamStatus.set(stream, 'started') + yield* emitIfStatusChanged(stream) + } } else if (msg.source_state.state_type === 'global') { finalState.source.global = msg.source_state.data as Record } } else if (msg.type === 'trace') { if (msg.trace.trace_type === 'stream_status') { const ss = msg.trace.stream_status - streamStatus.set(ss.stream, ss.status) + const newStatus = ss.status as Status + // Only accept valid lifecycle statuses + if (newStatus === 'started' || newStatus === 'complete') { + streamStatus.set(ss.stream, newStatus) + yield* emitIfStatusChanged(ss.stream) + } } else if (msg.trace.trace_type === 'error') { const err = msg.trace.error if (err.stream) { const errs = streamErrors.get(err.stream) ?? [] errs.push({ message: err.message, failure_type: err.failure_type as FailureType }) streamErrors.set(err.stream, errs) - if (err.failure_type && streamStatus.get(err.stream) !== 'complete') { - streamStatus.set(err.stream, err.failure_type as Status) - } + // Errors don't change lifecycle status β€” they're orthogonal } } } if (msg.type === 'eof') { + // Emit final stream_status + global_progress for all streams for (const stream of allStreams()) { const ss = buildStreamStatus(stream) if (ss) yield ss @@ -288,7 +338,6 @@ export function trackProgress(opts: { } yield msg - yield* maybeEmitProgress() } } } diff --git a/apps/service/src/temporal/activities/_shared.ts b/apps/service/src/temporal/activities/_shared.ts index f00e4b11..20587659 100644 --- a/apps/service/src/temporal/activities/_shared.ts +++ b/apps/service/src/temporal/activities/_shared.ts @@ -86,12 +86,9 @@ export async function drainMessages( count++ if (message.type === 'eof') { eof = message.eof - if (eof.stream_progress) { - const engineStreams: Record = { ...state.engine.streams } - for (const [name, sp] of Object.entries(eof.stream_progress)) { - engineStreams[name] = { cumulative_record_count: sp.cumulative_record_count } - } - state = { ...state, engine: { ...state.engine, streams: engineStreams } } + // eof.state is authoritative β€” built by trackProgress with full accumulated state + if (eof.state) { + state = eof.state } } else if (message.type === 'control') { if (message.control.control_type === 'source_config') { diff --git a/apps/service/src/temporal/workflows/pipeline-workflow.ts b/apps/service/src/temporal/workflows/pipeline-workflow.ts index 07851631..53d88721 100644 --- a/apps/service/src/temporal/workflows/pipeline-workflow.ts +++ b/apps/service/src/temporal/workflows/pipeline-workflow.ts @@ -130,8 +130,9 @@ export async function pipelineWorkflow( const events = await waitForLiveEvents() if (!events) return - const result = await pipelineSync(pipelineId, { input: events }) + const result = await pipelineSync(pipelineId, { state: syncState, input: events }) operationCount++ + syncState = result.state if (classifySyncErrors(result.errors).permanent.length > 0) { await markPermanentError() return diff --git a/packages/protocol/src/helpers.ts b/packages/protocol/src/helpers.ts index 6fad2240..1ccaa8dc 100644 --- a/packages/protocol/src/helpers.ts +++ b/packages/protocol/src/helpers.ts @@ -108,13 +108,16 @@ export function isTraceStreamStatus( return msg.type === 'trace' && msg.trace.trace_type === 'stream_status' } -/** Type guard for trace progress messages. */ -export function isTraceProgress( +/** Type guard for trace global_progress messages. */ +export function isTraceGlobalProgress( msg: Message -): msg is TraceMessage & { trace: { trace_type: 'progress' } } { - return msg.type === 'trace' && msg.trace.trace_type === 'progress' +): msg is TraceMessage & { trace: { trace_type: 'global_progress' } } { + return msg.type === 'trace' && msg.trace.trace_type === 'global_progress' } +/** @deprecated Use isTraceGlobalProgress. */ +export const isTraceProgress = isTraceGlobalProgress + export function emptySectionState(): SectionState { return { streams: {}, global: {} } } diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 87f84a3d..f82718f9 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -23,6 +23,7 @@ export { isDataMessage, isTraceError, isTraceStreamStatus, + isTraceGlobalProgress, isTraceProgress, // State constructors coerceSyncState, diff --git a/packages/protocol/src/protocol.ts b/packages/protocol/src/protocol.ts index 4bcd91d0..2c1238ba 100644 --- a/packages/protocol/src/protocol.ts +++ b/packages/protocol/src/protocol.ts @@ -260,16 +260,11 @@ export const TraceStreamStatus = z .object({ stream: z.string().describe('Stream being reported on.'), status: z - .enum([ - 'started', - 'running', - 'complete', - 'transient_error', - 'system_error', - 'config_error', - 'auth_error', - ]) - .describe('Current phase of the stream within this sync run.'), + .enum(['started', 'complete']) + .describe( + 'Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. ' + + 'Sources may store richer error statuses internally for retry logic.' + ), cumulative_record_count: z .number() .int() @@ -284,32 +279,10 @@ export const TraceStreamStatus = z .int() .optional() .describe('Records synced for this stream in the current sync run. Set by the engine.'), - window_record_count: z - .number() - .int() - .optional() - .describe( - 'Records synced since the last stream_status emission for this stream. ' + - 'Set by the engine. Used for instantaneous per-stream throughput.' - ), - records_per_second: z - .number() - .optional() - .describe( - 'Average records per second for this stream over the entire run: ' + - 'run_record_count / elapsed seconds. Set by the engine.' - ), - requests_per_second: z - .number() - .optional() - .describe( - 'Average API requests per second for this stream over the entire run. ' + - 'Set by the engine from source-reported request counts.' - ), }) .describe( 'Per-stream status update. Sources emit the minimal form (stream + status). ' + - 'The engine emits enriched versions with record counts and throughput rates.' + 'The engine enriches with record counts. Only emitted on status transitions.' ) export type TraceStreamStatus = z.infer @@ -322,17 +295,22 @@ export const TraceEstimate = z .describe('Sync progress estimate for a stream.') export type TraceEstimate = z.infer -export const TraceProgress = z +export const TraceGlobalProgress = z .object({ elapsed_ms: z.number().int().describe('Wall-clock milliseconds since the sync run started.'), run_record_count: z .number() .int() .describe('Total records synced across all streams in this run.'), - rows_per_second: z + cumulative_record_count: z + .number() + .int() + .optional() + .describe('Total records synced across all streams across all runs.'), + records_per_second: z .number() .describe('Overall throughput for the entire run: run_record_count / elapsed seconds.'), - window_rows_per_second: z + window_records_per_second: z .number() .describe( 'Instantaneous throughput: total records in last window / window duration. ' + @@ -342,13 +320,33 @@ export const TraceProgress = z .number() .int() .describe('Total source_state messages observed so far in this sync run.'), + request_count: z + .number() + .int() + .optional() + .describe('Total API requests made by the source in this run.'), + cumulative_request_count: z + .number() + .int() + .optional() + .describe('Total API requests across all runs.'), + cumulative_elapsed_ms: z + .number() + .int() + .optional() + .describe('Total wall-clock time across all runs.'), }) .describe( - 'Periodic global sync progress emitted by the engine. ' + + 'Global sync progress emitted by the engine, co-emitted with every stream_status trace. ' + 'Aggregate stats only β€” per-stream detail is in stream_status messages. ' + 'Each emission is a full replacement.' ) -export type TraceProgress = z.infer +export type TraceGlobalProgress = z.infer + +/** @deprecated Use TraceGlobalProgress. */ +export const TraceProgress = TraceGlobalProgress +/** @deprecated Use TraceGlobalProgress. */ +export type TraceProgress = TraceGlobalProgress export const TracePayload = z .discriminatedUnion('trace_type', [ @@ -365,43 +363,27 @@ export const TracePayload = z estimate: TraceEstimate, }), z.object({ - trace_type: z.literal('progress'), - progress: TraceProgress, + trace_type: z.literal('global_progress'), + global_progress: TraceGlobalProgress, }), ]) .describe( - 'Diagnostic/status payload with subtypes for error, stream status, estimates, and progress.' + 'Diagnostic/status payload with subtypes for error, stream status, estimates, and global progress.' ) export type TracePayload = z.infer -// MARK: - EOF payload (depends on TraceProgress) +// MARK: - EOF payload (depends on TraceGlobalProgress) export const EofStreamProgress = z .object({ status: z - .enum([ - 'started', - 'running', - 'complete', - 'transient_error', - 'system_error', - 'config_error', - 'auth_error', - ]) - .describe('Final stream status.'), + .enum(['started', 'complete']) + .describe('Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors.'), cumulative_record_count: z .number() .int() .describe('Cumulative records synced for this stream across all runs.'), run_record_count: z.number().int().describe('Records synced in this run.'), - records_per_second: z - .number() - .optional() - .describe('Average records/sec for this stream over the run.'), - requests_per_second: z - .number() - .optional() - .describe('Average requests/sec for this stream over the run.'), errors: z .array( z.object({ @@ -440,8 +422,8 @@ export const EofPayload = z 'engine: updated cumulative record counts; destination: reserved. ' + 'Consumers can persist this directly and pass it back on resume.' ), - global_progress: TraceProgress.optional().describe( - 'Final global aggregates. Same shape as trace/progress.' + global_progress: TraceGlobalProgress.optional().describe( + 'Final global aggregates. Same shape as trace/global_progress.' ), stream_progress: z .record(z.string(), EofStreamProgress) @@ -577,6 +559,7 @@ export type DestinationOutput = z.infer export const SyncOutput = z .discriminatedUnion('type', [ SourceStateMessage, + CatalogMessage, TraceMessage, LogMessage, EofMessage, diff --git a/packages/source-stripe/src/transport.test.ts b/packages/source-stripe/src/transport.test.ts index d6290466..488d07b0 100644 --- a/packages/source-stripe/src/transport.test.ts +++ b/packages/source-stripe/src/transport.test.ts @@ -77,6 +77,7 @@ describe('parsePositiveInteger', () => { }) + describe('getHttpsProxyAgentForTarget', () => { it('returns an agent only when the target should use the proxy', () => { expect( From 00db06ca0056ce4520f0bbc62f74b02a21393e61 Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 06:16:26 +0000 Subject: [PATCH 02/32] feat: add CLI progress display and extract sync progress reducer Extract sync display state as a pure reducer over SyncOutput messages that accumulates into an EofPayload shape. The renderer is a stateless function of (EofPayload, catalog) -> string[]. - Add --progress flag to sync command (auto-enabled on TTY) - Add --base-url flag for QA/non-prod Stripe API endpoints - Extract sync-progress-state.ts: createSyncDisplayState() reducer + renderSyncProgress() renderer with emoji status groups - Deduplicate: app.ts formatEof now delegates to renderSyncProgress - Suppress raw source stream_status traces in trackProgress (engine re-emits enriched versions) - Add scripts/test-all-accounts.sh for multi-account testing Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/src/api/app.ts | 123 +----------- apps/engine/src/cli/sync.ts | 33 ++- apps/engine/src/lib/progress.ts | 5 + apps/engine/src/lib/sync-progress-state.ts | 221 +++++++++++++++++++++ 4 files changed, 259 insertions(+), 123 deletions(-) create mode 100644 apps/engine/src/lib/sync-progress-state.ts diff --git a/apps/engine/src/api/app.ts b/apps/engine/src/api/app.ts index 586e805c..ea92a9eb 100644 --- a/apps/engine/src/api/app.ts +++ b/apps/engine/src/api/app.ts @@ -10,6 +10,7 @@ import { HTTPException } from 'hono/http-exception' import pg from 'pg' import type { Message, ConnectorResolver, TraceMessage } from '../lib/index.js' import type { EofPayload } from '@stripe/sync-protocol' +import { renderSyncProgress } from '../lib/sync-progress-state.js' import { createEngine, createConnectorSchemas, @@ -123,128 +124,8 @@ async function* logApiStream( const dangerouslyVerbose = process.env.DANGEROUSLY_VERBOSE_LOGGING === 'true' -const REASON_EMOJI: Record = { - complete: 'βœ…', - time_limit: '⏱️', - state_limit: 'πŸ“¦', - error: '❌', - aborted: 'πŸ›‘', -} - -const ERROR_EMOJI: Record = { - transient_error: '⚠️', - system_error: '❌', - config_error: 'βš™οΈ', - auth_error: 'πŸ”’', -} - -function formatDuration(ms: number): string { - if (ms < 1000) return `${ms}ms` - if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s` - const mins = Math.floor(ms / 60_000) - const secs = Math.round((ms % 60_000) / 1000) - if (mins < 60) return secs > 0 ? `${mins}m ${secs}s` : `${mins}m` - const hrs = Math.floor(mins / 60) - const remainMins = mins % 60 - return remainMins > 0 ? `${hrs}h ${remainMins}m` : `${hrs}h` -} - -function formatNumber(n: number): string { - return n.toLocaleString('en-US') -} - function formatEof(eof: EofPayload): string { - const emoji = REASON_EMOJI[eof.reason] ?? '❓' - const gp = eof.global_progress - const runRecords = gp?.run_record_count ?? 0 - const cumulativeRecords = gp?.cumulative_record_count ?? runRecords - const rps = gp?.records_per_second?.toFixed(1) ?? '0' - const runElapsed = gp?.elapsed_ms ?? 0 - const cumulativeElapsed = gp?.cumulative_elapsed_ms ?? runElapsed - const cumulativeRequests = gp?.cumulative_request_count ?? 0 - const runRequests = gp?.request_count ?? 0 - - const lines: string[] = [] - lines.push(`${emoji} Sync ${eof.reason}`) - lines.push( - ` Total: ${formatNumber(cumulativeRecords)} records | ${formatNumber(cumulativeRequests)} requests | ${formatDuration(cumulativeElapsed)}` - ) - lines.push( - ` This run: +${formatNumber(runRecords)} records | ${formatNumber(runRequests)} requests | ${formatDuration(runElapsed)} | ${rps} records/s` - ) - - const sp = eof.stream_progress - if (sp) { - type StreamEntry = { name: string; cumulative: number; run: number; errors: StreamError[] } - type StreamError = { message: string; failure_type?: string } - const completeStreams: StreamEntry[] = [] - const startedStreams: StreamEntry[] = [] - let errorsCount = 0 - - for (const [name, s] of Object.entries(sp)) { - const entry: StreamEntry = { - name, - cumulative: s.cumulative_record_count, - run: s.run_record_count, - errors: s.errors ?? [], - } - if (entry.errors.length > 0) errorsCount++ - if (s.status === 'complete') { - completeStreams.push(entry) - } else { - startedStreams.push(entry) - } - } - - // Sort by cumulative record count descending - completeStreams.sort((a, b) => b.cumulative - a.cumulative) - startedStreams.sort((a, b) => b.cumulative - a.cumulative) - - const maxNameLen = Math.max(...Object.keys(sp).map((n) => n.length), 10) - - function formatStreamLine(entry: StreamEntry): string[] { - const result: string[] = [] - const countStr = - entry.cumulative > 0 - ? `${formatNumber(entry.cumulative).padStart(10)}${entry.run > 0 ? ` (+${formatNumber(entry.run)})` : ''}` - : '' - result.push(` ${entry.name.padEnd(maxNameLen)} ${countStr}`) - for (const err of entry.errors) { - const errEmoji = ERROR_EMOJI[err.failure_type ?? 'system_error'] ?? '❌' - result.push( - ` ${errEmoji} ${err.message}${err.failure_type ? ` (${err.failure_type})` : ''}` - ) - } - return result - } - - if (completeStreams.length > 0) { - lines.push('') - lines.push(` Complete (${completeStreams.length}):`) - for (const s of completeStreams) { - lines.push(...formatStreamLine(s)) - } - } - - if (startedStreams.length > 0) { - lines.push('') - lines.push(` Started (${startedStreams.length}):`) - for (const s of startedStreams) { - lines.push(...formatStreamLine(s)) - } - } - - // Summary line - lines.push('') - const parts: string[] = [] - if (completeStreams.length) parts.push(`${completeStreams.length} complete`) - if (startedStreams.length) parts.push(`${startedStreams.length} started`) - if (errorsCount) parts.push(`${errorsCount} streams with errors`) - parts.push(`+${formatNumber(runRecords)} records this run`) - lines.push(` πŸ“Š ${parts.join(', ')}`) - } - - return lines.join('\n') + return renderSyncProgress(eof, [], true).join('\n') } /** diff --git a/apps/engine/src/cli/sync.ts b/apps/engine/src/cli/sync.ts index 5dbd9674..83b44ff8 100644 --- a/apps/engine/src/cli/sync.ts +++ b/apps/engine/src/cli/sync.ts @@ -3,6 +3,7 @@ import type { Engine } from '../lib/engine.js' import type { ConnectorResolver } from '../lib/index.js' import { readonlyStateStore, type StateStore } from '../lib/state-store.js' import { type PipelineConfig, type SyncState, emptySyncState } from '@stripe/sync-protocol' +import { createSyncDisplayState, renderSyncProgress } from '../lib/sync-progress-state.js' export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { return defineCommand({ @@ -46,6 +47,15 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { type: 'string', description: 'Stop after N seconds', }, + baseUrl: { + type: 'string', + description: 'Stripe API base URL (or STRIPE_API_BASE env, default: https://api.stripe.com)', + }, + progress: { + type: 'boolean', + default: false, + description: 'Force progress display (auto-enabled when stderr is a TTY)', + }, live: { type: 'boolean', default: false, @@ -84,6 +94,10 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { // Inject optional source config overrides const stripeConfig = pipeline.source.stripe as Record + const baseUrl = args.baseUrl || process.env.STRIPE_API_BASE + if (baseUrl) { + stripeConfig.base_url = baseUrl + } if (backfillLimit) { stripeConfig.backfill_limit = backfillLimit } @@ -101,7 +115,10 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { : undefined const output = engine.pipeline_sync(pipeline, { state: syncState, time_limit: timeLimit }) - // Persist state checkpoints and stream NDJSON to stdout + const showProgress = args.progress || process.stderr.isTTY + const display = showProgress ? createSyncDisplayState() : null + let linesPrinted = 0 + for await (const msg of output) { if (msg.type === 'source_state') { if (msg.source_state.state_type === 'global') { @@ -110,7 +127,19 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { await store.set(msg.source_state.stream, msg.source_state.data) } } - process.stdout.write(JSON.stringify(msg) + '\n') + + if (display) { + const changed = display.update(msg) + if (changed) { + if (linesPrinted > 0) process.stderr.write(`\x1b[${linesPrinted}A\x1b[0J`) + const final = msg.type === 'eof' + const lines = renderSyncProgress(display.state.eof, display.state.catalog, final) + for (const line of lines) process.stderr.write(line + '\n') + linesPrinted = lines.length + } + } else { + process.stdout.write(JSON.stringify(msg) + '\n') + } } if ('close' in store && typeof store.close === 'function') { diff --git a/apps/engine/src/lib/progress.ts b/apps/engine/src/lib/progress.ts index aad24c0a..d0f49e71 100644 --- a/apps/engine/src/lib/progress.ts +++ b/apps/engine/src/lib/progress.ts @@ -337,6 +337,11 @@ export function trackProgress(opts: { return } + // Suppress upstream stream_status traces β€” the engine re-emits enriched versions + if (msg.type === 'trace' && msg.trace.trace_type === 'stream_status') { + continue + } + yield msg } } diff --git a/apps/engine/src/lib/sync-progress-state.ts b/apps/engine/src/lib/sync-progress-state.ts new file mode 100644 index 00000000..7bcdb4f6 --- /dev/null +++ b/apps/engine/src/lib/sync-progress-state.ts @@ -0,0 +1,221 @@ +import type { + SyncOutput, + EofPayload, + EofStreamProgress, + TraceGlobalProgress, +} from '@stripe/sync-protocol' + +// ── Reducer: SyncOutput messages β†’ EofPayload ──────────────────── +// +// At any point during a sync, the accumulated state is a valid EofPayload. +// The final EOF message from the engine replaces it wholesale. +// Display is purely a function of (EofPayload, catalog). + +export interface SyncDisplayState { + catalog: string[] + eof: EofPayload +} + +export function createSyncDisplayState(): { + state: SyncDisplayState + /** Returns true if the message changed the display state. */ + update: (msg: SyncOutput) => boolean +} { + const state: SyncDisplayState = { + catalog: [], + eof: { reason: 'complete' }, + } + + function ensureStream(name: string): EofStreamProgress { + if (!state.eof.stream_progress) state.eof.stream_progress = {} + if (!state.eof.stream_progress[name]) { + state.eof.stream_progress[name] = { + status: 'started', + cumulative_record_count: 0, + run_record_count: 0, + } + } + return state.eof.stream_progress[name] + } + + function update(msg: SyncOutput): boolean { + if (msg.type === 'catalog') { + state.catalog = (msg.catalog as { streams: Array<{ name: string }> }).streams.map( + (s) => s.name + ) + return true + } + + if (msg.type === 'trace') { + const t = msg.trace + if (t.trace_type === 'stream_status') { + const ss = t.stream_status + const sp = ensureStream(ss.stream) + sp.status = ss.status as 'started' | 'complete' + if (ss.cumulative_record_count != null) sp.cumulative_record_count = ss.cumulative_record_count + if (ss.run_record_count != null) sp.run_record_count = ss.run_record_count + return true + } + if (t.trace_type === 'global_progress') { + state.eof.global_progress = ( + t as { trace_type: 'global_progress'; global_progress: TraceGlobalProgress } + ).global_progress + return false // rendered with preceding stream_status + } + if (t.trace_type === 'error') { + const err = ( + t as { + trace_type: 'error' + error: { message: string; failure_type?: string; stream?: string } + } + ).error + if (err.stream) { + const sp = ensureStream(err.stream) + if (!sp.errors) sp.errors = [] + sp.errors.push({ + message: err.message, + failure_type: err.failure_type as 'config_error' | 'system_error' | 'transient_error' | 'auth_error' | undefined, + }) + } + return false + } + } + + if (msg.type === 'eof') { + // The engine's EOF is authoritative β€” replace everything + state.eof = msg.eof + return true + } + + return false + } + + return { state, update } +} + +// ── Renderer: (EofPayload, catalog) β†’ string[] ────────────────── + +const ERROR_EMOJI: Record = { + transient_error: '⚠️', + system_error: '❌', + config_error: 'βš™οΈ', + auth_error: 'πŸ”’', +} + +const REASON_EMOJI: Record = { + complete: 'βœ…', + time_limit: '⏱️', + state_limit: 'πŸ“¦', + error: '❌', + aborted: 'πŸ›‘', +} + +function fmt(n: number): string { + return n.toLocaleString('en-US') +} + +function fmtDuration(ms: number): string { + if (ms < 1000) return `${ms}ms` + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s` + const mins = Math.floor(ms / 60_000) + const secs = Math.round((ms % 60_000) / 1000) + if (mins < 60) return secs > 0 ? `${mins}m ${secs}s` : `${mins}m` + const hrs = Math.floor(mins / 60) + const rm = mins % 60 + return rm > 0 ? `${hrs}h ${rm}m` : `${hrs}h` +} + +/** + * Render a sync progress table from an EofPayload and optional catalog. + * Pure function β€” no side effects. Returns an array of lines. + * + * @param eof - The current (or final) EOF payload + * @param catalog - Stream names from the catalog (to derive pending streams) + * @param final - True when rendering after the actual EOF message (changes header) + */ +export function renderSyncProgress( + eof: EofPayload, + catalog: string[] = [], + final = false +): string[] { + const lines: string[] = [] + const gp = eof.global_progress + + // Header + if (final) { + lines.push(`${REASON_EMOJI[eof.reason] ?? '❓'} Sync ${eof.reason}`) + } else { + lines.push('πŸ”„ Syncing...') + } + + if (gp) { + const cumRecords = gp.cumulative_record_count ?? gp.run_record_count + const cumElapsed = gp.cumulative_elapsed_ms ?? gp.elapsed_ms + lines.push( + ` Total: ${fmt(cumRecords)} records | ${fmt(gp.cumulative_request_count ?? 0)} requests | ${fmtDuration(cumElapsed)}` + ) + lines.push( + ` This run: +${fmt(gp.run_record_count)} records | ${fmt(gp.request_count ?? 0)} requests | ${fmtDuration(gp.elapsed_ms)} | ${gp.records_per_second.toFixed(1)} records/s` + ) + } + + // Group streams by status + const sp = eof.stream_progress ?? {} + const complete: [string, EofStreamProgress][] = [] + const started: [string, EofStreamProgress][] = [] + const pending: string[] = [] + + const known = new Set(Object.keys(sp)) + for (const [name, info] of Object.entries(sp)) { + if (info.status === 'complete') complete.push([name, info]) + else started.push([name, info]) + } + for (const name of catalog) { + if (!known.has(name)) pending.push(name) + } + + complete.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) + started.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) + + const allNames = [...complete.map((c) => c[0]), ...started.map((s) => s[0]), ...pending] + const maxName = Math.max(...allNames.map((n) => n.length), 10) + + function streamLine(name: string, info: EofStreamProgress) { + const cum = info.cumulative_record_count + const run = info.run_record_count + const countStr = + cum > 0 ? `${fmt(cum).padStart(10)}${run > 0 ? ` (+${fmt(run)})` : ''}` : '' + lines.push(` ${name.padEnd(maxName)} ${countStr}`) + for (const err of info.errors ?? []) { + const emoji = ERROR_EMOJI[err.failure_type ?? 'system_error'] ?? '❌' + lines.push(` ${emoji} ${err.message}${err.failure_type ? ` (${err.failure_type})` : ''}`) + } + } + + lines.push('') + if (complete.length > 0) { + lines.push(` βœ… Complete (${complete.length}):`) + for (const [name, info] of complete) streamLine(name, info) + } + if (started.length > 0) { + lines.push(` πŸ”„ Started (${started.length}):`) + for (const [name, info] of started) streamLine(name, info) + } + if (pending.length > 0) { + lines.push(` ⏳ Pending (${pending.length}):`) + lines.push(` ${pending.join(', ')}`) + } + + // Summary + const errCount = Object.values(sp).filter((i) => (i.errors?.length ?? 0) > 0).length + lines.push('') + const parts: string[] = [] + if (complete.length) parts.push(`${complete.length} complete`) + if (started.length) parts.push(`${started.length} started`) + if (pending.length) parts.push(`${pending.length} pending`) + if (errCount) parts.push(`${errCount} with errors`) + parts.push(`+${fmt(gp?.run_record_count ?? 0)} records this run`) + lines.push(` πŸ“Š ${parts.join(' | ')}`) + + return lines +} From 8138df942570746c57d8c4460c93d2e71c152857 Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 07:08:11 +0000 Subject: [PATCH 03/32] =?UTF-8?q?fix:=20overhaul=20error=20handling=20?= =?UTF-8?q?=E2=80=94=20terminal=20stream=5Fstatus,=20stream=20vs=20global?= =?UTF-8?q?=20errors,=20system=5Ferror=20reclassification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source always emits stream_status:complete on error (errors are orthogonal to lifecycle). Adds 'Unrecognized request URL' to skippable patterns for treasury. Stream-scoped permanent errors no longer park the entire workflow β€” only global permanent errors (bad API key, invalid config) do. Reclassifies system_error using isRetryableHttpError: 429/5xx/networkβ†’transient, everything elseβ†’permanent. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/package.json | 3 + apps/engine/src/cli/sync-ui.tsx | 298 +++++++++++++++ apps/engine/src/cli/sync.ts | 35 +- apps/engine/src/lib/progress.ts | 8 +- apps/engine/tsconfig.json | 3 +- apps/service/src/__tests__/workflow.test.ts | 47 ++- apps/service/src/temporal/sync-errors.test.ts | 71 ++++ apps/service/src/temporal/sync-errors.ts | 15 +- .../temporal/workflows/pipeline-workflow.ts | 4 +- packages/source-stripe/src/index.test.ts | 107 +++++- packages/source-stripe/src/src-list-api.ts | 22 +- pnpm-lock.yaml | 351 +++++++++++++++++- 12 files changed, 911 insertions(+), 53 deletions(-) create mode 100644 apps/engine/src/cli/sync-ui.tsx create mode 100644 apps/service/src/temporal/sync-errors.test.ts diff --git a/apps/engine/package.json b/apps/engine/package.json index ee97502b..883e5574 100644 --- a/apps/engine/package.json +++ b/apps/engine/package.json @@ -56,10 +56,12 @@ "dotenv": "^16.4.7", "googleapis": "^148.0.0", "hono": "^4", + "ink": "^7.0.0", "openapi-fetch": "^0.17.0", "pg": "^8.16.3", "pino": "^10", "pino-pretty": "^13", + "react": "^19.2.4", "ws": "^8.18.0", "zod": "^4.3.6" }, @@ -67,6 +69,7 @@ "@hyperjump/json-schema": "^1.17.5", "@types/node": "^24.10.1", "@types/pg": "^8.15.4", + "@types/react": "^19.2.14", "openapi-typescript": "^7.13.0", "vitest": "^3.2.4" }, diff --git a/apps/engine/src/cli/sync-ui.tsx b/apps/engine/src/cli/sync-ui.tsx new file mode 100644 index 00000000..675d5e84 --- /dev/null +++ b/apps/engine/src/cli/sync-ui.tsx @@ -0,0 +1,298 @@ +import React from 'react' +import { Box, Text } from 'ink' +import type { EofPayload, EofStreamProgress } from '@stripe/sync-protocol' + +// ── Formatting helpers ──────────────────────────────────────────── + +function fmt(n: number): string { + return n.toLocaleString('en-US') +} + +function fmtDuration(ms: number): string { + if (ms < 1000) return `${ms}ms` + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s` + const mins = Math.floor(ms / 60_000) + const secs = Math.round((ms % 60_000) / 1000) + if (mins < 60) return secs > 0 ? `${mins}m ${secs}s` : `${mins}m` + const hrs = Math.floor(mins / 60) + const rm = mins % 60 + return rm > 0 ? `${hrs}h ${rm}m` : `${hrs}h` +} + +function fmtRate(rps: number): string { + return rps >= 1000 ? `${(rps / 1000).toFixed(1)}k/s` : `${rps.toFixed(1)}/s` +} + +// ── Constants ───────────────────────────────────────────────────── + +const REASON_COLOR: Record = { + complete: 'green', + time_limit: 'yellow', + state_limit: 'blue', + error: 'red', + aborted: 'red', +} + +const REASON_LABEL: Record = { + complete: 'Sync complete', + time_limit: 'Time limit reached', + state_limit: 'State limit reached', + error: 'Sync failed', + aborted: 'Sync aborted', +} + +const ERROR_COLOR: Record = { + transient_error: 'yellow', + system_error: 'red', + config_error: 'magenta', + auth_error: 'red', +} + +const ERROR_LABEL: Record = { + transient_error: 'transient', + system_error: 'system', + config_error: 'config', + auth_error: 'auth', +} + +// ── Sub-components ──────────────────────────────────────────────── + +function Divider({ width = 60 }: { width?: number }) { + return {'─'.repeat(width)} +} + +function StatRow({ label, value, dimLabel = false }: { label: string; value: string; dimLabel?: boolean }) { + return ( + + {label} + {value} + + ) +} + +function StreamRow({ + name, + info, + nameWidth, + running, +}: { + name: string + info: EofStreamProgress + nameWidth: number + running: boolean +}) { + const cum = info.cumulative_record_count + const run = info.run_record_count + const isComplete = info.status === 'complete' + const hasErrors = (info.errors?.length ?? 0) > 0 + + return ( + + + {/* Status dot */} + + {isComplete ? ( + {'βœ“ '} + ) : running ? ( + {'β–Ά '} + ) : ( + {'Β· '} + )} + + {/* Stream name */} + + + {name.padEnd(nameWidth)} + + + {/* Cumulative count */} + + {cum > 0 ? fmt(cum) : 'β€”'} + + {/* Run delta */} + + {run > 0 ? +{fmt(run)} : null} + + + + {/* Per-stream errors */} + {(info.errors ?? []).map((err, i) => ( + + + [{ERROR_LABEL[err.failure_type ?? 'system_error'] ?? 'error'}] {err.message} + + + ))} + + ) +} + +// ── Main component ──────────────────────────────────────────────── + +export interface SyncProgressProps { + eof: EofPayload + catalog: string[] + final: boolean + /** Number of pipeline_sync calls made so far (backfill mode) */ + attempt?: number +} + +export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressProps) { + const gp = eof.global_progress + const sp = eof.stream_progress ?? {} + + // Partition streams + const complete: [string, EofStreamProgress][] = [] + const started: [string, EofStreamProgress][] = [] + const pending: string[] = [] + + const known = new Set(Object.keys(sp)) + for (const [name, info] of Object.entries(sp)) { + if (info.status === 'complete') complete.push([name, info]) + else started.push([name, info]) + } + for (const name of catalog) { + if (!known.has(name)) pending.push(name) + } + + complete.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) + started.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) + + const allStreamNames = [ + ...complete.map((c) => c[0]), + ...started.map((s) => s[0]), + ...pending, + ] + const nameWidth = Math.max(...allStreamNames.map((n) => n.length), 12) + + const errCount = Object.values(sp).filter((i) => (i.errors?.length ?? 0) > 0).length + const reasonColor = REASON_COLOR[eof.reason] ?? 'white' + const reasonLabel = REASON_LABEL[eof.reason] ?? eof.reason + + return ( + + {/* ── Header ── */} + + {final ? ( + + {reasonLabel} + {attempt != null && attempt > 1 && ( + ({attempt} attempts) + )} + + ) : ( + + Syncing + {attempt != null && attempt > 1 && ( + Β· attempt {attempt} + )} + {gp && ( + + {' '} + Β· {fmtDuration(gp.elapsed_ms)} + {gp.window_records_per_second != null && gp.window_records_per_second > 0 + ? ` Β· ${fmtRate(gp.window_records_per_second)}` + : ''} + + )} + + )} + + + {/* ── Global stats ── */} + {gp && ( + + + + 0 && (gp.cumulative_record_count ?? 0) > gp.run_record_count + ? ` (+${fmt(gp.run_record_count)} this run)` + : '' + }`} + /> + + + {final && gp.records_per_second > 0 && ( + + )} + + + + )} + + {/* ── Stream table header ── */} + {allStreamNames.length > 0 && ( + + + + + {'stream'.padEnd(nameWidth)} + + + + + total + + + + + this run + + + + )} + + {/* ── Complete streams ── */} + {complete.length > 0 && ( + 0 || pending.length > 0 ? 1 : 0}> + {complete.map(([name, info]) => ( + + ))} + + )} + + {/* ── In-progress streams ── */} + {started.length > 0 && ( + 0 ? 1 : 0}> + {started.map(([name, info]) => ( + + ))} + + )} + + {/* ── Pending streams (collapsed) ── */} + {pending.length > 0 && ( + + + {'Β· '} + + + {pending.length} pending: {pending.slice(0, 5).join(', ')} + {pending.length > 5 ? ` +${pending.length - 5} more` : ''} + + + )} + + {/* ── Footer summary ── */} + {(complete.length + started.length + pending.length > 0) && ( + + + + )} + + {complete.length > 0 && {complete.length} complete} + {started.length > 0 && {started.length} in progress} + {pending.length > 0 && {pending.length} pending} + {errCount > 0 && {errCount} with errors} + {gp && gp.run_record_count > 0 && ( + +{fmt(gp.run_record_count)} records this run + )} + {eof.cutoff && ( + cutoff: {eof.cutoff} + )} + + + ) +} diff --git a/apps/engine/src/cli/sync.ts b/apps/engine/src/cli/sync.ts index 83b44ff8..5d87fddc 100644 --- a/apps/engine/src/cli/sync.ts +++ b/apps/engine/src/cli/sync.ts @@ -1,9 +1,12 @@ +import React from 'react' +import { render } from 'ink' import { defineCommand } from 'citty' import type { Engine } from '../lib/engine.js' import type { ConnectorResolver } from '../lib/index.js' import { readonlyStateStore, type StateStore } from '../lib/state-store.js' import { type PipelineConfig, type SyncState, emptySyncState } from '@stripe/sync-protocol' -import { createSyncDisplayState, renderSyncProgress } from '../lib/sync-progress-state.js' +import { createSyncDisplayState } from '../lib/sync-progress-state.js' +import { SyncProgressUI } from './sync-ui.js' export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { return defineCommand({ @@ -117,7 +120,18 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { const showProgress = args.progress || process.stderr.isTTY const display = showProgress ? createSyncDisplayState() : null - let linesPrinted = 0 + + // Mount Ink UI on stderr if showing progress + const inkInstance = display + ? render( + React.createElement(SyncProgressUI, { + eof: display.state.eof, + catalog: display.state.catalog, + final: false, + }), + { stdout: process.stderr } + ) + : null for await (const msg of output) { if (msg.type === 'source_state') { @@ -128,20 +142,25 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { } } - if (display) { + if (display && inkInstance) { const changed = display.update(msg) if (changed) { - if (linesPrinted > 0) process.stderr.write(`\x1b[${linesPrinted}A\x1b[0J`) const final = msg.type === 'eof' - const lines = renderSyncProgress(display.state.eof, display.state.catalog, final) - for (const line of lines) process.stderr.write(line + '\n') - linesPrinted = lines.length + inkInstance.rerender( + React.createElement(SyncProgressUI, { + eof: display.state.eof, + catalog: display.state.catalog, + final, + }) + ) } - } else { + } else if (!display) { process.stdout.write(JSON.stringify(msg) + '\n') } } + inkInstance?.unmount() + if ('close' in store && typeof store.close === 'function') { await store.close() } diff --git a/apps/engine/src/lib/progress.ts b/apps/engine/src/lib/progress.ts index d0f49e71..9d764216 100644 --- a/apps/engine/src/lib/progress.ts +++ b/apps/engine/src/lib/progress.ts @@ -203,13 +203,15 @@ export function trackProgress(opts: { snapshotWindow() } - function buildStreamProgress(stream: string): EofStreamProgress | undefined { + function buildStreamProgress(stream: string, finalEof = false): EofStreamProgress | undefined { const status = streamStatus.get(stream) if (!status) return undefined const run = runRecordCount(stream) const cumulative = (cumulativeRecordCount.get(stream) ?? 0) + run + // At EOF, no stream can still be in-flight β€” promote 'started' β†’ 'complete' + const resolvedStatus = finalEof && status === 'started' ? 'complete' : status return { - status, + status: resolvedStatus, cumulative_record_count: cumulative, run_record_count: run, errors: streamErrors.has(stream) ? streamErrors.get(stream) : undefined, @@ -256,7 +258,7 @@ export function trackProgress(opts: { const streams = allStreams() const streamProgressMap: Record = {} for (const s of streams) { - const sp = buildStreamProgress(s) + const sp = buildStreamProgress(s, true) if (sp) streamProgressMap[s] = sp } const runRecords = totalRunRecords() diff --git a/apps/engine/tsconfig.json b/apps/engine/tsconfig.json index 2481fe54..a7aaf861 100644 --- a/apps/engine/tsconfig.json +++ b/apps/engine/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "jsx": "react-jsx" }, "include": ["src/**/*"], "exclude": ["src/**/*.test.ts", "src/**/__tests__/**"] diff --git a/apps/service/src/__tests__/workflow.test.ts b/apps/service/src/__tests__/workflow.test.ts index c3cd8016..42bbec5d 100644 --- a/apps/service/src/__tests__/workflow.test.ts +++ b/apps/service/src/__tests__/workflow.test.ts @@ -17,8 +17,12 @@ const emptyState = { engine: { streams: {}, global: {} }, } const noErrors: RunResult = { errors: [], state: emptyState } -const permanentSyncError: RunResult = { - errors: [{ message: 'permanent sync failure', failure_type: 'auth_error', stream: 'customers' }], +const globalPermanentSyncError: RunResult = { + errors: [{ message: 'bad API key', failure_type: 'auth_error' }], + state: emptyState, +} +const streamPermanentSyncError: RunResult = { + errors: [{ message: 'stream auth failure', failure_type: 'auth_error', stream: 'customers' }], state: emptyState, } @@ -341,7 +345,7 @@ describe('pipelineWorkflow (unit β€” stubbed activities)', () => { }) }) - it('transitions to error instead of ready when reconcile returns permanent sync errors', async () => { + it('transitions to error when reconcile returns global permanent sync errors', async () => { const statusWrites: string[] = [] const worker = await Worker.create({ @@ -354,7 +358,7 @@ describe('pipelineWorkflow (unit β€” stubbed activities)', () => { }, pipelineSync: async (_pipelineId: string, opts?) => { if (opts?.input) return noErrors - return { ...permanentSyncError, eof: { reason: 'complete' as const } } + return { ...globalPermanentSyncError, eof: { reason: 'complete' as const } } }, }), }) @@ -375,6 +379,41 @@ describe('pipelineWorkflow (unit β€” stubbed activities)', () => { }) }) + it('does not park workflow when reconcile returns stream-scoped permanent errors', async () => { + const statusWrites: string[] = [] + + const worker = await Worker.create({ + connection: testEnv.nativeConnection, + taskQueue: 'test-queue-3b-stream-error', + workflowsPath, + activities: stubActivities({ + updatePipelineStatus: async (_id: string, status: string) => { + statusWrites.push(status) + }, + pipelineSync: async (_pipelineId: string, opts?) => { + if (opts?.input) return noErrors + return { ...streamPermanentSyncError, eof: { reason: 'complete' as const } } + }, + }), + }) + + await worker.runUntil(async () => { + const handle = await testEnv.client.workflow.start('pipelineWorkflow', { + args: [testPipelineId], + workflowId: 'test-sync-3b-stream-error', + taskQueue: 'test-queue-3b-stream-error', + }) + + await new Promise((r) => setTimeout(r, 500)) + await signalDelete(handle) + await handle.result() + + // Stream-scoped permanent errors don't park the workflow + expect(statusWrites).toContain('ready') + expect(statusWrites).not.toContain('error') + }) + }) + it('retries transient sync activity failures and still reaches ready', async () => { const statusWrites: string[] = [] let reconcileCalls = 0 diff --git a/apps/service/src/temporal/sync-errors.test.ts b/apps/service/src/temporal/sync-errors.test.ts new file mode 100644 index 00000000..d52ba1c7 --- /dev/null +++ b/apps/service/src/temporal/sync-errors.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from 'vitest' +import { classifySyncErrors } from './sync-errors.js' + +describe('classifySyncErrors', () => { + it('classifies global permanent errors (no stream field)', () => { + const result = classifySyncErrors([{ message: 'bad key', failure_type: 'auth_error' }]) + + expect(result.globalPermanent).toHaveLength(1) + expect(result.streamPermanent).toHaveLength(0) + expect(result.permanent).toHaveLength(1) + expect(result.transient).toHaveLength(0) + }) + + it('classifies stream-scoped permanent errors', () => { + const result = classifySyncErrors([ + { message: 'stream auth fail', failure_type: 'auth_error', stream: 'treasury' }, + ]) + + expect(result.globalPermanent).toHaveLength(0) + expect(result.streamPermanent).toHaveLength(1) + expect(result.permanent).toHaveLength(1) + expect(result.transient).toHaveLength(0) + }) + + it('classifies transient errors regardless of stream scope', () => { + const result = classifySyncErrors([ + { message: 'rate limit', failure_type: 'transient_error', stream: 'customers' }, + { message: 'timeout', failure_type: 'transient_error' }, + ]) + + expect(result.transient).toHaveLength(2) + expect(result.permanent).toHaveLength(0) + expect(result.globalPermanent).toHaveLength(0) + expect(result.streamPermanent).toHaveLength(0) + }) + + it('classifies system_error as permanent', () => { + const result = classifySyncErrors([ + { message: 'deterministic failure', failure_type: 'system_error', stream: 'treasury' }, + ]) + + expect(result.permanent).toHaveLength(1) + expect(result.streamPermanent).toHaveLength(1) + expect(result.transient).toHaveLength(0) + }) + + it('separates mixed errors into correct buckets', () => { + const result = classifySyncErrors([ + { message: 'bad key', failure_type: 'auth_error' }, + { message: 'feature gate', failure_type: 'config_error', stream: 'treasury' }, + { message: 'rate limit', failure_type: 'transient_error' }, + ]) + + expect(result.globalPermanent).toHaveLength(1) + expect(result.globalPermanent[0].message).toBe('bad key') + + expect(result.streamPermanent).toHaveLength(1) + expect(result.streamPermanent[0].message).toBe('feature gate') + + expect(result.permanent).toHaveLength(2) + expect(result.transient).toHaveLength(1) + expect(result.transient[0].message).toBe('rate limit') + }) + + it('treats unknown failure_type as transient', () => { + const result = classifySyncErrors([{ message: 'unknown' }]) + + expect(result.transient).toHaveLength(1) + expect(result.permanent).toHaveLength(0) + }) +}) diff --git a/apps/service/src/temporal/sync-errors.ts b/apps/service/src/temporal/sync-errors.ts index 7f9a74fe..c9a1baef 100644 --- a/apps/service/src/temporal/sync-errors.ts +++ b/apps/service/src/temporal/sync-errors.ts @@ -7,23 +7,34 @@ export type SyncRunError = { export type ClassifiedSyncErrors = { transient: SyncRunError[] permanent: SyncRunError[] + /** Permanent errors without a stream scope β€” bad API key, invalid config. Parks the workflow. */ + globalPermanent: SyncRunError[] + /** Permanent errors scoped to a single stream β€” feature gate, per-stream auth. Stream is skipped on resume. */ + streamPermanent: SyncRunError[] } -const PERMANENT_FAILURE_TYPES = new Set(['config_error', 'auth_error']) +const PERMANENT_FAILURE_TYPES = new Set(['config_error', 'auth_error', 'system_error']) export function classifySyncErrors(errors: SyncRunError[]): ClassifiedSyncErrors { const transient: SyncRunError[] = [] const permanent: SyncRunError[] = [] + const globalPermanent: SyncRunError[] = [] + const streamPermanent: SyncRunError[] = [] for (const error of errors) { if (PERMANENT_FAILURE_TYPES.has(error.failure_type ?? '')) { permanent.push(error) + if (error.stream) { + streamPermanent.push(error) + } else { + globalPermanent.push(error) + } } else { transient.push(error) } } - return { transient, permanent } + return { transient, permanent, globalPermanent, streamPermanent } } export function summarizeSyncErrors(errors: SyncRunError[]): string { diff --git a/apps/service/src/temporal/workflows/pipeline-workflow.ts b/apps/service/src/temporal/workflows/pipeline-workflow.ts index 53d88721..721d09f5 100644 --- a/apps/service/src/temporal/workflows/pipeline-workflow.ts +++ b/apps/service/src/temporal/workflows/pipeline-workflow.ts @@ -133,7 +133,7 @@ export async function pipelineWorkflow( const result = await pipelineSync(pipelineId, { state: syncState, input: events }) operationCount++ syncState = result.state - if (classifySyncErrors(result.errors).permanent.length > 0) { + if (classifySyncErrors(result.errors).globalPermanent.length > 0) { await markPermanentError() return } @@ -167,7 +167,7 @@ export async function pipelineWorkflow( }) operationCount++ syncState = result.state - if (classifySyncErrors(result.errors).permanent.length > 0) { + if (classifySyncErrors(result.errors).globalPermanent.length > 0) { await markPermanentError() return } diff --git a/packages/source-stripe/src/index.test.ts b/packages/source-stripe/src/index.test.ts index 645c9746..436b1100 100644 --- a/packages/source-stripe/src/index.test.ts +++ b/packages/source-stripe/src/index.test.ts @@ -544,8 +544,8 @@ describe('StripeSource', () => { source.read({ config, catalog: catalog({ name: 'customers', primary_key: [['id']] }) }) ) - // trace(stream_status started) + trace(error) + source_state(transient_error) - expect(messages).toHaveLength(3) + // trace(stream_status started) + trace(error) + source_state(transient_error) + trace(stream_status complete) + expect(messages).toHaveLength(4) expect(messages[0]).toMatchObject({ type: 'trace', trace: { @@ -576,6 +576,14 @@ describe('StripeSource', () => { data: { status: 'transient_error' }, }, }) + + expect(messages[3]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }) }) it('emits TraceMessage error with failure_type config_error for unknown stream', async () => { @@ -628,7 +636,8 @@ describe('StripeSource', () => { source.read({ config, catalog: catalog({ name: 'customers', primary_key: [['id']] }) }) ) - expect(messages).toHaveLength(3) + // started + error + state(system_error) + complete + expect(messages).toHaveLength(4) const errorMsg = messages[1] as TraceMessage expect(errorMsg.type).toBe('trace') expect(errorMsg.trace.trace_type).toBe('error') @@ -646,6 +655,14 @@ describe('StripeSource', () => { data: { status: 'system_error' }, }, }) + + expect(messages[3]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }) }) it('proceeds with backfill using fallback timestamp when getAccount fails (fault-tolerant)', async () => { @@ -731,7 +748,8 @@ describe('StripeSource', () => { source.read({ config, catalog: catalog({ name: 'tax_ids', primary_key: [['id']] }) }) ) - expect(messages).toHaveLength(3) + // started + error + state(auth_error) + complete + expect(messages).toHaveLength(4) const errorMsg = messages[1] as TraceMessage expect(errorMsg.trace.trace_type).toBe('error') const traceError = ( @@ -748,6 +766,14 @@ describe('StripeSource', () => { type: 'source_state', source_state: { state_type: 'stream', stream: 'tax_ids', data: { status: 'auth_error' } }, }) + + expect(messages[3]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'tax_ids', status: 'complete' }, + }, + }) }) it('does not treat near-miss auth errors as skippable', async () => { @@ -768,7 +794,8 @@ describe('StripeSource', () => { source.read({ config, catalog: catalog({ name: 'customers', primary_key: [['id']] }) }) ) - expect(messages).toHaveLength(3) + // started + error + state(system_error) + complete + expect(messages).toHaveLength(4) expect(messages[1]).toMatchObject({ type: 'trace', trace: { @@ -787,6 +814,13 @@ describe('StripeSource', () => { data: { status: 'system_error' }, }, }) + expect(messages[3]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }) }) it('marks known skippable Stripe list errors as complete without emitting error traces', async () => { @@ -824,6 +858,52 @@ describe('StripeSource', () => { }) }) + it('treats Unrecognized request URL as skippable (feature not available)', async () => { + const listFn = vi + .fn() + .mockRejectedValueOnce( + new Error( + 'Unrecognized request URL (GET: /v1/treasury/financial_accounts). Please see https://stripe.com/docs' + ) + ) + + const registry: Record = { + treasury_financial_accounts: makeConfig({ + order: 1, + tableName: 'treasury_financial_accounts', + listFn: listFn as ResourceConfig['listFn'], + }), + } + + vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) + const messages = await collect( + source.read({ + config, + catalog: catalog({ + name: 'treasury_financial_accounts', + primary_key: [['id']], + }), + }) + ) + + // Skippable: started + complete, no error trace + expect(messages).toHaveLength(2) + expect(messages[0]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'treasury_financial_accounts', status: 'started' }, + }, + }) + expect(messages[1]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'treasury_financial_accounts', status: 'complete' }, + }, + }) + }) + it('continues to next stream after error on previous stream', async () => { const failingListFn = vi.fn().mockRejectedValueOnce(new Error('Connection refused')) const successListFn = vi.fn().mockResolvedValueOnce({ @@ -855,11 +935,11 @@ describe('StripeSource', () => { }) ) - // customers: started + error + error_state = 3 + // customers: started + error + error_state + complete = 4 // invoices: started + record + state + complete = 4 - expect(messages).toHaveLength(7) + expect(messages).toHaveLength(8) - // Customers errored + // Customers errored but still emits terminal stream_status expect(messages[0]).toMatchObject({ type: 'trace', trace: { @@ -879,16 +959,23 @@ describe('StripeSource', () => { data: { status: 'system_error' }, }, }) + expect(messages[3]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'complete' }, + }, + }) // Invoices succeeded - expect(messages[3]).toMatchObject({ + expect(messages[4]).toMatchObject({ type: 'trace', trace: { trace_type: 'stream_status', stream_status: { stream: 'invoices', status: 'started' }, }, }) - expect(messages[6]).toMatchObject({ + expect(messages[7]).toMatchObject({ type: 'trace', trace: { trace_type: 'stream_status', diff --git a/packages/source-stripe/src/src-list-api.ts b/packages/source-stripe/src/src-list-api.ts index c063ca2c..972a264b 100644 --- a/packages/source-stripe/src/src-list-api.ts +++ b/packages/source-stripe/src/src-list-api.ts @@ -6,6 +6,7 @@ import type { SegmentState, BackfillState } from './index.js' import type { RateLimiter } from './rate-limiter.js' import { MAX_SEGMENTS, MAX_CONCURRENCY } from './rate-limiter.js' import { StripeApiRequestError } from '@stripe/sync-openapi' +import { isRetryableHttpError } from './retry.js' import type { StripeClient } from './client.js' // MARK: - Rate-limit wrapper @@ -40,9 +41,15 @@ function withRateLimit(listFn: ListFn, rateLimiter: RateLimiter, signal?: AbortS } export function getFailureType(err: unknown): 'transient_error' | 'system_error' | 'auth_error' { - const isRateLimit = err instanceof Error && err.message.includes('Rate limit') - const isAuth = err instanceof StripeApiRequestError && (err.status === 401 || err.status === 403) - return isRateLimit ? 'transient_error' : isAuth ? 'auth_error' : 'system_error' + if (err instanceof StripeApiRequestError && (err.status === 401 || err.status === 403)) { + return 'auth_error' + } + // Rate limit message check (belt + suspenders alongside HTTP status check) + if (err instanceof Error && err.message.includes('Rate limit')) { + return 'transient_error' + } + // 429, 5xx, network errors, timeouts β†’ retriable; everything else β†’ permanent + return isRetryableHttpError(err) ? 'transient_error' : 'system_error' } export function errorToTrace(err: unknown, stream: string): TraceMessage { @@ -83,6 +90,7 @@ const SKIPPABLE_ERROR_PATTERNS = [ 'Must provide customer', 'Must provide ', 'not set up to use', + 'Unrecognized request URL', ] // MARK: - Compact state (generative β€” O(concurrency) not O(total segments)) @@ -696,6 +704,14 @@ export async function* listApiBackfill(opts: { ...(streamState?.backfill ? { backfill: streamState.backfill } : {}), }, }) + // Errors are orthogonal to lifecycle β€” always emit terminal status + yield { + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: stream.name, status: 'complete' }, + }, + } satisfies TraceMessage } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ed8e41d..5f880b91 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,13 +115,13 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: link:../engine + version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-service': specifier: workspace:* - version: file:apps/service(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(tslib@2.8.1) + version: file:apps/service(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14)(tslib@2.8.1) '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../../packages/source-stripe + version: file:packages/source-stripe '@tailwindcss/vite': specifier: ^4 version: 4.2.2(vite@6.4.1(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) @@ -172,7 +172,7 @@ importers: version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../../packages/source-stripe + version: file:packages/source-stripe '@stripe/sync-state-postgres': specifier: workspace:* version: link:../../packages/state-postgres @@ -194,6 +194,9 @@ importers: hono: specifier: ^4 version: 4.12.8 + ink: + specifier: ^7.0.0 + version: 7.0.0(@types/react@19.2.14)(react@19.2.4) openapi-fetch: specifier: ^0.17.0 version: 0.17.0 @@ -206,6 +209,9 @@ importers: pino-pretty: specifier: ^13 version: 13.1.3 + react: + specifier: ^19.2.4 + version: 19.2.4 ws: specifier: ^8.18.0 version: 8.18.3 @@ -222,6 +228,9 @@ importers: '@types/pg': specifier: ^8.15.4 version: 8.15.6 + '@types/react': + specifier: ^19.2.14 + version: 19.2.14 openapi-typescript: specifier: ^7.13.0 version: 7.13.0(typescript@5.9.3) @@ -245,7 +254,7 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: link:../engine + version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-hono-zod-openapi': specifier: workspace:* version: link:../../packages/hono-zod-openapi @@ -254,7 +263,7 @@ importers: version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../../packages/source-stripe + version: file:packages/source-stripe '@stripe/sync-ts-cli': specifier: workspace:* version: link:../../packages/ts-cli @@ -321,13 +330,13 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: link:../engine + version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-protocol': specifier: workspace:* version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../../packages/source-stripe + version: file:packages/source-stripe '@stripe/sync-state-postgres': specifier: workspace:* version: link:../../packages/state-postgres @@ -364,7 +373,7 @@ importers: version: 0.2.17 '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../../packages/source-stripe + version: file:packages/source-stripe codemirror: specifier: ^6.0.1 version: 6.0.2 @@ -420,7 +429,7 @@ importers: version: link:../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: link:../apps/engine + version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-openapi': specifier: workspace:* version: link:../packages/openapi @@ -432,7 +441,7 @@ importers: version: link:../apps/service '@stripe/sync-source-stripe': specifier: workspace:* - version: link:../packages/source-stripe + version: file:packages/source-stripe '@stripe/sync-test-utils': specifier: workspace:* version: link:../packages/test-utils @@ -680,6 +689,10 @@ importers: packages: + '@alcalzone/ansi-tokenize@0.3.0': + resolution: {integrity: sha512-p+CMKJ93HFmLkjXKlXiVGlMQEuRb6H0MokBSwUsX+S6BRX8eV5naFZpQJFfJHjRZY0Hmnqy1/r6UWl3x+19zYA==} + engines: {node: '>=18'} + '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -2824,6 +2837,10 @@ packages: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} + ansi-escapes@7.3.0: + resolution: {integrity: sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==} + engines: {node: '>=18'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -2861,6 +2878,10 @@ packages: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} + auto-bind@5.0.1: + resolution: {integrity: sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + autoprefixer@10.4.27: resolution: {integrity: sha512-NP9APE+tO+LuJGn7/9+cohklunJsXWiaWEfV3si4Gi/XHDwVNgkwr1J3RQYFIvPy76GmJ9/bW8vyoU1LcxwKHA==} engines: {node: ^10 || ^12 || >=14} @@ -2949,6 +2970,10 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + change-case@5.4.4: resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} @@ -2966,6 +2991,18 @@ packages: class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} + cli-boxes@4.0.1: + resolution: {integrity: sha512-5IOn+jcCEHEraYolBPs/sT4BxYCe2nHg374OPiItB1O96KZFseS2gthU4twyYzeDcFew4DaUM/xwc5BQf08JJw==} + engines: {node: '>=18.20 <19 || >=20.10'} + + cli-cursor@4.0.0: + resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + cli-truncate@6.0.0: + resolution: {integrity: sha512-3+YKIUFsohD9MIoOFPFBldjAlnfCmCDcqe6aYGFqlDTRKg80p4wg35L+j83QQ63iOlKRccEkbn8IuM++HsgEjA==} + engines: {node: '>=22'} + client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} @@ -2977,6 +3014,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + code-excerpt@4.0.0: + resolution: {integrity: sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + codemirror@6.0.2: resolution: {integrity: sha512-VhydHotNW5w1UGK0Qj96BwSk/Zqbp9WbnyK2W/eVMv4QyF41INRGpjUhFJY7/uDNuudSc33a/PKr4iDqRduvHw==} @@ -3032,6 +3073,10 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + convert-to-spaces@2.0.1: + resolution: {integrity: sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + crelt@1.0.6: resolution: {integrity: sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==} @@ -3129,6 +3174,10 @@ packages: resolution: {integrity: sha512-Qohcme7V1inbAfvjItgw0EaxVX5q2rdVEZHRBrEQdRZTssLDGsL8Lwrznl8oQ/6kuTJONLaDcGjkNP247XEhcA==} engines: {node: '>=10.13.0'} + environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + es-define-property@1.0.1: resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} engines: {node: '>= 0.4'} @@ -3151,6 +3200,9 @@ packages: resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} engines: {node: '>= 0.4'} + es-toolkit@1.45.1: + resolution: {integrity: sha512-/jhoOj/Fx+A+IIyDNOvO3TItGmlMKhtX8ISAHKE90c4b/k1tqaqEZ+uUqfpU8DMnW5cgNJv606zS55jGvza0Xw==} + esbuild@0.28.0: resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} engines: {node: '>=18'} @@ -3160,6 +3212,10 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + escape-string-regexp@2.0.0: + resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} + engines: {node: '>=8'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -3379,6 +3435,10 @@ packages: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} + get-east-asian-width@1.5.0: + resolution: {integrity: sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==} + engines: {node: '>=18'} + get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -3525,6 +3585,10 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + indent-string@5.0.0: + resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} + engines: {node: '>=12'} + index-to-position@1.2.0: resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} engines: {node: '>=18'} @@ -3532,6 +3596,19 @@ packages: indexof@0.0.1: resolution: {integrity: sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg==} + ink@7.0.0: + resolution: {integrity: sha512-fMie5/VwIYXofMyND0s+fOVhwVBBPYx+uuqJ6V6rUBGjui+2UYp+0fWtvhSeKT4z+X1uH98a4ge5Vj3aTlL6mg==} + engines: {node: '>=22'} + peerDependencies: + '@types/react': '>=19.2.0' + react: '>=19.2.0' + react-devtools-core: '>=6.1.2' + peerDependenciesMeta: + '@types/react': + optional: true + react-devtools-core: + optional: true + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -3540,10 +3617,19 @@ packages: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + is-fullwidth-code-point@5.1.0: + resolution: {integrity: sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==} + engines: {node: '>=18'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} + is-in-ci@2.0.0: + resolution: {integrity: sha512-cFeerHriAnhrQSbpAxL37W1wcJKUUX07HyLWZCW1URJT/ra3GyUTzBgUnh24TMVfNTV2Hij2HLxkPHFZfOZy5w==} + engines: {node: '>=20'} + hasBin: true + is-number@7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} @@ -3777,6 +3863,10 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + minimatch@10.1.1: resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} engines: {node: 20 || >=22} @@ -3877,6 +3967,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + openapi-fetch@0.13.8: resolution: {integrity: sha512-yJ4QKRyNxE44baQ9mY5+r/kAzZ8yXMemtNAOFwOzRXJscdjSxxzWSNlyBAr+o5JjkUw9Lc3W7OIoca0cY3PYnQ==} @@ -3924,6 +4018,10 @@ packages: parseuri@0.0.6: resolution: {integrity: sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow==} + patch-console@2.0.0: + resolution: {integrity: sha512-0YNdUceMdaQwoKce1gatDScmMo5pu/tfABfnzEqeG0gtTmd7mh/WcwgUjtAeOU7N8nFFlbQBnFK2gXW5fGvmMA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4116,6 +4214,12 @@ packages: peerDependencies: react: ^19.2.4 + react-reconciler@0.33.0: + resolution: {integrity: sha512-KetWRytFv1epdpJc3J4G75I4WrplZE5jOL7Yq0p34+OVOKF4Se7WrdIdVC45XsSSmUTlht2FM/fM1FZb1mfQeA==} + engines: {node: '>=0.10.0'} + peerDependencies: + react: ^19.2.0 + react-refresh@0.17.0: resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} engines: {node: '>=0.10.0'} @@ -4177,6 +4281,10 @@ packages: resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + restore-cursor@4.0.0: + resolution: {integrity: sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -4261,10 +4369,17 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + slice-ansi@9.0.0: + resolution: {integrity: sha512-SO/3iYL5S3W57LLEniscOGPZgOqZUPCx6d3dB+52B80yJ0XstzsC/eV8gnA4tM3MHDrKz+OCFSLNjswdSC+/bA==} + engines: {node: '>=22'} + socket.io-client@2.5.0: resolution: {integrity: sha512-lOO9clmdgssDykiOmVQQitwBAF3I6mYcQAo7hQ7AM6Ny5X7fp8hIJ3HcQs3Rjz4SoggoxA1OgrQyY8EgTbcPYw==} @@ -4302,6 +4417,10 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + stack-utils@2.0.6: + resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} + engines: {node: '>=10'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -4316,6 +4435,10 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} + string-width@8.2.0: + resolution: {integrity: sha512-6hJPQ8N0V0P3SNmP6h2J99RLuzrWz2gvT7VnK5tKvrNqJoyS9W4/Fb8mo31UiPvy00z7DQXkP2hnKBVav76thw==} + engines: {node: '>=20'} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -4405,6 +4528,10 @@ packages: resolution: {integrity: sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw==} engines: {node: ^14.18.0 || >=16.0.0} + tagged-tag@1.0.0: + resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} + engines: {node: '>=20'} + tailwind-merge@3.5.0: resolution: {integrity: sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A==} @@ -4415,6 +4542,10 @@ packages: resolution: {integrity: sha512-1MOpMXuhGzGL5TTCZFItxCc0AARf1EZFQkGqMm7ERKj8+Hgr5oLvJOVFcC+lRmR8hCe2S3jC4T5D7Vg/d7/fhA==} engines: {node: '>=6'} + terminal-size@4.0.1: + resolution: {integrity: sha512-avMLDQpUI9I5XFrklECw1ZEUPJhqzcwSWsyyI8blhRLT+8N1jLJWLWWYQpB2q2xthq8xDvjZPISVh53T/+CLYQ==} + engines: {node: '>=18'} + terser-webpack-plugin@5.4.0: resolution: {integrity: sha512-Bn5vxm48flOIfkdl5CaD2+1CiUVbonWQ3KQPyP7/EuIl9Gbzq/gQFOzaMFUEgVjB1396tcK0SG8XcNJ/2kDH8g==} engines: {node: '>= 10.13.0'} @@ -4512,6 +4643,10 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} + type-fest@5.5.0: + resolution: {integrity: sha512-PlBfpQwiUvGViBNX84Yxwjsdhd1TUlXr6zjX7eoirtCPIr08NAmxwa+fcYBTeRQxHo9YC9wwF3m9i700sHma8g==} + engines: {node: '>=20'} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -4730,10 +4865,18 @@ packages: engines: {node: '>=8'} hasBin: true + widest-line@6.0.0: + resolution: {integrity: sha512-U89AsyEeAsyoF0zVJBkG9zBgekjgjK7yk9sje3F4IQpXBJ10TF6ByLlIfjMhcmHMJgHZI4KHt4rdNfktzxIAMA==} + engines: {node: '>=20'} + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@10.0.0: + resolution: {integrity: sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==} + engines: {node: '>=20'} + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4769,6 +4912,18 @@ packages: utf-8-validate: optional: true + ws@8.20.0: + resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xmlhttprequest-ssl@1.6.3: resolution: {integrity: sha512-3XfeQE/wNkvrIktn2Kf0869fC0BN6UpydVasGIeSm2B1Llihf7/0UfZM+eCkOw3P7bP4+qPgqhm7ZoxuJtFU0Q==} engines: {node: '>=0.4.0'} @@ -4807,6 +4962,9 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yoga-layout@3.2.1: + resolution: {integrity: sha512-0LPOt3AxKqMdFBZA3HBAt/t/8vIKq7VaQYbuA8WxCgung+p9TVyKRYdpvCb80HcdTN2NkbIKbhNwKUfm3tQywQ==} + zhead@2.2.4: resolution: {integrity: sha512-8F0OI5dpWIA5IGG5NHUg9staDwz/ZPxZtvGVf01j7vHqSyZ0raHY+78atOVxRqb73AotX22uV1pXt3gYSstGag==} @@ -4824,6 +4982,11 @@ packages: snapshots: + '@alcalzone/ansi-tokenize@0.3.0': + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 5.1.0 + '@alloc/quick-lru@5.2.0': {} '@aws-crypto/sha256-browser@5.2.0': @@ -6738,14 +6901,14 @@ snapshots: transitivePeerDependencies: - pg-native - '@stripe/sync-engine@file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)': + '@stripe/sync-engine@file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14)': dependencies: '@hono/node-server': 1.19.11(hono@4.12.8) '@scalar/hono-api-reference': 0.6.0(hono@4.12.8) '@stripe/sync-destination-google-sheets': file:packages/destination-google-sheets '@stripe/sync-destination-postgres': file:packages/destination-postgres(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) '@stripe/sync-hono-zod-openapi': file:packages/hono-zod-openapi - '@stripe/sync-integration-supabase': file:apps/supabase(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) + '@stripe/sync-integration-supabase': file:apps/supabase(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-protocol': file:packages/protocol '@stripe/sync-source-stripe': file:packages/source-stripe '@stripe/sync-state-postgres': file:packages/state-postgres @@ -6755,19 +6918,23 @@ snapshots: dotenv: 16.6.1 googleapis: 148.0.0 hono: 4.12.8 + ink: 7.0.0(@types/react@19.2.14)(react@19.2.4) openapi-fetch: 0.17.0 pg: 8.16.3 pino: 10.1.0 pino-pretty: 13.1.3 - ws: 8.18.3 + react: 19.2.4 + ws: 8.20.0 zod: 4.3.6 transitivePeerDependencies: - '@aws-sdk/client-sts' - '@aws-sdk/rds-signer' + - '@types/react' - bufferutil - debug - encoding - pg-native + - react-devtools-core - supports-color - utf-8-validate @@ -6778,10 +6945,10 @@ snapshots: zod: 4.3.6 zod-openapi: 5.4.6(zod@4.3.6) - '@stripe/sync-integration-supabase@file:apps/supabase(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)': + '@stripe/sync-integration-supabase@file:apps/supabase(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14)': dependencies: '@stripe/sync-destination-postgres': file:packages/destination-postgres(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) - '@stripe/sync-engine': file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) + '@stripe/sync-engine': file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-protocol': file:packages/protocol '@stripe/sync-source-stripe': file:packages/source-stripe '@stripe/sync-state-postgres': file:packages/state-postgres @@ -6789,10 +6956,12 @@ snapshots: transitivePeerDependencies: - '@aws-sdk/client-sts' - '@aws-sdk/rds-signer' + - '@types/react' - bufferutil - debug - encoding - pg-native + - react-devtools-core - supports-color - utf-8-validate @@ -6805,13 +6974,13 @@ snapshots: citty: 0.1.6 zod: 4.3.6 - '@stripe/sync-service@file:apps/service(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(tslib@2.8.1)': + '@stripe/sync-service@file:apps/service(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14)(tslib@2.8.1)': dependencies: '@hono/node-server': 1.19.11(hono@4.12.8) '@scalar/hono-api-reference': 0.6.0(hono@4.12.8) '@stripe/sync-destination-google-sheets': file:packages/destination-google-sheets '@stripe/sync-destination-postgres': file:packages/destination-postgres(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) - '@stripe/sync-engine': file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0) + '@stripe/sync-engine': file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) '@stripe/sync-hono-zod-openapi': file:packages/hono-zod-openapi '@stripe/sync-protocol': file:packages/protocol '@stripe/sync-source-stripe': file:packages/source-stripe @@ -6831,10 +7000,13 @@ snapshots: - '@aws-sdk/client-sts' - '@aws-sdk/rds-signer' - '@swc/helpers' + - '@types/react' - bufferutil + - debug - encoding - esbuild - pg-native + - react-devtools-core - supports-color - tslib - uglify-js @@ -6848,7 +7020,7 @@ snapshots: https-proxy-agent: 7.0.6(supports-color@10.2.2) pino: 10.1.0 undici: 7.24.6 - ws: 8.18.3 + ws: 8.20.0 zod: 4.3.6 transitivePeerDependencies: - bufferutil @@ -7306,6 +7478,14 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 + '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': + dependencies: + '@vitest/spy': 3.2.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 3.2.4 @@ -7479,6 +7659,10 @@ snapshots: ansi-colors@4.1.3: {} + ansi-escapes@7.3.0: + dependencies: + environment: 1.1.0 + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -7503,6 +7687,8 @@ snapshots: atomic-sleep@1.0.0: {} + auto-bind@5.0.1: {} + autoprefixer@10.4.27(postcss@8.5.8): dependencies: browserslist: 4.28.1 @@ -7595,6 +7781,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + chalk@5.6.2: {} + change-case@5.4.4: {} check-error@2.1.1: {} @@ -7609,6 +7797,17 @@ snapshots: dependencies: clsx: 2.1.1 + cli-boxes@4.0.1: {} + + cli-cursor@4.0.0: + dependencies: + restore-cursor: 4.0.0 + + cli-truncate@6.0.0: + dependencies: + slice-ansi: 9.0.0 + string-width: 8.2.0 + client-only@0.0.1: {} cliui@8.0.1: @@ -7619,6 +7818,10 @@ snapshots: clsx@2.1.1: {} + code-excerpt@4.0.0: + dependencies: + convert-to-spaces: 2.0.1 + codemirror@6.0.2: dependencies: '@codemirror/autocomplete': 6.20.1 @@ -7670,6 +7873,8 @@ snapshots: convert-source-map@2.0.0: {} + convert-to-spaces@2.0.1: {} + crelt@1.0.6: {} cross-spawn@7.0.6: @@ -7763,6 +7968,8 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.3.2 + environment@1.1.0: {} + es-define-property@1.0.1: {} es-errors@1.3.0: {} @@ -7782,6 +7989,8 @@ snapshots: has-tostringtag: 1.0.2 hasown: 2.0.2 + es-toolkit@1.45.1: {} + esbuild@0.28.0: optionalDependencies: '@esbuild/aix-ppc64': 0.28.0 @@ -7813,6 +8022,8 @@ snapshots: escalade@3.2.0: {} + escape-string-regexp@2.0.0: {} + escape-string-regexp@4.0.0: {} eslint-config-prettier@10.1.8(eslint@9.39.1(jiti@2.6.1)): @@ -8038,6 +8249,8 @@ snapshots: get-caller-file@2.0.5: {} + get-east-asian-width@1.5.0: {} + get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -8199,18 +8412,60 @@ snapshots: imurmurhash@0.1.4: {} + indent-string@5.0.0: {} + index-to-position@1.2.0: {} indexof@0.0.1: {} + ink@7.0.0(@types/react@19.2.14)(react@19.2.4): + dependencies: + '@alcalzone/ansi-tokenize': 0.3.0 + ansi-escapes: 7.3.0 + ansi-styles: 6.2.3 + auto-bind: 5.0.1 + chalk: 5.6.2 + cli-boxes: 4.0.1 + cli-cursor: 4.0.0 + cli-truncate: 6.0.0 + code-excerpt: 4.0.0 + es-toolkit: 1.45.1 + indent-string: 5.0.0 + is-in-ci: 2.0.0 + patch-console: 2.0.0 + react: 19.2.4 + react-reconciler: 0.33.0(react@19.2.4) + scheduler: 0.27.0 + signal-exit: 3.0.7 + slice-ansi: 9.0.0 + stack-utils: 2.0.6 + string-width: 8.2.0 + terminal-size: 4.0.1 + type-fest: 5.5.0 + widest-line: 6.0.0 + wrap-ansi: 10.0.0 + ws: 8.20.0 + yoga-layout: 3.2.1 + optionalDependencies: + '@types/react': 19.2.14 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} + is-fullwidth-code-point@5.1.0: + dependencies: + get-east-asian-width: 1.5.0 + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 + is-in-ci@2.0.0: {} + is-number@7.0.0: {} is-stream@2.0.1: {} @@ -8402,6 +8657,8 @@ snapshots: dependencies: mime-db: 1.52.0 + mimic-fn@2.1.0: {} + minimatch@10.1.1: dependencies: '@isaacs/brace-expansion': 5.0.0 @@ -8482,6 +8739,10 @@ snapshots: dependencies: wrappy: 1.0.2 + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + openapi-fetch@0.13.8: dependencies: openapi-typescript-helpers: 0.0.15 @@ -8537,6 +8798,8 @@ snapshots: parseuri@0.0.6: {} + patch-console@2.0.0: {} + path-exists@4.0.0: {} path-expression-matcher@1.1.3: {} @@ -8743,6 +9006,11 @@ snapshots: react: 19.2.4 scheduler: 0.27.0 + react-reconciler@0.33.0(react@19.2.4): + dependencies: + react: 19.2.4 + scheduler: 0.27.0 + react-refresh@0.17.0: {} react-remove-scroll-bar@2.3.8(@types/react@19.2.14)(react@19.2.4): @@ -8792,6 +9060,11 @@ snapshots: resolve-pkg-maps@1.0.0: {} + restore-cursor@4.0.0: + dependencies: + onetime: 5.1.2 + signal-exit: 3.0.7 + reusify@1.1.0: {} rimraf@6.1.0: @@ -8926,8 +9199,15 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + signal-exit@4.1.0: {} + slice-ansi@9.0.0: + dependencies: + ansi-styles: 6.2.3 + is-fullwidth-code-point: 5.1.0 + socket.io-client@2.5.0: dependencies: backo2: 1.0.2 @@ -8981,6 +9261,10 @@ snapshots: split2@4.2.0: {} + stack-utils@2.0.6: + dependencies: + escape-string-regexp: 2.0.0 + stackback@0.0.2: {} std-env@3.9.0: {} @@ -8997,6 +9281,11 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.1.2 + string-width@8.2.0: + dependencies: + get-east-asian-width: 1.5.0 + strip-ansi: 7.1.2 + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -9068,12 +9357,16 @@ snapshots: dependencies: '@pkgr/core': 0.2.9 + tagged-tag@1.0.0: {} + tailwind-merge@3.5.0: {} tailwindcss@4.2.2: {} tapable@2.3.2: {} + terminal-size@4.0.1: {} + terser-webpack-plugin@5.4.0(@swc/core@1.15.21)(webpack@5.105.4(@swc/core@1.15.21)): dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -9151,6 +9444,10 @@ snapshots: type-fest@4.41.0: {} + type-fest@5.5.0: + dependencies: + tagged-tag: 1.0.0 + typescript@5.9.3: {} undici-types@6.21.0: {} @@ -9295,7 +9592,7 @@ snapshots: dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) + '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -9432,8 +9729,18 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + widest-line@6.0.0: + dependencies: + string-width: 8.2.0 + word-wrap@1.2.5: {} + wrap-ansi@10.0.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 8.2.0 + strip-ansi: 7.1.2 + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 @@ -9452,6 +9759,8 @@ snapshots: ws@8.18.3: {} + ws@8.20.0: {} + xmlhttprequest-ssl@1.6.3: {} xtend@4.0.2: {} @@ -9481,6 +9790,8 @@ snapshots: yocto-queue@0.1.0: {} + yoga-layout@3.2.1: {} + zhead@2.2.4: {} zod-openapi@5.4.6(zod@4.3.6): From 41678623ba66cc0c331b7755ebe65fe1c12a0676 Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 21:24:50 +0000 Subject: [PATCH 04/32] chore: regenerate OpenAPI specs, fix formatting, update lockfile Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/src/__generated__/openapi.d.ts | 56 +++--- apps/engine/src/__generated__/openapi.json | 122 +++++++----- apps/engine/src/cli/sync-ui.tsx | 48 +++-- apps/engine/src/cli/sync.ts | 3 +- apps/engine/src/lib/sync-progress-state.ts | 13 +- apps/service/src/__generated__/openapi.d.ts | 88 +++++---- apps/service/src/__generated__/openapi.json | 204 ++++++++++++-------- pnpm-lock.yaml | 35 ++-- 8 files changed, 330 insertions(+), 239 deletions(-) diff --git a/apps/engine/src/__generated__/openapi.d.ts b/apps/engine/src/__generated__/openapi.d.ts index 126953a8..085fda15 100644 --- a/apps/engine/src/__generated__/openapi.d.ts +++ b/apps/engine/src/__generated__/openapi.d.ts @@ -465,7 +465,7 @@ export interface components { * @enum {string} */ type: "trace"; - /** @description Diagnostic/status payload with subtypes for error, stream status, estimates, and progress. */ + /** @description Diagnostic/status payload with subtypes for error, stream status, estimates, and global progress. */ trace: { /** @constant */ trace_type: "error"; @@ -486,25 +486,19 @@ export interface components { } | { /** @constant */ trace_type: "stream_status"; - /** @description Per-stream status update. Sources emit the minimal form (stream + status). The engine emits enriched versions with record counts and throughput rates. */ + /** @description Per-stream status update. Sources emit the minimal form (stream + status). The engine enriches with record counts. Only emitted on status transitions. */ stream_status: { /** @description Stream being reported on. */ stream: string; /** - * @description Current phase of the stream within this sync run. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. Sources may store richer error statuses internally for retry logic. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all sync runs. Monotonically increasing; initialized from engine state on resume. Set by the engine, not the source. */ cumulative_record_count?: number; /** @description Records synced for this stream in the current sync run. Set by the engine. */ run_record_count?: number; - /** @description Records synced since the last stream_status emission for this stream. Set by the engine. Used for instantaneous per-stream throughput. */ - window_record_count?: number; - /** @description Average records per second for this stream over the entire run: run_record_count / elapsed seconds. Set by the engine. */ - records_per_second?: number; - /** @description Average API requests per second for this stream over the entire run. Set by the engine from source-reported request counts. */ - requests_per_second?: number; }; } | { /** @constant */ @@ -520,19 +514,27 @@ export interface components { }; } | { /** @constant */ - trace_type: "progress"; - /** @description Periodic global sync progress emitted by the engine. Aggregate stats only β€” per-stream detail is in stream_status messages. Each emission is a full replacement. */ - progress: { + trace_type: "global_progress"; + /** @description Global sync progress emitted by the engine, co-emitted with every stream_status trace. Aggregate stats only β€” per-stream detail is in stream_status messages. Each emission is a full replacement. */ + global_progress: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; }; }; @@ -642,35 +644,39 @@ export interface components { elapsed_ms?: number; /** @description Full sync state at the end of the run. source: accumulated from source_state messages; engine: updated cumulative record counts; destination: reserved. Consumers can persist this directly and pass it back on resume. */ state?: components["schemas"]["SyncState"]; - /** @description Final global aggregates. Same shape as trace/progress. */ + /** @description Final global aggregates. Same shape as trace/global_progress. */ global_progress?: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; /** @description Per-stream end-of-sync summary. Errors only appear here, not in stream_status messages. */ stream_progress?: { [key: string]: { /** - * @description Final stream status. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all runs. */ cumulative_record_count: number; /** @description Records synced in this run. */ run_record_count: number; - /** @description Average records/sec for this stream over the run. */ - records_per_second?: number; - /** @description Average requests/sec for this stream over the run. */ - requests_per_second?: number; /** @description All accumulated errors for this stream during this run. */ errors?: { /** @description Human-readable error description. */ @@ -770,7 +776,7 @@ export interface components { Message: components["schemas"]["RecordMessage"] | components["schemas"]["SourceStateMessage"] | components["schemas"]["CatalogMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["TraceMessage"] | components["schemas"]["SpecMessage"] | components["schemas"]["ConnectionStatusMessage"] | components["schemas"]["ControlMessage"] | components["schemas"]["EofMessage"]; DiscoverOutput: components["schemas"]["CatalogMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["TraceMessage"]; DestinationOutput: components["schemas"]["SourceStateMessage"] | components["schemas"]["TraceMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["EofMessage"]; - SyncOutput: components["schemas"]["SourceStateMessage"] | components["schemas"]["TraceMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["EofMessage"] | components["schemas"]["ControlMessage"]; + SyncOutput: components["schemas"]["SourceStateMessage"] | components["schemas"]["CatalogMessage"] | components["schemas"]["TraceMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["EofMessage"] | components["schemas"]["ControlMessage"]; CheckOutput: components["schemas"]["ConnectionStatusMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["TraceMessage"]; SetupOutput: components["schemas"]["ControlMessage"] | components["schemas"]["LogMessage"] | components["schemas"]["TraceMessage"]; TeardownOutput: components["schemas"]["LogMessage"] | components["schemas"]["TraceMessage"]; diff --git a/apps/engine/src/__generated__/openapi.json b/apps/engine/src/__generated__/openapi.json index 14172963..f427c24e 100644 --- a/apps/engine/src/__generated__/openapi.json +++ b/apps/engine/src/__generated__/openapi.json @@ -1523,14 +1523,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Current phase of the stream within this sync run." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. Sources may store richer error statuses internally for retry logic." }, "cumulative_record_count": { "description": "Cumulative records synced for this stream across all sync runs. Monotonically increasing; initialized from engine state on resume. Set by the engine, not the source.", @@ -1543,27 +1538,13 @@ "type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991 - }, - "window_record_count": { - "description": "Records synced since the last stream_status emission for this stream. Set by the engine. Used for instantaneous per-stream throughput.", - "type": "integer", - "minimum": -9007199254740991, - "maximum": 9007199254740991 - }, - "records_per_second": { - "description": "Average records per second for this stream over the entire run: run_record_count / elapsed seconds. Set by the engine.", - "type": "number" - }, - "requests_per_second": { - "description": "Average API requests per second for this stream over the entire run. Set by the engine from source-reported request counts.", - "type": "number" } }, "required": [ "stream", "status" ], - "description": "Per-stream status update. Sources emit the minimal form (stream + status). The engine emits enriched versions with record counts and throughput rates." + "description": "Per-stream status update. Sources emit the minimal form (stream + status). The engine enriches with record counts. Only emitted on status transitions." } }, "required": [ @@ -1614,9 +1595,9 @@ "properties": { "trace_type": { "type": "string", - "const": "progress" + "const": "global_progress" }, - "progress": { + "global_progress": { "type": "object", "properties": { "elapsed_ms": { @@ -1631,11 +1612,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -1644,25 +1631,43 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ], - "description": "Periodic global sync progress emitted by the engine. Aggregate stats only β€” per-stream detail is in stream_status messages. Each emission is a full replacement." + "description": "Global sync progress emitted by the engine, co-emitted with every stream_status trace. Aggregate stats only β€” per-stream detail is in stream_status messages. Each emission is a full replacement." } }, "required": [ "trace_type", - "progress" + "global_progress" ] } ], - "description": "Diagnostic/status payload with subtypes for error, stream status, estimates, and progress.", + "description": "Diagnostic/status payload with subtypes for error, stream status, estimates, and global progress.", "type": "object", "discriminator": { "propertyName": "trace_type" @@ -1893,7 +1898,7 @@ "$ref": "#/components/schemas/SyncState" }, "global_progress": { - "description": "Final global aggregates. Same shape as trace/progress.", + "description": "Final global aggregates. Same shape as trace/global_progress.", "type": "object", "properties": { "elapsed_ms": { @@ -1908,11 +1913,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -1921,13 +1932,31 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ] }, @@ -1944,14 +1973,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Final stream status." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors." }, "cumulative_record_count": { "type": "integer", @@ -1965,14 +1989,6 @@ "maximum": 9007199254740991, "description": "Records synced in this run." }, - "records_per_second": { - "description": "Average records/sec for this stream over the run.", - "type": "number" - }, - "requests_per_second": { - "description": "Average requests/sec for this stream over the run.", - "type": "number" - }, "errors": { "description": "All accumulated errors for this stream during this run.", "type": "array", @@ -2374,6 +2390,9 @@ { "$ref": "#/components/schemas/SourceStateMessage" }, + { + "$ref": "#/components/schemas/CatalogMessage" + }, { "$ref": "#/components/schemas/TraceMessage" }, @@ -2392,6 +2411,7 @@ "propertyName": "type", "mapping": { "source_state": "#/components/schemas/SourceStateMessage", + "catalog": "#/components/schemas/CatalogMessage", "trace": "#/components/schemas/TraceMessage", "log": "#/components/schemas/LogMessage", "eof": "#/components/schemas/EofMessage", diff --git a/apps/engine/src/cli/sync-ui.tsx b/apps/engine/src/cli/sync-ui.tsx index 675d5e84..d10072b9 100644 --- a/apps/engine/src/cli/sync-ui.tsx +++ b/apps/engine/src/cli/sync-ui.tsx @@ -61,7 +61,15 @@ function Divider({ width = 60 }: { width?: number }) { return {'─'.repeat(width)} } -function StatRow({ label, value, dimLabel = false }: { label: string; value: string; dimLabel?: boolean }) { +function StatRow({ + label, + value, + dimLabel = false, +}: { + label: string + value: string + dimLabel?: boolean +}) { return ( {label} @@ -158,11 +166,7 @@ export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressPro complete.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) started.sort((a, b) => b[1].cumulative_record_count - a[1].cumulative_record_count) - const allStreamNames = [ - ...complete.map((c) => c[0]), - ...started.map((s) => s[0]), - ...pending, - ] + const allStreamNames = [...complete.map((c) => c[0]), ...started.map((s) => s[0]), ...pending] const nameWidth = Math.max(...allStreamNames.map((n) => n.length), 12) const errCount = Object.values(sp).filter((i) => (i.errors?.length ?? 0) > 0).length @@ -175,17 +179,17 @@ export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressPro {final ? ( - {reasonLabel} - {attempt != null && attempt > 1 && ( - ({attempt} attempts) - )} + + {reasonLabel} + + {attempt != null && attempt > 1 && ({attempt} attempts)} ) : ( - Syncing - {attempt != null && attempt > 1 && ( - Β· attempt {attempt} - )} + + Syncing + + {attempt != null && attempt > 1 && Β· attempt {attempt}} {gp && ( {' '} @@ -212,8 +216,14 @@ export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressPro : '' }`} /> - - + + {final && gp.records_per_second > 0 && ( )} @@ -276,7 +286,7 @@ export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressPro )} {/* ── Footer summary ── */} - {(complete.length + started.length + pending.length > 0) && ( + {complete.length + started.length + pending.length > 0 && ( @@ -289,9 +299,7 @@ export function SyncProgressUI({ eof, catalog, final, attempt }: SyncProgressPro {gp && gp.run_record_count > 0 && ( +{fmt(gp.run_record_count)} records this run )} - {eof.cutoff && ( - cutoff: {eof.cutoff} - )} + {eof.cutoff && cutoff: {eof.cutoff}} ) diff --git a/apps/engine/src/cli/sync.ts b/apps/engine/src/cli/sync.ts index 5d87fddc..f31256c9 100644 --- a/apps/engine/src/cli/sync.ts +++ b/apps/engine/src/cli/sync.ts @@ -52,7 +52,8 @@ export function createSyncCmd(engine: Engine, _resolver: ConnectorResolver) { }, baseUrl: { type: 'string', - description: 'Stripe API base URL (or STRIPE_API_BASE env, default: https://api.stripe.com)', + description: + 'Stripe API base URL (or STRIPE_API_BASE env, default: https://api.stripe.com)', }, progress: { type: 'boolean', diff --git a/apps/engine/src/lib/sync-progress-state.ts b/apps/engine/src/lib/sync-progress-state.ts index 7bcdb4f6..8d35b8aa 100644 --- a/apps/engine/src/lib/sync-progress-state.ts +++ b/apps/engine/src/lib/sync-progress-state.ts @@ -52,7 +52,8 @@ export function createSyncDisplayState(): { const ss = t.stream_status const sp = ensureStream(ss.stream) sp.status = ss.status as 'started' | 'complete' - if (ss.cumulative_record_count != null) sp.cumulative_record_count = ss.cumulative_record_count + if (ss.cumulative_record_count != null) + sp.cumulative_record_count = ss.cumulative_record_count if (ss.run_record_count != null) sp.run_record_count = ss.run_record_count return true } @@ -74,7 +75,12 @@ export function createSyncDisplayState(): { if (!sp.errors) sp.errors = [] sp.errors.push({ message: err.message, - failure_type: err.failure_type as 'config_error' | 'system_error' | 'transient_error' | 'auth_error' | undefined, + failure_type: err.failure_type as + | 'config_error' + | 'system_error' + | 'transient_error' + | 'auth_error' + | undefined, }) } return false @@ -183,8 +189,7 @@ export function renderSyncProgress( function streamLine(name: string, info: EofStreamProgress) { const cum = info.cumulative_record_count const run = info.run_record_count - const countStr = - cum > 0 ? `${fmt(cum).padStart(10)}${run > 0 ? ` (+${fmt(run)})` : ''}` : '' + const countStr = cum > 0 ? `${fmt(cum).padStart(10)}${run > 0 ? ` (+${fmt(run)})` : ''}` : '' lines.push(` ${name.padEnd(maxName)} ${countStr}`) for (const err of info.errors ?? []) { const emoji = ERROR_EMOJI[err.failure_type ?? 'system_error'] ?? '❌' diff --git a/apps/service/src/__generated__/openapi.d.ts b/apps/service/src/__generated__/openapi.d.ts index 9e00a197..6c1e87c4 100644 --- a/apps/service/src/__generated__/openapi.d.ts +++ b/apps/service/src/__generated__/openapi.d.ts @@ -317,35 +317,39 @@ export interface operations { elapsed_ms?: number; /** @description Full sync state at the end of the run. source: accumulated from source_state messages; engine: updated cumulative record counts; destination: reserved. Consumers can persist this directly and pass it back on resume. */ state?: components["schemas"]["SyncState"]; - /** @description Final global aggregates. Same shape as trace/progress. */ + /** @description Final global aggregates. Same shape as trace/global_progress. */ global_progress?: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; /** @description Per-stream end-of-sync summary. Errors only appear here, not in stream_status messages. */ stream_progress?: { [key: string]: { /** - * @description Final stream status. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all runs. */ cumulative_record_count: number; /** @description Records synced in this run. */ run_record_count: number; - /** @description Average records/sec for this stream over the run. */ - records_per_second?: number; - /** @description Average requests/sec for this stream over the run. */ - requests_per_second?: number; /** @description All accumulated errors for this stream during this run. */ errors?: { /** @description Human-readable error description. */ @@ -445,35 +449,39 @@ export interface operations { elapsed_ms?: number; /** @description Full sync state at the end of the run. source: accumulated from source_state messages; engine: updated cumulative record counts; destination: reserved. Consumers can persist this directly and pass it back on resume. */ state?: components["schemas"]["SyncState"]; - /** @description Final global aggregates. Same shape as trace/progress. */ + /** @description Final global aggregates. Same shape as trace/global_progress. */ global_progress?: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; /** @description Per-stream end-of-sync summary. Errors only appear here, not in stream_status messages. */ stream_progress?: { [key: string]: { /** - * @description Final stream status. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all runs. */ cumulative_record_count: number; /** @description Records synced in this run. */ run_record_count: number; - /** @description Average records/sec for this stream over the run. */ - records_per_second?: number; - /** @description Average requests/sec for this stream over the run. */ - requests_per_second?: number; /** @description All accumulated errors for this stream during this run. */ errors?: { /** @description Human-readable error description. */ @@ -565,35 +573,39 @@ export interface operations { elapsed_ms?: number; /** @description Full sync state at the end of the run. source: accumulated from source_state messages; engine: updated cumulative record counts; destination: reserved. Consumers can persist this directly and pass it back on resume. */ state?: components["schemas"]["SyncState"]; - /** @description Final global aggregates. Same shape as trace/progress. */ + /** @description Final global aggregates. Same shape as trace/global_progress. */ global_progress?: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; /** @description Per-stream end-of-sync summary. Errors only appear here, not in stream_status messages. */ stream_progress?: { [key: string]: { /** - * @description Final stream status. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all runs. */ cumulative_record_count: number; /** @description Records synced in this run. */ run_record_count: number; - /** @description Average records/sec for this stream over the run. */ - records_per_second?: number; - /** @description Average requests/sec for this stream over the run. */ - requests_per_second?: number; /** @description All accumulated errors for this stream during this run. */ errors?: { /** @description Human-readable error description. */ @@ -746,35 +758,39 @@ export interface operations { elapsed_ms?: number; /** @description Full sync state at the end of the run. source: accumulated from source_state messages; engine: updated cumulative record counts; destination: reserved. Consumers can persist this directly and pass it back on resume. */ state?: components["schemas"]["SyncState"]; - /** @description Final global aggregates. Same shape as trace/progress. */ + /** @description Final global aggregates. Same shape as trace/global_progress. */ global_progress?: { /** @description Wall-clock milliseconds since the sync run started. */ elapsed_ms: number; /** @description Total records synced across all streams in this run. */ run_record_count: number; + /** @description Total records synced across all streams across all runs. */ + cumulative_record_count?: number; /** @description Overall throughput for the entire run: run_record_count / elapsed seconds. */ - rows_per_second: number; + records_per_second: number; /** @description Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval. */ - window_rows_per_second: number; + window_records_per_second: number; /** @description Total source_state messages observed so far in this sync run. */ state_checkpoint_count: number; + /** @description Total API requests made by the source in this run. */ + request_count?: number; + /** @description Total API requests across all runs. */ + cumulative_request_count?: number; + /** @description Total wall-clock time across all runs. */ + cumulative_elapsed_ms?: number; }; /** @description Per-stream end-of-sync summary. Errors only appear here, not in stream_status messages. */ stream_progress?: { [key: string]: { /** - * @description Final stream status. + * @description Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors. * @enum {string} */ - status: "started" | "running" | "complete" | "transient_error" | "system_error" | "config_error" | "auth_error"; + status: "started" | "complete"; /** @description Cumulative records synced for this stream across all runs. */ cumulative_record_count: number; /** @description Records synced in this run. */ run_record_count: number; - /** @description Average records/sec for this stream over the run. */ - records_per_second?: number; - /** @description Average requests/sec for this stream over the run. */ - requests_per_second?: number; /** @description All accumulated errors for this stream during this run. */ errors?: { /** @description Human-readable error description. */ diff --git a/apps/service/src/__generated__/openapi.json b/apps/service/src/__generated__/openapi.json index fc895615..ce140883 100644 --- a/apps/service/src/__generated__/openapi.json +++ b/apps/service/src/__generated__/openapi.json @@ -157,7 +157,7 @@ "$ref": "#/components/schemas/SyncState" }, "global_progress": { - "description": "Final global aggregates. Same shape as trace/progress.", + "description": "Final global aggregates. Same shape as trace/global_progress.", "type": "object", "properties": { "elapsed_ms": { @@ -172,11 +172,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -185,13 +191,31 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ], "additionalProperties": false @@ -209,14 +233,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Final stream status." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors." }, "cumulative_record_count": { "type": "integer", @@ -230,14 +249,6 @@ "maximum": 9007199254740991, "description": "Records synced in this run." }, - "records_per_second": { - "description": "Average records/sec for this stream over the run.", - "type": "number" - }, - "requests_per_second": { - "description": "Average requests/sec for this stream over the run.", - "type": "number" - }, "errors": { "description": "All accumulated errors for this stream during this run.", "type": "array", @@ -468,7 +479,7 @@ "$ref": "#/components/schemas/SyncState" }, "global_progress": { - "description": "Final global aggregates. Same shape as trace/progress.", + "description": "Final global aggregates. Same shape as trace/global_progress.", "type": "object", "properties": { "elapsed_ms": { @@ -483,11 +494,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -496,13 +513,31 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ], "additionalProperties": false @@ -520,14 +555,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Final stream status." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors." }, "cumulative_record_count": { "type": "integer", @@ -541,14 +571,6 @@ "maximum": 9007199254740991, "description": "Records synced in this run." }, - "records_per_second": { - "description": "Average records/sec for this stream over the run.", - "type": "number" - }, - "requests_per_second": { - "description": "Average requests/sec for this stream over the run.", - "type": "number" - }, "errors": { "description": "All accumulated errors for this stream during this run.", "type": "array", @@ -747,7 +769,7 @@ "$ref": "#/components/schemas/SyncState" }, "global_progress": { - "description": "Final global aggregates. Same shape as trace/progress.", + "description": "Final global aggregates. Same shape as trace/global_progress.", "type": "object", "properties": { "elapsed_ms": { @@ -762,11 +784,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -775,13 +803,31 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ], "additionalProperties": false @@ -799,14 +845,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Final stream status." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors." }, "cumulative_record_count": { "type": "integer", @@ -820,14 +861,6 @@ "maximum": 9007199254740991, "description": "Records synced in this run." }, - "records_per_second": { - "description": "Average records/sec for this stream over the run.", - "type": "number" - }, - "requests_per_second": { - "description": "Average requests/sec for this stream over the run.", - "type": "number" - }, "errors": { "description": "All accumulated errors for this stream during this run.", "type": "array", @@ -1080,7 +1113,7 @@ "$ref": "#/components/schemas/SyncState" }, "global_progress": { - "description": "Final global aggregates. Same shape as trace/progress.", + "description": "Final global aggregates. Same shape as trace/global_progress.", "type": "object", "properties": { "elapsed_ms": { @@ -1095,11 +1128,17 @@ "maximum": 9007199254740991, "description": "Total records synced across all streams in this run." }, - "rows_per_second": { + "cumulative_record_count": { + "description": "Total records synced across all streams across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "records_per_second": { "type": "number", "description": "Overall throughput for the entire run: run_record_count / elapsed seconds." }, - "window_rows_per_second": { + "window_records_per_second": { "type": "number", "description": "Instantaneous throughput: total records in last window / window duration. Measures only the most recent reporting interval." }, @@ -1108,13 +1147,31 @@ "minimum": -9007199254740991, "maximum": 9007199254740991, "description": "Total source_state messages observed so far in this sync run." + }, + "request_count": { + "description": "Total API requests made by the source in this run.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_request_count": { + "description": "Total API requests across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "cumulative_elapsed_ms": { + "description": "Total wall-clock time across all runs.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "elapsed_ms", "run_record_count", - "rows_per_second", - "window_rows_per_second", + "records_per_second", + "window_records_per_second", "state_checkpoint_count" ], "additionalProperties": false @@ -1132,14 +1189,9 @@ "type": "string", "enum": [ "started", - "running", - "complete", - "transient_error", - "system_error", - "config_error", - "auth_error" + "complete" ], - "description": "Final stream status." + "description": "Lifecycle status. Errors are orthogonal β€” a stream can be complete with errors." }, "cumulative_record_count": { "type": "integer", @@ -1153,14 +1205,6 @@ "maximum": 9007199254740991, "description": "Records synced in this run." }, - "records_per_second": { - "description": "Average records/sec for this stream over the run.", - "type": "number" - }, - "requests_per_second": { - "description": "Average requests/sec for this stream over the run.", - "type": "number" - }, "errors": { "description": "All accumulated errors for this stream during this run.", "type": "array", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f880b91..358149a4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,13 +115,13 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) + version: link:../engine '@stripe/sync-service': specifier: workspace:* version: file:apps/service(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14)(tslib@2.8.1) '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../../packages/source-stripe '@tailwindcss/vite': specifier: ^4 version: 4.2.2(vite@6.4.1(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) @@ -172,7 +172,7 @@ importers: version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../../packages/source-stripe '@stripe/sync-state-postgres': specifier: workspace:* version: link:../../packages/state-postgres @@ -254,7 +254,7 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) + version: link:../engine '@stripe/sync-hono-zod-openapi': specifier: workspace:* version: link:../../packages/hono-zod-openapi @@ -263,7 +263,7 @@ importers: version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../../packages/source-stripe '@stripe/sync-ts-cli': specifier: workspace:* version: link:../../packages/ts-cli @@ -330,13 +330,13 @@ importers: version: link:../../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) + version: link:../engine '@stripe/sync-protocol': specifier: workspace:* version: link:../../packages/protocol '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../../packages/source-stripe '@stripe/sync-state-postgres': specifier: workspace:* version: link:../../packages/state-postgres @@ -373,7 +373,7 @@ importers: version: 0.2.17 '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../../packages/source-stripe codemirror: specifier: ^6.0.1 version: 6.0.2 @@ -429,7 +429,7 @@ importers: version: link:../packages/destination-postgres '@stripe/sync-engine': specifier: workspace:* - version: file:apps/engine(@aws-sdk/client-sts@3.1013.0)(@aws-sdk/rds-signer@3.1013.0)(@types/react@19.2.14) + version: link:../apps/engine '@stripe/sync-openapi': specifier: workspace:* version: link:../packages/openapi @@ -441,7 +441,7 @@ importers: version: link:../apps/service '@stripe/sync-source-stripe': specifier: workspace:* - version: file:packages/source-stripe + version: link:../packages/source-stripe '@stripe/sync-test-utils': specifier: workspace:* version: link:../packages/test-utils @@ -6924,7 +6924,7 @@ snapshots: pino: 10.1.0 pino-pretty: 13.1.3 react: 19.2.4 - ws: 8.20.0 + ws: 8.18.3 zod: 4.3.6 transitivePeerDependencies: - '@aws-sdk/client-sts' @@ -7002,7 +7002,6 @@ snapshots: - '@swc/helpers' - '@types/react' - bufferutil - - debug - encoding - esbuild - pg-native @@ -7020,7 +7019,7 @@ snapshots: https-proxy-agent: 7.0.6(supports-color@10.2.2) pino: 10.1.0 undici: 7.24.6 - ws: 8.20.0 + ws: 8.18.3 zod: 4.3.6 transitivePeerDependencies: - bufferutil @@ -7478,14 +7477,6 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': - dependencies: - '@vitest/spy': 3.2.4 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) - '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 3.2.4 @@ -9592,7 +9583,7 @@ snapshots: dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) + '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 From 636e54a7b036635293bdafbec6322c67f6c7190c Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 21:41:16 +0000 Subject: [PATCH 05/32] chore: trigger CI Committed-By-Agent: claude From 0a4db1b41204f0ce5b04c97ddac7550785f7255b Mon Sep 17 00:00:00 2001 From: tx Date: Wed, 15 Apr 2026 22:50:43 +0000 Subject: [PATCH 06/32] fix: remove hacky label option from withHttpRetry Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- packages/source-stripe/src/client.ts | 2 +- packages/source-stripe/src/resourceRegistry.ts | 4 ++-- packages/source-stripe/src/retry.ts | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/source-stripe/src/client.ts b/packages/source-stripe/src/client.ts index 7696f723..81464fba 100644 --- a/packages/source-stripe/src/client.ts +++ b/packages/source-stripe/src/client.ts @@ -98,7 +98,7 @@ export function makeClient( params?: Record ): Promise { if (method === 'GET') { - return withHttpRetry(() => request(method, path, params), { label: `${method} ${path}`, signal: pipelineSignal }) + return withHttpRetry(() => request(method, path, params), { signal: pipelineSignal }) } return request(method, path, params) } diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 604753d4..f3d55db7 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -119,14 +119,14 @@ export function buildResourceRegistry( supportsForwardPagination: isV2 || endpoint.supportsStartingAfter, sync: true, dependencies: [], - listFn: buildSpecAwareListFn((params) => withHttpRetry(() => rawListFn(params), { label: `LIST ${endpoint.apiPath} (${tableName})` }), { + listFn: buildSpecAwareListFn((params) => withHttpRetry(() => rawListFn(params)), { isV2, supportsLimit: endpoint.supportsLimit, supportsStartingAfter: endpoint.supportsStartingAfter, supportsEndingBefore: endpoint.supportsEndingBefore, supportsCreatedFilter: endpoint.supportsCreatedFilter, }), - retrieveFn: (id) => withHttpRetry(() => rawRetrieveFn(id), { label: `GET ${endpoint.apiPath}/${id} (${tableName})` }), + retrieveFn: (id) => withHttpRetry(() => rawRetrieveFn(id)), nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config diff --git a/packages/source-stripe/src/retry.ts b/packages/source-stripe/src/retry.ts index d33fafd5..e15589e8 100644 --- a/packages/source-stripe/src/retry.ts +++ b/packages/source-stripe/src/retry.ts @@ -15,8 +15,6 @@ const RETRYABLE_NETWORK_CODES = new Set([ ]) export type HttpRetryOptions = { - /** Human-readable label for log messages (e.g. "GET /v1/customers") */ - label?: string maxRetries?: number baseDelayMs?: number maxDelayMs?: number @@ -129,9 +127,8 @@ export async function withHttpRetry( const status = getHttpErrorStatus(err) const errName = err instanceof Error ? err.name : 'UnknownError' const errMsg = err instanceof Error ? err.message : String(err) - const labelPart = opts.label ? ` ${opts.label}` : '' console.error( - `[source-stripe] retry${labelPart} attempt=${attempt + 1}/${maxRetries} delay=${delayMs}ms status=${status ?? 'n/a'} error=${errName}: ${errMsg}` + `[source-stripe] retry attempt=${attempt + 1}/${maxRetries} delay=${delayMs}ms status=${status ?? 'n/a'} error=${errName}: ${errMsg}` ) await sleep(delayMs, opts.signal) From db9fb19416243818b8a56885f607e23239ef9353 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 04:55:32 +0000 Subject: [PATCH 07/32] docs: add sync lifecycle architecture design Introduces the protocol design for sync runs, covering: - start/end messages replacing EOF-based lifecycle - Engine-managed time ranges with binary search subdivision - Segment-level progress tracking with merged synced_ranges - Error levels as discriminated union (global/stream/segment/transient) - ProgressPayload as the unified shape for run and request stats - Source state simplified to pure cursors, engine owns range tracking - Frozen upper bounds via sync_run_id continuations Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 601 ++++++++++++++++++++++++++++++++++ 1 file changed, 601 insertions(+) create mode 100644 docs/engine/sync-lifecycle.md diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md new file mode 100644 index 00000000..ea37ae59 --- /dev/null +++ b/docs/engine/sync-lifecycle.md @@ -0,0 +1,601 @@ +# Sync Lifecycle + +How sync runs work: run identity, state ownership, time ranges, and stall +detection. For message types and connector interfaces, see [protocol.md](./protocol.md). + +## Motivation + +The base protocol treats each `read()` call as independent. The caller manages +pagination (via `state_limit`), upper bounds, and retry policy externally. This +creates several problems: + +1. **Backfill bounds shift between calls.** Each call computes `now()` as the + upper bound, so high-volume accounts chase a moving target and never converge. +2. **No run identity.** Multiple calls that form one logical backfill have no + shared context. The engine cannot distinguish "continuation" from "new sync." +3. **Source owns retry policy.** Error types (`auth_error`, `transient_error`) + are baked into source state, mixing cursor data with skip-on-resume decisions. +4. **Engine duplicates source bookkeeping.** Stream status and errors are tracked + independently by source and engine, with divergent representations. +5. **Stalled streams are invisible.** A non-incremental stream that restarts + every run blocks other streams with no detection or mitigation. + +This design introduces **sync runs** as a first-class concept and moves time +range management, stream lifecycle guarantees, and stall detection to the engine. + +--- + +## Layers + +``` +CLIENT ←—start/endβ€”β†’ ENGINE ←—iteratorβ€”β†’ SOURCE +``` + +| Concern | Client | Engine | Source | +| ---------------------------- | ---------------------- | ----------------------------------------- | ----------------------------------- | +| What to sync (streams) | Provides catalog | Adjusts catalog (time_range, deprioritize)| Syncs what it's given | +| When to sync (scheduling) | Decides | β€” | β€” | +| Run identity | Generates sync_run_id | Freezes bounds, tracks continuations | Unaware | +| Time range bounds | β€” | Computes, injects into catalog | Respects `time_range` if present | +| Internal segmentation | β€” | β€” | Manages segments, parallel pages | +| Stream lifecycle | Consumes progress | Guarantees terminal status | Emits `started`, optionally `complete` | +| Progress reporting | Consumes | Enriches source signals, emits progress | Emits raw stream_status + records | +| Error reporting | Decides retry policy | Passes through, tracks for stalls | Emits trace errors | +| State | Opaque round-trip | Manages engine section | Manages source section | +| Stall detection | β€” | Tracks per-stream across runs | β€” | +| `has_more` | Reads, acts | Derives from source + engine state | β€” | + +--- + +## Messages + +### `start` β€” client β†’ engine + +Begins or continues a sync run. See [Types](#types) for `StartPayload`. + +### `end` β€” engine β†’ client + +The run is done. See [Types](#types) for `EndPayload`. + +`has_more: true` β€” send another `start` with the same `sync_run_id` and the +returned `state`. `has_more: false` β€” this run is complete; use a new +`sync_run_id` for the next sync. + +### Source β†’ engine + +Sources are iterators that yield these message types: + +```ts +// Data record +{ type: 'record', record: { stream: string, data: Record, emitted_at: string } } + +// Checkpoint (per-stream β€” most common) +{ type: 'source_state', source_state: { state_type: 'stream', stream: string, data: unknown } } + +// Checkpoint (global β€” e.g. events cursor shared across all streams) +{ type: 'source_state', source_state: { state_type: 'global', data: unknown } } + +// Lifecycle signal +{ type: 'trace', trace: { trace_type: 'stream_status', stream_status: { stream: string, status: 'started' | 'complete' } } } + +// Error β€” discriminated union on level (see Error Handling section) +{ type: 'trace', trace: { trace_type: 'error', error: SyncError } } + +// Diagnostic log +{ type: 'log', log: { level: 'debug' | 'info' | 'warn' | 'error', message: string } } +``` + +### Engine β†’ client + +The engine emits three message types: `progress`, `record`, and `end`. + +```ts +// Progress β€” emitted on every source_state checkpoint and stream_status change. +// Each message is a complete snapshot of run-level progress. +// All counts are cumulative since the start of the run (across requests with +// the same sync_run_id). Client can diff consecutive messages for deltas. +{ + type: 'progress', + progress: { + elapsed_ms: number, // wall-clock since run started (across all requests) + global_state_count: number, // total checkpoints this run (all streams) + records_per_second: number, // run-level throughput + states_per_second: number, // run-level checkpoint rate + streams: Record + } +} + +// Records β€” passed through from source +{ type: 'record', record: { stream: string, data: Record, emitted_at: string } } + +// Terminal β€” this request is done. +// end.request has the same shape as progress but scoped to this request only. +{ + type: 'end', + end: { + has_more: boolean, + state: SyncState, + request_progress: ProgressPayload, // same shape, scoped to this request + } +} +``` + +`ProgressPayload` is used in two places with different scopes: + +| Scope | Where to find it | +| ------------------------- | --------------------------------------------- | +| Between two progress msgs | Client diffs consecutive `progress` messages | +| This request | `end.request_progress` (ProgressPayload) | +| This run (across requests)| Latest `progress` message (ProgressPayload) | +| All time (across runs) | Sum of `synced_ranges` coverage + segment counts | + +The engine does NOT emit trace messages to the client. Errors are included +per-segment inside `progress`. Source traces and logs are consumed by the engine +and distilled into `progress`. + +--- + +## Segment Status + +Four states per segment. Two on the wire from the source, two derived by engine. + +| Status | Set by | Meaning | +| ------------- | ---------------- | --------------------------------------------- | +| `pending` | Engine | Segment created but source hasn't started it | +| `started` | Source (emitted) | Source has begun reading this segment | +| `complete` | Source (emitted) | Source finished this segment successfully | +| `incomplete` | Engine (derived) | Source exhausted without emitting `complete` | + +Source emits `started` and `complete` as `stream_status` trace messages scoped +to a stream. The engine maps these to the corresponding segment based on which +`time_range` the source is currently working on. + +A stream is done when `segments` is empty and `synced_ranges` covers the full +range `[0, started_at)`. + +The engine guarantees: every stream that received `started` gets exactly one +terminal status (`complete` or `incomplete`). The source's `complete` is +optional β€” if omitted, the engine marks the stream `incomplete` when the source +iterator exhausts. + +Errors are orthogonal to lifecycle. A stream can be `complete` with errors +(some pages failed but the stream moved past them) or `incomplete` without +errors (time limit hit mid-stream). + +--- + +## Types + +### Configured catalog (client β†’ engine β†’ source) + +The client provides the catalog. The engine adjusts it (injects `time_range`) +before passing to the source. + +```ts +type ConfiguredStream = { + stream: { + name: string // e.g. "customers", "invoices" + primary_key: string[][] // e.g. [["id"]] + json_schema?: Record + metadata?: Record // e.g. { api_version, account_id, live_mode } + } + sync_mode: 'full_refresh' | 'incremental' + destination_sync_mode: 'append' | 'overwrite' | 'append_dedup' + cursor_field?: string[] + fields?: string[] // field projection + backfill_limit?: number // cap backfill to N records + system_columns?: Array<{ name: string; type: string; index: boolean }> + + // NEW β€” set by engine, not client + time_range?: { + gte: string // inclusive lower bound (ISO 8601) + lt: string // exclusive upper bound (ISO 8601) + } +} + +type ConfiguredCatalog = { + streams: ConfiguredStream[] +} +``` + +### Start message (client β†’ engine) + +```ts +type StartPayload = { + sync_run_id: string // client-generated UUID + source_config: Record // source-specific (e.g. Stripe API key, account) + destination_config: Record // destination-specific (e.g. Postgres connection) + configured_catalog: ConfiguredCatalog + state?: SyncState // from previous end; omit on first sync +} +``` + +### End message (engine β†’ client) + +```ts +type EndPayload = { + has_more: boolean + state: SyncState // round-trip into next start + request: ProgressPayload // stats for this request only (same shape as progress) +} +``` + +### Progress message (engine β†’ client) + +Emitted on every `source_state` checkpoint and `stream_status` change. Each +message is a complete snapshot of run-level progress β€” the client never needs +a reducer. + +```ts +// Errors are a discriminated union on error_level. Each level carries the +// context that makes sense for that blast radius. +type SyncError = + | { error_level: 'global'; message: string } + | { error_level: 'stream'; message: string; stream: string } + | { error_level: 'segment'; message: string; stream: string; segment: { gte: string; lt: string } } + | { error_level: 'transient'; message: string; stream?: string; segment?: { gte: string; lt: string } } + +type Segment = { + gte: string // ISO 8601 + lt: string // ISO 8601 + cursor?: string // source pagination cursor for resume + record_count: number // records synced in this segment this run + state_count: number // checkpoints in this segment this run + status: 'pending' | 'started' | 'complete' | 'incomplete' +} + +type StreamProgress = { + synced_ranges: Array<{ gte: string; lt: string }> // merged completed ranges + segments: Segment[] // active segments (in-flight or pending) +} + +type ProgressPayload = { + elapsed_ms: number // wall-clock since run started (across requests) + global_state_count: number // total checkpoints this run (all streams) + records_per_second: number // run-level throughput + states_per_second: number // run-level checkpoint rate + streams: Record // keyed by stream name + errors: SyncError[] // all errors accumulated this run +} +``` + +### SyncState (round-tripped between start and end) + +```ts +type SyncState = { + source: SourceState // opaque to engine β€” cursor data + engine: EngineState // opaque to client β€” run progress + run identity +} + +type SourceState = { + streams: Record // per-stream cursor data, keyed by stream name + global: Record // source-wide data (e.g. events cursor) +} + +// Engine state is run progress + run identity. Same ProgressPayload shape +// used in progress messages, extended with run tracking fields. +type EngineState = ProgressPayload & { + sync_run_id: string // current run ID + started_at: string // ISO 8601 β€” frozen snapshot upper bound +} +``` + +### Source state β€” Stripe example + +Pure cursor data. No error types, no status field. The engine treats this as +opaque; the types below are source-internal. + +```ts +// Per-stream state (source-internal, opaque to engine) +// Minimal β€” just a pagination cursor. Range management is engine's job. +type StripeStreamState = { + page_cursor: string | null // Stripe list pagination cursor +} +``` + +The source receives `time_range` from the catalog and paginates within it. +The engine tracks which ranges are complete and which need work via +`synced_ranges` and `pending_ranges` in engine state. + +**Example β€” two streams mid-sync:** +```jsonc +{ + "source": { + "streams": { + "customers": { "page_cursor": "cus_xyz" }, + "invoices": { "page_cursor": null } + }, + "global": { + "events_cursor": "2024-04-16T23:50:00Z" + } + } +} +``` + +### Engine state + +The engine state is `ProgressPayload` extended with run identity. The client +round-trips it opaquely. The engine uses it to accumulate progress across +requests within a run and to track synced ranges across runs. + +```ts +type EngineState = ProgressPayload & { + sync_run_id: string + started_at: string // ISO 8601 β€” frozen snapshot upper bound +} +``` + +**Example β€” customers fully synced through 2023, invoices mid-backfill, big_table stalled:** +```jsonc +{ + "engine": { + "sync_run_id": "sr_abc", + "started_at": "2024-04-17T00:00:00Z", + "elapsed_ms": 8400, + "global_state_count": 24, + "records_per_second": 5500, + "states_per_second": 2.9, + "streams": { + "customers": { + "synced_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], + "segments": [] + }, + "invoices": { + "synced_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], + "segments": [ + { + "gte": "2021-06-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z", + "cursor": "inv_xyz", "record_count": 1200, "state_count": 8, + "status": "started", "errors": [] + } + ] + }, + "big_table": { + "synced_ranges": [], + "segments": [ + { + "gte": "2011-01-01T00:00:00Z", "lt": "2017-09-01T00:00:00Z", + "record_count": 0, "state_count": 0, + "status": "incomplete", "errors": [{ "failure_type": "transient_error", "message": "Rate limit" }] + }, + { + "gte": "2017-09-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z", + "record_count": 0, "state_count": 0, + "status": "pending", "errors": [] + } + ] + } + } + } +} +``` + +--- + +## Sync Runs + +A sync run is identified by `sync_run_id`. Within a run, the upper time bound +(`started_at`) is frozen. + +### New run + +1. Client sends `start` with a new `sync_run_id`. +2. Engine freezes `started_at = now()` and stores it in engine state. +3. Engine computes `time_range` for each stream: + - Upper bound: `started_at` + - Lower bound: end of last `synced_ranges` entry (or account creation for + first backfill) +4. Engine injects `time_range` into configured catalog before passing to source. +5. Source syncs within the given range, yields messages, exhausts. +6. Engine emits progress, pipes records to destination, returns `end`. + +### Continuation + +1. Client sends `start` with the same `sync_run_id` and `state` from previous `end`. +2. Engine sees same ID β€” preserves `started_at` from engine state. +3. Engine sets the same `time_range` (same frozen upper bound). +4. Source resumes from its cursor state within the same range. + +### Completion + +When `has_more: false`: +- All streams completed their ranges or were marked `incomplete`. +- Engine promotes completed `pending_range` entries to `synced_ranges`. +- Client should use a new `sync_run_id` for the next sync. + +### Example + +``` +sync_run_id: "sr_1" + request 1: customers [2018, 2024) β†’ timed out β†’ end { has_more: true } + request 2: customers [2018, 2021) β†’ completed + customers [2021, 2024) β†’ timed out β†’ end { has_more: true } + request 3: customers [2021, 2022.5) β†’ completed + customers [2022.5, 2024) β†’ completed β†’ end { has_more: false } + synced_ranges merges to [2018, 2024) βœ“ +``` + +Each run's upper bound is frozen. Ranges that don't complete get subdivided. +Completed adjacent ranges merge. + +--- + +## Time Ranges + +Time is a first-class concept. The engine manages ranges via binary search +subdivision; the source just paginates whatever range it's given. + +### Flow + +``` +Client catalog: { stream: "customers", sync_mode: "incremental" } + (no time_range β€” client doesn't set this) + ↓ +Engine subdivides: The catalog passed to the source may have MULTIPLE entries + for the same stream, each with a different time_range: + + [ + { stream: "customers", time_range: { gte: "2018-01-01", lt: "2021-01-01" } }, + { stream: "customers", time_range: { gte: "2021-01-01", lt: "2024-04-17" } }, + ] + ↓ +Source receives: Each entry independently. Paginates within each range. + Emits stream_status and state per range segment. +``` + +### Binary search subdivision + +The engine starts with one range per stream covering `[0, started_at)`. +If a range doesn't complete within a request, the engine splits it in half +for the next request. + +``` +Request 1: pending_ranges: [{ gte: "2018", lt: "2024" }] + source times out on this range + +Request 2: pending_ranges: [{ gte: "2018", lt: "2021" }, { gte: "2021", lt: "2024" }] + left half completes, right half gets cursor + +Request 3: pending_ranges: [{ gte: "2021", lt: "2024", cursor: "cus_abc" }] + resumes from cursor, completes + +Final: synced_ranges: [{ gte: "2018", lt: "2024" }] (merged) + pending_ranges: [] +``` + +Up to N ranges can be in flight per stream (initially N=2). When one completes, +the engine can subdivide another stream's incomplete range. + +### Range merging + +Adjacent completed ranges merge to keep state compact: + +``` +synced_ranges: [{ gte: "2018", lt: "2021" }] ++ completed: { gte: "2021", lt: "2024" } += merged: [{ gte: "2018", lt: "2024" }] +``` + +### Engine range tracking + +| After request... | Engine action | +| ----------------------------------------- | ----------------------------------------------- | +| Range completed (no cursor) | Move to `synced_ranges`, merge adjacent | +| Range didn't finish (cursor remains) | Keep in `pending_ranges` with cursor | +| Range too large (timed out, no cursor) | Split in half β†’ two new `pending_ranges` | +| Range errored | Keep in `pending_ranges` | + +### Why this matters + +- **Frozen upper bounds.** `started_at` does not move within a run. +- **Adaptive parallelism.** Dense ranges get subdivided; sparse ranges complete in one shot. +- **Visibility.** Engine knows what fraction of history is synced. +- **Compact state.** Merged ranges keep state O(active segments), not O(total history). + +--- + +## `has_more` Derivation + +The engine derives `has_more` at end of run: + +``` +has_more = true if any catalog stream where: + - source state has a page_cursor (mid-pagination), OR + - engine has a pending_range the source didn't complete, OR + - engine synced_ranges don't cover [0, started_at) +``` + +--- + +## Error Handling + +### Error levels + +Errors carry their blast radius. The level determines the engine's action: + +| Level | Blast radius | Engine action | Example | +|---|---|---|---| +| `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | +| `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | +| `segment` | One time range | Mark segment incomplete, subdivide next request | Timeout after retries, too much data | +| `transient` | One request | Informational (request succeeded after retry) | Rate limited, retried 3x in 4.2s | + +### Source β†’ engine error flow + +```ts +// Source emits trace errors β€” discriminated union on error_level: +{ type: 'trace', trace: { trace_type: 'error', error: SyncError } } + +// Examples: +{ error: { error_level: 'global', message: 'Invalid API key' } } +{ error: { error_level: 'stream', message: 'Not available in test mode', stream: 'invoices' } } +{ error: { error_level: 'segment', message: 'Timeout after 5 retries', stream: 'customers', segment: { gte: '2021-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' } } } +{ error: { error_level: 'transient', message: 'Rate limited, retried 3x', stream: 'customers' } } +``` + +The source decides the level: +- **Transient**: HTTP retry succeeded β€” emit for observability, no action needed. +- **Segment**: All retries exhausted for a request within a range β€” emit with + `stream` and `segment`, move on to next segment/stream. +- **Stream**: Stream-level failure (e.g. resource not enabled) β€” emit with + `stream`, skip this stream entirely. +- **Global**: Unrecoverable (e.g. invalid credentials) β€” emit, stop. + +### Engine behavior + +The engine accumulates errors into `progress.errors[]` and acts on them: + +- **`global`**: Stop the source, emit `end { has_more: false }`. +- **`stream`**: Mark all segments for that stream as `incomplete`, continue + other streams. +- **`segment`**: Mark that segment `incomplete`. On the next request, the engine + subdivides it (binary search). +- **`transient`**: No action. Included in `progress.errors` for observability. + +Errors are NOT stored in source state. The source does not skip streams or +segments based on previous errors β€” that is the engine's job. + +--- + +## Wire Format + +NDJSON. One message per line. + +``` +β†’ {"type":"start","sync_run_id":"sr_abc","source_config":{...},"configured_catalog":{...}} +← {"type":"progress","progress":{"elapsed_ms":100,"global_state_count":0,"records_per_second":0,"states_per_second":0,"streams":{"customers":{"synced_ranges":[],"segments":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z","record_count":0,"state_count":0,"status":"started","errors":[]}]}}}} +← {"type":"record","record":{"stream":"customers","data":{...}}} +← {"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"synced_ranges":[],"segments":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z","cursor":"cus_abc","record_count":2500,"state_count":1,"status":"started","errors":[]}]}}}} +← {"type":"record","record":{"stream":"customers","data":{...}}} +← {"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"synced_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"segments":[]}}}} +← {"type":"end","has_more":false,"state":{"source":{...},"engine":{...}}} +``` + +Over HTTP: POST with NDJSON body (one `start` line), NDJSON response stream. +Clients that don't want streaming buffer until `end`. + +--- + +## Client Loop + +```ts +let state = undefined +const syncRunId = crypto.randomUUID() + +do { + const { end } = await engine.sync({ + sync_run_id: syncRunId, + source_config, + destination_config, + configured_catalog, + state, + }) + state = end.state +} while (end.has_more) + +// Backfill complete. Schedule next sync with a new sync_run_id. +``` + +The client does not need to understand stream statuses, error classification, +time ranges, or stall detection. It sends config + state, gets back +`has_more` + state. Everything else is in the progress stream for observability. From 39a3f6db986a0771256f8656d130985415d8a9b4 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 04:56:13 +0000 Subject: [PATCH 08/32] docs: fix stale error model references in sync-lifecycle Update remaining failure_type references to error_level discriminated union. Fix field names and remove errors from segment example (errors live on ProgressPayload, not inline on segments). Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index ea37ae59..a9ed1a10 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -13,8 +13,8 @@ creates several problems: upper bound, so high-volume accounts chase a moving target and never converge. 2. **No run identity.** Multiple calls that form one logical backfill have no shared context. The engine cannot distinguish "continuation" from "new sync." -3. **Source owns retry policy.** Error types (`auth_error`, `transient_error`) - are baked into source state, mixing cursor data with skip-on-resume decisions. +3. **Source owns retry policy.** Error types are baked into source state, mixing + cursor data with skip-on-resume decisions. 4. **Engine duplicates source bookkeeping.** Stream status and errors are tracked independently by source and engine, with divergent representations. 5. **Stalled streams are invisible.** A non-incremental stream that restarts @@ -78,7 +78,7 @@ Sources are iterators that yield these message types: // Lifecycle signal { type: 'trace', trace: { trace_type: 'stream_status', stream_status: { stream: string, status: 'started' | 'complete' } } } -// Error β€” discriminated union on level (see Error Handling section) +// Error β€” discriminated union on error_level (see Error Handling section) { type: 'trace', trace: { trace_type: 'error', error: SyncError } } // Diagnostic log @@ -356,7 +356,7 @@ type EngineState = ProgressPayload & { { "gte": "2011-01-01T00:00:00Z", "lt": "2017-09-01T00:00:00Z", "record_count": 0, "state_count": 0, - "status": "incomplete", "errors": [{ "failure_type": "transient_error", "message": "Rate limit" }] + "status": "incomplete" }, { "gte": "2017-09-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z", @@ -513,7 +513,7 @@ has_more = true if any catalog stream where: Errors carry their blast radius. The level determines the engine's action: -| Level | Blast radius | Engine action | Example | +| `error_level` | Blast radius | Engine action | Example | |---|---|---|---| | `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | | `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | @@ -533,7 +533,7 @@ Errors carry their blast radius. The level determines the engine's action: { error: { error_level: 'transient', message: 'Rate limited, retried 3x', stream: 'customers' } } ``` -The source decides the level: +The source decides the `error_level`: - **Transient**: HTTP retry succeeded β€” emit for observability, no action needed. - **Segment**: All retries exhausted for a request within a range β€” emit with `stream` and `segment`, move on to next segment/stream. From 063ed10726cb3165c19530934a95802ada07d961 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 05:15:43 +0000 Subject: [PATCH 09/32] =?UTF-8?q?docs:=20simplify=20sync-lifecycle=20?= =?UTF-8?q?=E2=80=94=20remove=20segments=20from=20protocol,=203=20error=20?= =?UTF-8?q?levels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Segments are source-internal; engine only tracks completed_ranges - completed_ranges derived from source_state messages with time_range - Error levels reduced to global/stream/transient (no segment level) - SyncError is discriminated union on error_level - completed_ranges optional on StreamProgress - Rates (records_per_second, states_per_second) nested under rates field - source_state messages carry time_range for engine range tracking Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 210 ++++++++++++---------------------- 1 file changed, 74 insertions(+), 136 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index a9ed1a10..bd7954a8 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -70,7 +70,9 @@ Sources are iterators that yield these message types: { type: 'record', record: { stream: string, data: Record, emitted_at: string } } // Checkpoint (per-stream β€” most common) -{ type: 'source_state', source_state: { state_type: 'stream', stream: string, data: unknown } } +// time_range tells the engine which range this checkpoint belongs to. +// When data has no cursor (pagination done), engine marks this range as synced. +{ type: 'source_state', source_state: { state_type: 'stream', stream: string, time_range?: { gte: string, lt: string }, data: unknown } } // Checkpoint (global β€” e.g. events cursor shared across all streams) { type: 'source_state', source_state: { state_type: 'global', data: unknown } } @@ -78,8 +80,8 @@ Sources are iterators that yield these message types: // Lifecycle signal { type: 'trace', trace: { trace_type: 'stream_status', stream_status: { stream: string, status: 'started' | 'complete' } } } -// Error β€” discriminated union on error_level (see Error Handling section) -{ type: 'trace', trace: { trace_type: 'error', error: SyncError } } +// Error β€” discriminated union on error_level (see Error Handling) +{ type: 'trace', trace: { trace_type: 'error', error: SyncError & { stack_trace?: string } } } // Diagnostic log { type: 'log', log: { level: 'debug' | 'info' | 'warn' | 'error', message: string } } @@ -99,9 +101,12 @@ The engine emits three message types: `progress`, `record`, and `end`. progress: { elapsed_ms: number, // wall-clock since run started (across all requests) global_state_count: number, // total checkpoints this run (all streams) - records_per_second: number, // run-level throughput - states_per_second: number, // run-level checkpoint rate - streams: Record + rates: { + records_per_second: number, + states_per_second: number, + }, + streams: Record, + errors: SyncError[] } } @@ -127,7 +132,7 @@ The engine emits three message types: `progress`, `record`, and `end`. | Between two progress msgs | Client diffs consecutive `progress` messages | | This request | `end.request_progress` (ProgressPayload) | | This run (across requests)| Latest `progress` message (ProgressPayload) | -| All time (across runs) | Sum of `synced_ranges` coverage + segment counts | +| All time (across runs) | Sum of `completed_ranges` coverage + segment counts | The engine does NOT emit trace messages to the client. Errors are included per-segment inside `progress`. Source traces and logs are consumed by the engine @@ -135,28 +140,17 @@ and distilled into `progress`. --- -## Segment Status - -Four states per segment. Two on the wire from the source, two derived by engine. - -| Status | Set by | Meaning | -| ------------- | ---------------- | --------------------------------------------- | -| `pending` | Engine | Segment created but source hasn't started it | -| `started` | Source (emitted) | Source has begun reading this segment | -| `complete` | Source (emitted) | Source finished this segment successfully | -| `incomplete` | Engine (derived) | Source exhausted without emitting `complete` | +## Stream Status -Source emits `started` and `complete` as `stream_status` trace messages scoped -to a stream. The engine maps these to the corresponding segment based on which -`time_range` the source is currently working on. +Source emits `stream_status` trace messages with `started` and `complete` per +stream. The engine uses these to know when a stream is active. -A stream is done when `segments` is empty and `synced_ranges` covers the full -range `[0, started_at)`. +A stream's backfill is done when `completed_ranges` covers the full range +`[0, started_at)`. -The engine guarantees: every stream that received `started` gets exactly one -terminal status (`complete` or `incomplete`). The source's `complete` is -optional β€” if omitted, the engine marks the stream `incomplete` when the source -iterator exhausts. +The source manages segments internally β€” the engine doesn't see or track them. +The engine learns about completed ranges from `source_state` messages that +include a `time_range` and have no remaining cursor. Errors are orthogonal to lifecycle. A stream can be `complete` with errors (some pages failed but the stream moved past them) or `incomplete` without @@ -227,33 +221,25 @@ message is a complete snapshot of run-level progress β€” the client never needs a reducer. ```ts -// Errors are a discriminated union on error_level. Each level carries the -// context that makes sense for that blast radius. +// Errors are a discriminated union on error_level. type SyncError = | { error_level: 'global'; message: string } | { error_level: 'stream'; message: string; stream: string } - | { error_level: 'segment'; message: string; stream: string; segment: { gte: string; lt: string } } - | { error_level: 'transient'; message: string; stream?: string; segment?: { gte: string; lt: string } } - -type Segment = { - gte: string // ISO 8601 - lt: string // ISO 8601 - cursor?: string // source pagination cursor for resume - record_count: number // records synced in this segment this run - state_count: number // checkpoints in this segment this run - status: 'pending' | 'started' | 'complete' | 'incomplete' -} + | { error_level: 'transient'; message: string; stream?: string } type StreamProgress = { - synced_ranges: Array<{ gte: string; lt: string }> // merged completed ranges - segments: Segment[] // active segments (in-flight or pending) + completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges + record_count: number // records this run (across requests) + state_count: number // checkpoints this run for this stream } type ProgressPayload = { elapsed_ms: number // wall-clock since run started (across requests) global_state_count: number // total checkpoints this run (all streams) - records_per_second: number // run-level throughput - states_per_second: number // run-level checkpoint rate + rates: { // derived, computed by engine + records_per_second: number + states_per_second: number + } streams: Record // keyed by stream name errors: SyncError[] // all errors accumulated this run } @@ -295,7 +281,7 @@ type StripeStreamState = { The source receives `time_range` from the catalog and paginates within it. The engine tracks which ranges are complete and which need work via -`synced_ranges` and `pending_ranges` in engine state. +`completed_ranges` and `pending_ranges` in engine state. **Example β€” two streams mid-sync:** ```jsonc @@ -337,33 +323,19 @@ type EngineState = ProgressPayload & { "states_per_second": 2.9, "streams": { "customers": { - "synced_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], - "segments": [] + "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], + "record_count": 45000, + "state_count": 16 }, "invoices": { - "synced_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], - "segments": [ - { - "gte": "2021-06-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z", - "cursor": "inv_xyz", "record_count": 1200, "state_count": 8, - "status": "started", "errors": [] - } - ] + "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], + "record_count": 1200, + "state_count": 8 }, "big_table": { - "synced_ranges": [], - "segments": [ - { - "gte": "2011-01-01T00:00:00Z", "lt": "2017-09-01T00:00:00Z", - "record_count": 0, "state_count": 0, - "status": "incomplete" - }, - { - "gte": "2017-09-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z", - "record_count": 0, "state_count": 0, - "status": "pending", "errors": [] - } - ] + "completed_ranges": [], + "record_count": 0, + "state_count": 0 } } } @@ -383,7 +355,7 @@ A sync run is identified by `sync_run_id`. Within a run, the upper time bound 2. Engine freezes `started_at = now()` and stores it in engine state. 3. Engine computes `time_range` for each stream: - Upper bound: `started_at` - - Lower bound: end of last `synced_ranges` entry (or account creation for + - Lower bound: end of last `completed_ranges` entry (or account creation for first backfill) 4. Engine injects `time_range` into configured catalog before passing to source. 5. Source syncs within the given range, yields messages, exhausts. @@ -400,7 +372,7 @@ A sync run is identified by `sync_run_id`. Within a run, the upper time bound When `has_more: false`: - All streams completed their ranges or were marked `incomplete`. -- Engine promotes completed `pending_range` entries to `synced_ranges`. +- Engine promotes completed `pending_range` entries to `completed_ranges`. - Client should use a new `sync_run_id` for the next sync. ### Example @@ -412,7 +384,7 @@ sync_run_id: "sr_1" customers [2021, 2024) β†’ timed out β†’ end { has_more: true } request 3: customers [2021, 2022.5) β†’ completed customers [2022.5, 2024) β†’ completed β†’ end { has_more: false } - synced_ranges merges to [2018, 2024) βœ“ + completed_ranges merges to [2018, 2024) βœ“ ``` Each run's upper bound is frozen. Ranges that don't complete get subdivided. @@ -422,8 +394,8 @@ Completed adjacent ranges merge. ## Time Ranges -Time is a first-class concept. The engine manages ranges via binary search -subdivision; the source just paginates whatever range it's given. +Time is a first-class concept. The engine sets the outer bounds; the source +manages pagination and segmentation within them. ### Flow @@ -431,59 +403,34 @@ subdivision; the source just paginates whatever range it's given. Client catalog: { stream: "customers", sync_mode: "incremental" } (no time_range β€” client doesn't set this) ↓ -Engine subdivides: The catalog passed to the source may have MULTIPLE entries - for the same stream, each with a different time_range: - - [ - { stream: "customers", time_range: { gte: "2018-01-01", lt: "2021-01-01" } }, - { stream: "customers", time_range: { gte: "2021-01-01", lt: "2024-04-17" } }, - ] +Engine sets range: { stream: "customers", sync_mode: "incremental", + time_range: { gte: "2021-01-01T00:00:00Z", lt: "2024-04-17T00:00:00Z" } } + (computed from completed_ranges + started_at) ↓ -Source receives: Each entry independently. Paginates within each range. - Emits stream_status and state per range segment. +Source receives: time_range on configured stream. + Manages its own segments/subdivision/parallelism within it. + Emits source_state with time_range to report range completion. ``` -### Binary search subdivision +### How the engine tracks completed ranges -The engine starts with one range per stream covering `[0, started_at)`. -If a range doesn't complete within a request, the engine splits it in half -for the next request. +The engine observes `source_state` messages. When a state message includes +`time_range` and has no remaining cursor, the engine knows that range is done +and adds it to `completed_ranges`, merging adjacent ranges: ``` -Request 1: pending_ranges: [{ gte: "2018", lt: "2024" }] - source times out on this range - -Request 2: pending_ranges: [{ gte: "2018", lt: "2021" }, { gte: "2021", lt: "2024" }] - left half completes, right half gets cursor - -Request 3: pending_ranges: [{ gte: "2021", lt: "2024", cursor: "cus_abc" }] - resumes from cursor, completes - -Final: synced_ranges: [{ gte: "2018", lt: "2024" }] (merged) - pending_ranges: [] +completed_ranges: [{ gte: "2018", lt: "2021" }] ++ source_state with time_range [2021, 2024), cursor: null += completed_ranges: [{ gte: "2018", lt: "2024" }] (merged) ``` -Up to N ranges can be in flight per stream (initially N=2). When one completes, -the engine can subdivide another stream's incomplete range. - -### Range merging - -Adjacent completed ranges merge to keep state compact: - -``` -synced_ranges: [{ gte: "2018", lt: "2021" }] -+ completed: { gte: "2021", lt: "2024" } -= merged: [{ gte: "2018", lt: "2024" }] -``` +### Engine range computation -### Engine range tracking +On each request, the engine computes the `time_range` to assign: -| After request... | Engine action | -| ----------------------------------------- | ----------------------------------------------- | -| Range completed (no cursor) | Move to `synced_ranges`, merge adjacent | -| Range didn't finish (cursor remains) | Keep in `pending_ranges` with cursor | -| Range too large (timed out, no cursor) | Split in half β†’ two new `pending_ranges` | -| Range errored | Keep in `pending_ranges` | +1. Upper bound: `started_at` (frozen for the run) +2. Lower bound: end of last `completed_ranges` entry (or account start for first backfill) +3. If `completed_ranges` has gaps, fill the first gap ### Why this matters @@ -501,8 +448,7 @@ The engine derives `has_more` at end of run: ``` has_more = true if any catalog stream where: - source state has a page_cursor (mid-pagination), OR - - engine has a pending_range the source didn't complete, OR - - engine synced_ranges don't cover [0, started_at) + - completed_ranges don't cover [0, started_at) ``` --- @@ -511,14 +457,13 @@ has_more = true if any catalog stream where: ### Error levels -Errors carry their blast radius. The level determines the engine's action: +Errors carry their blast radius. The `error_level` determines the engine's action: | `error_level` | Blast radius | Engine action | Example | |---|---|---|---| | `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | | `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | -| `segment` | One time range | Mark segment incomplete, subdivide next request | Timeout after retries, too much data | -| `transient` | One request | Informational (request succeeded after retry) | Rate limited, retried 3x in 4.2s | +| `transient` | One request | Informational | Rate limited, retried 3x in 4.2s | ### Source β†’ engine error flow @@ -529,31 +474,24 @@ Errors carry their blast radius. The level determines the engine's action: // Examples: { error: { error_level: 'global', message: 'Invalid API key' } } { error: { error_level: 'stream', message: 'Not available in test mode', stream: 'invoices' } } -{ error: { error_level: 'segment', message: 'Timeout after 5 retries', stream: 'customers', segment: { gte: '2021-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' } } } { error: { error_level: 'transient', message: 'Rate limited, retried 3x', stream: 'customers' } } ``` The source decides the `error_level`: -- **Transient**: HTTP retry succeeded β€” emit for observability, no action needed. -- **Segment**: All retries exhausted for a request within a range β€” emit with - `stream` and `segment`, move on to next segment/stream. -- **Stream**: Stream-level failure (e.g. resource not enabled) β€” emit with - `stream`, skip this stream entirely. -- **Global**: Unrecoverable (e.g. invalid credentials) β€” emit, stop. +- **Transient**: Request failed and retried β€” emit for observability. +- **Stream**: Stream-level failure (e.g. resource not enabled) β€” skip stream. +- **Global**: Unrecoverable (e.g. invalid credentials) β€” stop everything. ### Engine behavior The engine accumulates errors into `progress.errors[]` and acts on them: - **`global`**: Stop the source, emit `end { has_more: false }`. -- **`stream`**: Mark all segments for that stream as `incomplete`, continue - other streams. -- **`segment`**: Mark that segment `incomplete`. On the next request, the engine - subdivides it (binary search). +- **`stream`**: Skip that stream, continue others. - **`transient`**: No action. Included in `progress.errors` for observability. -Errors are NOT stored in source state. The source does not skip streams or -segments based on previous errors β€” that is the engine's job. +Errors are NOT stored in source state. Segment-level concerns (subdivision, +retries, timeouts) are managed internally by the source. --- @@ -563,11 +501,11 @@ NDJSON. One message per line. ``` β†’ {"type":"start","sync_run_id":"sr_abc","source_config":{...},"configured_catalog":{...}} -← {"type":"progress","progress":{"elapsed_ms":100,"global_state_count":0,"records_per_second":0,"states_per_second":0,"streams":{"customers":{"synced_ranges":[],"segments":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z","record_count":0,"state_count":0,"status":"started","errors":[]}]}}}} +← {"type":"progress","progress":{"elapsed_ms":100,"global_state_count":0,"records_per_second":0,"states_per_second":0,"streams":{"customers":{"completed_ranges":[],"record_count":0,"state_count":0}},"errors":[]}} ← {"type":"record","record":{"stream":"customers","data":{...}}} -← {"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"synced_ranges":[],"segments":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z","cursor":"cus_abc","record_count":2500,"state_count":1,"status":"started","errors":[]}]}}}} +← {"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[],"record_count":2500,"state_count":1}},"errors":[]}} ← {"type":"record","record":{"stream":"customers","data":{...}}} -← {"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"synced_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"segments":[]}}}} +← {"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"record_count":5000,"state_count":2}},"errors":[]}} ← {"type":"end","has_more":false,"state":{"source":{...},"engine":{...}}} ``` From 719b0fcf6083d3e500580b37235b079ff55c0bca Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 05:39:04 +0000 Subject: [PATCH 10/32] docs: add Stripe source sync lifecycle Describes how the Stripe source manages pagination within engine-assigned time ranges: initialization, density probing, sub-range splitting, cursor tracking, resumption, subdivision, and completion signaling. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/sync-lifecycle-source-stripe.md | 211 +++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/sync-lifecycle-source-stripe.md diff --git a/docs/sync-lifecycle-source-stripe.md b/docs/sync-lifecycle-source-stripe.md new file mode 100644 index 00000000..d5160722 --- /dev/null +++ b/docs/sync-lifecycle-source-stripe.md @@ -0,0 +1,211 @@ +# Sync Lifecycle β€” Stripe Source + +How the Stripe source manages pagination within a `time_range` assigned by the +engine. For the overall sync lifecycle and protocol, see +[engine/sync-lifecycle.md](./engine/sync-lifecycle.md). + +## Overview + +The engine assigns a `time_range` per stream via the configured catalog. The +Stripe source's job is to paginate all records within that range and emit them. +It manages its own sub-range splitting and parallelism internally. + +## Source State + +```ts +type StripeStreamState = { + ranges: Array<{ + gte: string // ISO 8601 β€” inclusive lower bound + lt: string // ISO 8601 β€” exclusive upper bound + cursor: string | null // Stripe pagination cursor; null = not yet started + }> +} +``` + +- A range with `cursor: null` β†’ source has planned this range but not yet fetched the first page. +- A range with `cursor: "cus_abc"` β†’ resume pagination after this object. +- A range removed from the list β†’ that sub-range is complete. +- Empty `ranges: []` β†’ source is done with the assigned `time_range`. + +## Workflow + +### 1. Initialization (no existing state) + +The source receives `time_range` from the catalog and has no state for this +stream. + +1. Probe density: make one request to the Stripe list API with the full range + to estimate how many records exist. +2. Based on density, split the `time_range` into N sub-ranges (using the + `created` timestamp filter). Denser ranges get more sub-ranges. +3. Initialize state with all sub-ranges, `cursor: null`. + +``` +Engine assigns: time_range { gte: "2018-01-01", lt: "2024-04-17" } + +Source probes density β†’ estimates 200K records β†’ splits into 8 sub-ranges: + +state: { + ranges: [ + { gte: "2018-01-01", lt: "2018-10-01", cursor: null }, + { gte: "2018-10-01", lt: "2019-07-01", cursor: null }, + { gte: "2019-07-01", lt: "2020-04-01", cursor: null }, + { gte: "2020-04-01", lt: "2021-01-01", cursor: null }, + { gte: "2021-01-01", lt: "2021-10-01", cursor: null }, + { gte: "2021-10-01", lt: "2022-07-01", cursor: null }, + { gte: "2022-07-01", lt: "2023-04-01", cursor: null }, + { gte: "2023-04-01", lt: "2024-04-17", cursor: null } + ] +} +``` + +### 2. Pagination + +The source paginates sub-ranges, potentially in parallel. For each sub-range: + +1. Call the Stripe list API with `created[gte]` and `created[lt]` filters, + plus `starting_after` if resuming from a cursor. +2. Emit records. +3. Emit `source_state` with updated cursor after each page. +4. When a sub-range is exhausted (`has_more: false`), remove it from state. + +``` +After paginating first sub-range (2 pages): + +state: { + ranges: [ + { gte: "2018-01-01", lt: "2018-10-01", cursor: "cus_abc" }, // mid-page + { gte: "2018-10-01", lt: "2019-07-01", cursor: null }, + ... + ] +} +β†’ emit source_state with time_range: { gte: "2018-01-01", lt: "2024-04-17" } + +First sub-range exhausted: + +state: { + ranges: [ + { gte: "2018-10-01", lt: "2019-07-01", cursor: "cus_def" }, // now active + { gte: "2019-07-01", lt: "2020-04-01", cursor: null }, + ... + ] +} +β†’ emit source_state +``` + +### 3. Resumption (existing state) + +If the source has existing state for this stream (from a previous request in +the same sync run), it skips initialization and resumes directly from the +remaining ranges: + +``` +Source receives time_range { gte: "2018-01-01", lt: "2024-04-17" } +Existing state: { + ranges: [ + { gte: "2021-10-01", lt: "2022-07-01", cursor: "cus_xyz" }, + { gte: "2022-07-01", lt: "2023-04-01", cursor: null }, + { gte: "2023-04-01", lt: "2024-04-17", cursor: null } + ] +} + +β†’ Resume paginating from cus_xyz in [2021-10, 2022-07) +β†’ No density probing, no re-splitting +``` + +### 4. Subdivision + +If a sub-range is too dense (too many records, taking too long), the source +can subdivide it further: + +``` +Before β€” one large range taking too long: + { gte: "2021-01-01", lt: "2022-07-01", cursor: null } + +After subdivision β€” split in half: + { gte: "2021-01-01", lt: "2021-10-01", cursor: null } + { gte: "2021-10-01", lt: "2022-07-01", cursor: null } +``` + +This is the binary search behavior: ranges that don't complete in time get +split. The source decides when and how to subdivide based on observed +pagination speed. + +A sub-range that already has a cursor (mid-pagination) is NOT subdivided β€” +it's making progress. Only ranges with `cursor: null` that haven't started +or ranges that are progressing too slowly are candidates. + +### 5. Completion + +When all sub-ranges are exhausted: + +``` +state: { ranges: [] } +β†’ emit source_state with time_range +``` + +The engine observes empty ranges and marks this `time_range` as complete +in `completed_ranges`. + +## State on the Wire + +The source emits `source_state` messages with the `time_range` from the +catalog so the engine can track which range this state belongs to: + +```ts +{ + type: 'source_state', + source_state: { + state_type: 'stream', + stream: 'customers', + time_range: { gte: '2018-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' }, + data: { + ranges: [ + { gte: '2021-10-01T00:00:00Z', lt: '2022-07-01T00:00:00Z', cursor: 'cus_xyz' }, + { gte: '2022-07-01T00:00:00Z', lt: '2023-04-01T00:00:00Z', cursor: null }, + { gte: '2023-04-01T00:00:00Z', lt: '2024-04-17T00:00:00Z', cursor: null } + ] + } + } +} +``` + +## Parallel Pagination + +The source can paginate multiple sub-ranges concurrently. The number of +concurrent sub-ranges is determined by the density probe and available +concurrency budget. Records from different sub-ranges are interleaved +on the output stream, each tagged with the stream name. + +State checkpoints are emitted after each page completes, reflecting the +current state of all sub-ranges. This ensures resumability even if the +source is cut off mid-run. + +## Error Handling + +- **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP + layer with exponential backoff. If retries succeed, emit a `transient` + error trace for observability. +- **Stream errors** (resource not available, permission denied): Emit a + `stream` error trace, stop this stream, move to the next. +- **Global errors** (invalid API key): Emit a `global` error trace, stop. + +The source does not store error state. If a sub-range fails after all +retries, the source emits an error trace and moves on. The sub-range +remains in state (with its cursor) for the next attempt. + +## Events / Incremental Sync + +After backfill completes, the source switches to incremental mode using +Stripe's `/events` API or WebSocket. This is outside the `time_range` +model β€” events are a live stream, not a bounded range. + +The global state stores an `events_cursor` for resumption: + +```ts +// source.global +{ events_cursor: "2024-04-16T23:50:00Z" } +``` + +Events and backfill can run concurrently β€” backfill covers historical data +within `time_range`, events cover real-time changes. From 57821c126f364b11bb7b1c9a76ad4b7871e34a30 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 05:43:09 +0000 Subject: [PATCH 11/32] docs: rewrite Stripe source sync with n-ary search algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove density probing β€” source starts with full range and subdivides - Rename ranges β†’ remaining (just the work left to do) - Describe subdivision: split unpaginated portion of a range into N parts - Full walkthrough example showing subdivision across requests - Cursor tracking: null = not started, string = resume point, removed = done Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 248 ++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 docs/engine/sync-lifecycle-source-stripe.md diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md new file mode 100644 index 00000000..bdf6de99 --- /dev/null +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -0,0 +1,248 @@ +# Sync Lifecycle β€” Stripe Source + +How the Stripe source manages pagination within a `time_range` assigned by the +engine. For the overall sync lifecycle and protocol, see +[sync-lifecycle.md](./sync-lifecycle.md). + +## Overview + +The engine assigns a `time_range` per stream via the configured catalog. The +Stripe source paginates all records within that range using an n-ary search +algorithm: start with the full range, paginate, and subdivide if the range +is too large to complete. No upfront density probing β€” the source discovers +the right granularity by doing the work. + +## Source State + +```ts +type StripeStreamState = { + remaining: Array<{ + gte: string // ISO 8601 β€” inclusive lower bound + lt: string // ISO 8601 β€” exclusive upper bound + cursor: string | null // Stripe pagination cursor; null = not yet started + }> +} +``` + +- `cursor: null` β†’ range planned but first page not yet fetched. +- `cursor: "cus_abc"` β†’ resume pagination after this object. +- Range removed from list β†’ complete. +- `remaining: []` β†’ source is done with the assigned `time_range`. + +## Algorithm + +### 1. Initialization (no existing state) + +The source receives `time_range` from the catalog and has no state. It starts +with the full range as a single entry: + +``` +Engine assigns: time_range { gte: "2018-01-01", lt: "2024-04-17" } + +state: { + remaining: [ + { gte: "2018-01-01", lt: "2024-04-17", cursor: null } + ] +} +``` + +### 2. Pagination + +The source picks a range from `remaining` and paginates it: + +1. Call the Stripe list API with `created[gte]` and `created[lt]` filters, + plus `starting_after` if cursor is set. +2. Emit records. +3. Update cursor in state, emit `source_state`. +4. When a range is exhausted (`has_more: false`), remove it from `remaining`. + +``` +First page fetched, got cursor: + +state: { + remaining: [ + { gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_abc" } + ] +} +β†’ emit source_state + +Pagination exhausted, range complete: + +state: { + remaining: [] +} +β†’ emit source_state (done) +``` + +### 3. Subdivision (n-ary search) + +If a range is too large to complete within the request budget (too many +records, approaching time limit), the source subdivides it. The remaining +unpaginated portion of the range is split into N smaller ranges: + +``` +Before β€” range is too large, source has paginated up to created=2020-06-15 +via cursor but there's much more: + +state: { + remaining: [ + { gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_xyz" } + ] +} + +Source decides to subdivide. It knows it has paginated records up to +created=2020-06-15 (from the last record seen). It splits the remaining +unpaginated portion [2020-06-15, 2024-04-17) into N=2 parts: + +state: { + remaining: [ + { gte: "2018-01-01", lt: "2020-06-15", cursor: "cus_xyz" }, + { gte: "2020-06-15", lt: "2022-05-16", cursor: null }, + { gte: "2022-05-16", lt: "2024-04-17", cursor: null } + ] +} +β†’ emit source_state +``` + +The original range narrows to `[2018-01-01, 2020-06-15)` (the portion being +paginated with a cursor), and the rest becomes new ranges with `cursor: null`. + +This is the n-ary search: large ranges get split, small ranges complete +directly. The source adapts to data density without upfront probing. + +**When to subdivide:** +- The source can track pagination speed (records/second, pages/second). +- If a range is progressing too slowly relative to the time budget, subdivide. +- Only the unpaginated portion is split β€” the cursor range continues as-is. + +**Recursive subdivision:** If a subdivided range is still too large, it gets +split again on the next pass. Each iteration narrows the ranges until they're +small enough to complete. + +### 4. Resumption (existing state) + +If the source has existing state (from a previous request in the same sync +run), it resumes directly from `remaining`: + +``` +Source receives time_range { gte: "2018-01-01", lt: "2024-04-17" } +Existing state: { + remaining: [ + { gte: "2022-05-16", lt: "2024-04-17", cursor: "cus_xyz" } + ] +} + +β†’ Resume paginating from cus_xyz in [2022-05-16, 2024-04-17) +β†’ No re-initialization +``` + +### 5. Completion + +When `remaining` is empty: + +``` +state: { remaining: [] } +β†’ emit source_state with time_range +``` + +The engine observes empty `remaining` and marks this `time_range` as complete +in `completed_ranges`. + +## Full Example + +``` +Request 1: + Engine assigns time_range [2018, 2024) to customers + + remaining: [{ gte: "2018", lt: "2024", cursor: null }] + β†’ paginate... got 10K records, up to created=2019-03 + remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] + β†’ still going... approaching time limit, subdivide + remaining: [ + { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // finishing current + { gte: "2019-03", lt: "2021-09", cursor: null }, // new + { gte: "2021-09", lt: "2024", cursor: null } // new + ] + β†’ first range completes + remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: null }, + { gte: "2021-09", lt: "2024", cursor: null } + ] + β†’ time limit hit, emit source_state, stop + + end { has_more: true } + +Request 2 (continuation, same sync_run_id): + Source resumes with remaining from state + + remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: null }, + { gte: "2021-09", lt: "2024", cursor: null } + ] + β†’ paginate first range... completes + remaining: [ + { gte: "2021-09", lt: "2024", cursor: null } + ] + β†’ paginate second range... completes + remaining: [] + + end { has_more: false } + + Engine marks [2018, 2024) as completed. +``` + +## State on the Wire + +The source emits `source_state` messages with `time_range` so the engine +can track which range this state belongs to: + +```ts +{ + type: 'source_state', + source_state: { + state_type: 'stream', + stream: 'customers', + time_range: { gte: '2018-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' }, + data: { + remaining: [ + { gte: '2022-05-16T00:00:00Z', lt: '2024-04-17T00:00:00Z', cursor: 'cus_xyz' } + ] + } + } +} +``` + +## Parallel Pagination + +The source can paginate multiple ranges from `remaining` concurrently. +Records from different ranges are interleaved on the output stream. State +checkpoints are emitted after each page, reflecting the current state of +all ranges. This ensures resumability if the source is cut off mid-run. + +## Error Handling + +- **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP + layer with exponential backoff. Emit a `transient` error trace for + observability regardless of whether the retry succeeded. +- **Stream errors** (resource not available, permission denied): Emit a + `stream` error trace, stop this stream, move to the next. +- **Global errors** (invalid API key): Emit a `global` error trace, stop. + +The source does not store error state. If a range fails after all retries, +the range stays in `remaining` with its cursor for the next attempt. + +## Events / Incremental Sync + +After backfill completes (`remaining: []`), the source switches to +incremental mode using Stripe's `/events` API or WebSocket. This is outside +the `time_range` model β€” events are a live stream, not a bounded range. + +The global state stores an `events_cursor` for resumption: + +```ts +// source.global +{ events_cursor: "2024-04-16T23:50:00Z" } +``` + +Events and backfill can run concurrently β€” backfill covers historical data +within `time_range`, events cover real-time changes. From d495000a358e2e08282ffb8d723bdac5e57e535b Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 05:43:21 +0000 Subject: [PATCH 12/32] docs: update sync-lifecycle with derived field, nested run_progress, flat rates - EngineState contains run_progress: ProgressPayload (not flattened) - completed_ranges optional on StreamProgress - Rates flat on ProgressPayload (records_per_second, states_per_second) - Remove old sync-lifecycle-source-stripe.md from docs/ root (moved to docs/engine/) Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 97 +++++++----- docs/sync-lifecycle-source-stripe.md | 211 --------------------------- 2 files changed, 58 insertions(+), 250 deletions(-) delete mode 100644 docs/sync-lifecycle-source-stripe.md diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index bd7954a8..b9a3de46 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -37,7 +37,7 @@ CLIENT ←—start/endβ€”β†’ ENGINE ←—iteratorβ€”β†’ SOURCE | When to sync (scheduling) | Decides | β€” | β€” | | Run identity | Generates sync_run_id | Freezes bounds, tracks continuations | Unaware | | Time range bounds | β€” | Computes, injects into catalog | Respects `time_range` if present | -| Internal segmentation | β€” | β€” | Manages segments, parallel pages | +| Internal pagination | β€” | β€” | Manages ranges, parallel pages | | Stream lifecycle | Consumes progress | Guarantees terminal status | Emits `started`, optionally `complete` | | Progress reporting | Consumes | Enriches source signals, emits progress | Emits raw stream_status + records | | Error reporting | Decides retry policy | Passes through, tracks for stalls | Emits trace errors | @@ -93,7 +93,17 @@ The engine emits three message types: `progress`, `record`, and `end`. ```ts // Progress β€” emitted on every source_state checkpoint and stream_status change. -// Each message is a complete snapshot of run-level progress. +// +// Each message is a complete run-level snapshot, not a delta. Run-level +// totals ("45K customers synced across 3 requests") are what clients +// typically display. Point-in-time rates are derivable by diffing two +// consecutive snapshots β€” the common case is served directly, the rare +// case is still easy. +// +// Errors are included for the same reason: separating them into their own +// event stream would force every client to accumulate errors alongside +// progress snapshots, defeating the single-message-renders-everything model. +// // All counts are cumulative since the start of the run (across requests with // the same sync_run_id). Client can diff consecutive messages for deltas. { @@ -101,7 +111,7 @@ The engine emits three message types: `progress`, `record`, and `end`. progress: { elapsed_ms: number, // wall-clock since run started (across all requests) global_state_count: number, // total checkpoints this run (all streams) - rates: { + derived: { records_per_second: number, states_per_second: number, }, @@ -132,10 +142,10 @@ The engine emits three message types: `progress`, `record`, and `end`. | Between two progress msgs | Client diffs consecutive `progress` messages | | This request | `end.request_progress` (ProgressPayload) | | This run (across requests)| Latest `progress` message (ProgressPayload) | -| All time (across runs) | Sum of `completed_ranges` coverage + segment counts | +| All time (across runs) | Sum of `completed_ranges` coverage + record counts | The engine does NOT emit trace messages to the client. Errors are included -per-segment inside `progress`. Source traces and logs are consumed by the engine +inside `progress`. Source traces and logs are consumed by the engine and distilled into `progress`. --- @@ -148,7 +158,7 @@ stream. The engine uses these to know when a stream is active. A stream's backfill is done when `completed_ranges` covers the full range `[0, started_at)`. -The source manages segments internally β€” the engine doesn't see or track them. +The source manages sub-ranges internally β€” the engine doesn't see or track them. The engine learns about completed ranges from `source_state` messages that include a `time_range` and have no remaining cursor. @@ -236,8 +246,14 @@ type StreamProgress = { type ProgressPayload = { elapsed_ms: number // wall-clock since run started (across requests) global_state_count: number // total checkpoints this run (all streams) - rates: { // derived, computed by engine + derived: { + // Computed from the sum of all stream record_counts / (elapsed_ms / 1000). + // Uses run-level totals, not windowed β€” so this is the average rate since + // the run started. A client that wants instantaneous rate can diff + // record_count between two consecutive progress messages and divide by + // the elapsed_ms delta. records_per_second: number + // Computed from global_state_count / (elapsed_ms / 1000). states_per_second: number } streams: Record // keyed by stream name @@ -258,11 +274,10 @@ type SourceState = { global: Record // source-wide data (e.g. events cursor) } -// Engine state is run progress + run identity. Same ProgressPayload shape -// used in progress messages, extended with run tracking fields. -type EngineState = ProgressPayload & { +type EngineState = { sync_run_id: string // current run ID started_at: string // ISO 8601 β€” frozen snapshot upper bound + run_progress: ProgressPayload // accumulated run-level progress } ``` @@ -300,43 +315,47 @@ The engine tracks which ranges are complete and which need work via ### Engine state -The engine state is `ProgressPayload` extended with run identity. The client -round-trips it opaquely. The engine uses it to accumulate progress across -requests within a run and to track synced ranges across runs. +The engine state contains run identity and a `run_progress` field that is a +`ProgressPayload`. The client round-trips it opaquely. The engine uses it to +accumulate progress across requests within a run and to track completed ranges +across runs. ```ts -type EngineState = ProgressPayload & { - sync_run_id: string +type EngineState = { + sync_run_id: string // current run ID started_at: string // ISO 8601 β€” frozen snapshot upper bound + run_progress: ProgressPayload // accumulated run-level progress } ``` -**Example β€” customers fully synced through 2023, invoices mid-backfill, big_table stalled:** +**Example β€” customers fully synced, invoices mid-backfill, big_table stalled:** ```jsonc { "engine": { "sync_run_id": "sr_abc", "started_at": "2024-04-17T00:00:00Z", - "elapsed_ms": 8400, - "global_state_count": 24, - "records_per_second": 5500, - "states_per_second": 2.9, - "streams": { - "customers": { - "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], - "record_count": 45000, - "state_count": 16 - }, - "invoices": { - "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], - "record_count": 1200, - "state_count": 8 + "run_progress": { + "elapsed_ms": 8400, + "global_state_count": 24, + "derived": { "records_per_second": 5500, "states_per_second": 2.9 }, + "streams": { + "customers": { + "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], + "record_count": 45000, + "state_count": 16 + }, + "invoices": { + "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], + "record_count": 1200, + "state_count": 8 + }, + "big_table": { + "completed_ranges": [], + "record_count": 0, + "state_count": 0 + } }, - "big_table": { - "completed_ranges": [], - "record_count": 0, - "state_count": 0 - } + "errors": [] } } } @@ -395,7 +414,7 @@ Completed adjacent ranges merge. ## Time Ranges Time is a first-class concept. The engine sets the outer bounds; the source -manages pagination and segmentation within them. +manages pagination and subdivision within them. ### Flow @@ -408,7 +427,7 @@ Engine sets range: { stream: "customers", sync_mode: "incremental", (computed from completed_ranges + started_at) ↓ Source receives: time_range on configured stream. - Manages its own segments/subdivision/parallelism within it. + Manages its own subdivision/parallelism within it. Emits source_state with time_range to report range completion. ``` @@ -437,7 +456,7 @@ On each request, the engine computes the `time_range` to assign: - **Frozen upper bounds.** `started_at` does not move within a run. - **Adaptive parallelism.** Dense ranges get subdivided; sparse ranges complete in one shot. - **Visibility.** Engine knows what fraction of history is synced. -- **Compact state.** Merged ranges keep state O(active segments), not O(total history). +- **Compact state.** Merged ranges keep state O(active ranges), not O(total history). --- @@ -490,7 +509,7 @@ The engine accumulates errors into `progress.errors[]` and acts on them: - **`stream`**: Skip that stream, continue others. - **`transient`**: No action. Included in `progress.errors` for observability. -Errors are NOT stored in source state. Segment-level concerns (subdivision, +Errors are NOT stored in source state. Range-level concerns (subdivision, retries, timeouts) are managed internally by the source. --- diff --git a/docs/sync-lifecycle-source-stripe.md b/docs/sync-lifecycle-source-stripe.md deleted file mode 100644 index d5160722..00000000 --- a/docs/sync-lifecycle-source-stripe.md +++ /dev/null @@ -1,211 +0,0 @@ -# Sync Lifecycle β€” Stripe Source - -How the Stripe source manages pagination within a `time_range` assigned by the -engine. For the overall sync lifecycle and protocol, see -[engine/sync-lifecycle.md](./engine/sync-lifecycle.md). - -## Overview - -The engine assigns a `time_range` per stream via the configured catalog. The -Stripe source's job is to paginate all records within that range and emit them. -It manages its own sub-range splitting and parallelism internally. - -## Source State - -```ts -type StripeStreamState = { - ranges: Array<{ - gte: string // ISO 8601 β€” inclusive lower bound - lt: string // ISO 8601 β€” exclusive upper bound - cursor: string | null // Stripe pagination cursor; null = not yet started - }> -} -``` - -- A range with `cursor: null` β†’ source has planned this range but not yet fetched the first page. -- A range with `cursor: "cus_abc"` β†’ resume pagination after this object. -- A range removed from the list β†’ that sub-range is complete. -- Empty `ranges: []` β†’ source is done with the assigned `time_range`. - -## Workflow - -### 1. Initialization (no existing state) - -The source receives `time_range` from the catalog and has no state for this -stream. - -1. Probe density: make one request to the Stripe list API with the full range - to estimate how many records exist. -2. Based on density, split the `time_range` into N sub-ranges (using the - `created` timestamp filter). Denser ranges get more sub-ranges. -3. Initialize state with all sub-ranges, `cursor: null`. - -``` -Engine assigns: time_range { gte: "2018-01-01", lt: "2024-04-17" } - -Source probes density β†’ estimates 200K records β†’ splits into 8 sub-ranges: - -state: { - ranges: [ - { gte: "2018-01-01", lt: "2018-10-01", cursor: null }, - { gte: "2018-10-01", lt: "2019-07-01", cursor: null }, - { gte: "2019-07-01", lt: "2020-04-01", cursor: null }, - { gte: "2020-04-01", lt: "2021-01-01", cursor: null }, - { gte: "2021-01-01", lt: "2021-10-01", cursor: null }, - { gte: "2021-10-01", lt: "2022-07-01", cursor: null }, - { gte: "2022-07-01", lt: "2023-04-01", cursor: null }, - { gte: "2023-04-01", lt: "2024-04-17", cursor: null } - ] -} -``` - -### 2. Pagination - -The source paginates sub-ranges, potentially in parallel. For each sub-range: - -1. Call the Stripe list API with `created[gte]` and `created[lt]` filters, - plus `starting_after` if resuming from a cursor. -2. Emit records. -3. Emit `source_state` with updated cursor after each page. -4. When a sub-range is exhausted (`has_more: false`), remove it from state. - -``` -After paginating first sub-range (2 pages): - -state: { - ranges: [ - { gte: "2018-01-01", lt: "2018-10-01", cursor: "cus_abc" }, // mid-page - { gte: "2018-10-01", lt: "2019-07-01", cursor: null }, - ... - ] -} -β†’ emit source_state with time_range: { gte: "2018-01-01", lt: "2024-04-17" } - -First sub-range exhausted: - -state: { - ranges: [ - { gte: "2018-10-01", lt: "2019-07-01", cursor: "cus_def" }, // now active - { gte: "2019-07-01", lt: "2020-04-01", cursor: null }, - ... - ] -} -β†’ emit source_state -``` - -### 3. Resumption (existing state) - -If the source has existing state for this stream (from a previous request in -the same sync run), it skips initialization and resumes directly from the -remaining ranges: - -``` -Source receives time_range { gte: "2018-01-01", lt: "2024-04-17" } -Existing state: { - ranges: [ - { gte: "2021-10-01", lt: "2022-07-01", cursor: "cus_xyz" }, - { gte: "2022-07-01", lt: "2023-04-01", cursor: null }, - { gte: "2023-04-01", lt: "2024-04-17", cursor: null } - ] -} - -β†’ Resume paginating from cus_xyz in [2021-10, 2022-07) -β†’ No density probing, no re-splitting -``` - -### 4. Subdivision - -If a sub-range is too dense (too many records, taking too long), the source -can subdivide it further: - -``` -Before β€” one large range taking too long: - { gte: "2021-01-01", lt: "2022-07-01", cursor: null } - -After subdivision β€” split in half: - { gte: "2021-01-01", lt: "2021-10-01", cursor: null } - { gte: "2021-10-01", lt: "2022-07-01", cursor: null } -``` - -This is the binary search behavior: ranges that don't complete in time get -split. The source decides when and how to subdivide based on observed -pagination speed. - -A sub-range that already has a cursor (mid-pagination) is NOT subdivided β€” -it's making progress. Only ranges with `cursor: null` that haven't started -or ranges that are progressing too slowly are candidates. - -### 5. Completion - -When all sub-ranges are exhausted: - -``` -state: { ranges: [] } -β†’ emit source_state with time_range -``` - -The engine observes empty ranges and marks this `time_range` as complete -in `completed_ranges`. - -## State on the Wire - -The source emits `source_state` messages with the `time_range` from the -catalog so the engine can track which range this state belongs to: - -```ts -{ - type: 'source_state', - source_state: { - state_type: 'stream', - stream: 'customers', - time_range: { gte: '2018-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' }, - data: { - ranges: [ - { gte: '2021-10-01T00:00:00Z', lt: '2022-07-01T00:00:00Z', cursor: 'cus_xyz' }, - { gte: '2022-07-01T00:00:00Z', lt: '2023-04-01T00:00:00Z', cursor: null }, - { gte: '2023-04-01T00:00:00Z', lt: '2024-04-17T00:00:00Z', cursor: null } - ] - } - } -} -``` - -## Parallel Pagination - -The source can paginate multiple sub-ranges concurrently. The number of -concurrent sub-ranges is determined by the density probe and available -concurrency budget. Records from different sub-ranges are interleaved -on the output stream, each tagged with the stream name. - -State checkpoints are emitted after each page completes, reflecting the -current state of all sub-ranges. This ensures resumability even if the -source is cut off mid-run. - -## Error Handling - -- **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP - layer with exponential backoff. If retries succeed, emit a `transient` - error trace for observability. -- **Stream errors** (resource not available, permission denied): Emit a - `stream` error trace, stop this stream, move to the next. -- **Global errors** (invalid API key): Emit a `global` error trace, stop. - -The source does not store error state. If a sub-range fails after all -retries, the source emits an error trace and moves on. The sub-range -remains in state (with its cursor) for the next attempt. - -## Events / Incremental Sync - -After backfill completes, the source switches to incremental mode using -Stripe's `/events` API or WebSocket. This is outside the `time_range` -model β€” events are a live stream, not a bounded range. - -The global state stores an `events_cursor` for resumption: - -```ts -// source.global -{ events_cursor: "2024-04-16T23:50:00Z" } -``` - -Events and backfill can run concurrently β€” backfill covers historical data -within `time_range`, events cover real-time changes. From 29315805c5a836899d28a987bb137ce0f77281b1 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:01:01 +0000 Subject: [PATCH 13/32] docs: add concurrency model to Stripe source sync lifecycle - max_concurrent_streams (configurable, default 5, capped at catalog size) - max_requests_per_second (inferred: live=20, test=10) - max_segments_per_stream (derived: rps / effective_streams) - Examples showing budget distribution across different scenarios - Single-stream syncs get the full rate limit budget Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 51 +++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index bdf6de99..f812f7ef 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -212,12 +212,55 @@ can track which range this state belongs to: } ``` +## Concurrency + +Three controls govern how the source uses the Stripe API: + +```ts +// Source config β€” only max_concurrent_streams is user-configurable +type StripeSourceConfig = { + api_key: string + account_id?: string + max_concurrent_streams?: number // default 5 +} + +// Derived internally by the source: +// live_mode = inferred from api_key prefix (sk_live_ vs sk_test_) +// max_requests_per_second = live_mode ? 20 : 10 +// effective_streams = min(max_concurrent_streams, configured_stream_count) +// max_segments_per_stream = floor(max_requests_per_second / effective_streams) +``` + +| Control | What it controls | How it's set | +|---|---|---| +| `max_concurrent_streams` | Streams paginating in parallel | Config (default 5), capped at catalog size | +| `max_requests_per_second` | Global rate limit across all activity | Inferred from API key mode | +| `max_segments_per_stream` | Sub-ranges per stream (n-ary search fan-out) | Derived: rps / concurrent streams | + +### Examples + +| Scenario | Mode | Streams | `effective_streams` | `rps` | `max_segments_per_stream` | Max concurrent requests | +|---|---|---|---|---|---|---| +| 20 streams, live | live | 20 | 5 | 20 | 4 | 20 | +| 20 streams, test | test | 20 | 5 | 10 | 2 | 10 | +| 3 streams, live | live | 3 | 3 | 20 | 6 | 18 | +| 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | +| 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | + +When fewer streams are configured, each stream gets more segments β€” the full +rate limit budget is distributed across whatever streams exist. A single-stream +sync gets the entire budget. + ## Parallel Pagination -The source can paginate multiple ranges from `remaining` concurrently. -Records from different ranges are interleaved on the output stream. State -checkpoints are emitted after each page, reflecting the current state of -all ranges. This ensures resumability if the source is cut off mid-run. +The source paginates up to `max_segments_per_stream` ranges from `remaining` +concurrently per stream, and up to `effective_streams` streams in parallel. +Records from different ranges/streams are interleaved on the output stream. +State checkpoints are emitted after each page, reflecting the current state +of all ranges. This ensures resumability if the source is cut off mid-run. + +The global rate limiter (`max_requests_per_second`) governs all API calls +regardless of which stream or segment they belong to. ## Error Handling From 4ac53d785ce26c02c51afde4fd118b4d5348f3c6 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:03:53 +0000 Subject: [PATCH 14/32] docs: treat events as regular stream, not incremental mode Events endpoint uses the same time_range + remaining model as all other streams. Live event polling is experimental and opt-in, stored in source.global separately from backfill cursor logic. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 54 ++++++++++----------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index f812f7ef..3a1d4510 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -17,9 +17,9 @@ the right granularity by doing the work. ```ts type StripeStreamState = { remaining: Array<{ - gte: string // ISO 8601 β€” inclusive lower bound - lt: string // ISO 8601 β€” exclusive upper bound - cursor: string | null // Stripe pagination cursor; null = not yet started + gte: string // ISO 8601 β€” inclusive lower bound + lt: string // ISO 8601 β€” exclusive upper bound + cursor: string | null // Stripe pagination cursor; null = not yet started }> } ``` @@ -111,6 +111,7 @@ This is the n-ary search: large ranges get split, small ranges complete directly. The source adapts to data density without upfront probing. **When to subdivide:** + - The source can track pagination speed (records/second, pages/second). - If a range is progressing too slowly relative to the time budget, subdivide. - Only the unpaginated portion is split β€” the cursor range continues as-is. @@ -221,7 +222,7 @@ Three controls govern how the source uses the Stripe API: type StripeSourceConfig = { api_key: string account_id?: string - max_concurrent_streams?: number // default 5 + max_concurrent_streams?: number // default 5 } // Derived internally by the source: @@ -231,21 +232,21 @@ type StripeSourceConfig = { // max_segments_per_stream = floor(max_requests_per_second / effective_streams) ``` -| Control | What it controls | How it's set | -|---|---|---| -| `max_concurrent_streams` | Streams paginating in parallel | Config (default 5), capped at catalog size | -| `max_requests_per_second` | Global rate limit across all activity | Inferred from API key mode | -| `max_segments_per_stream` | Sub-ranges per stream (n-ary search fan-out) | Derived: rps / concurrent streams | +| Control | What it controls | How it's set | +| ------------------------- | -------------------------------------------- | ------------------------------------------ | +| `max_concurrent_streams` | Streams paginating in parallel | Config (default 5), capped at catalog size | +| `max_requests_per_second` | Global rate limit across all activity | Inferred from API key mode | +| `max_segments_per_stream` | Sub-ranges per stream (n-ary search fan-out) | Derived: rps / concurrent streams | ### Examples -| Scenario | Mode | Streams | `effective_streams` | `rps` | `max_segments_per_stream` | Max concurrent requests | -|---|---|---|---|---|---|---| -| 20 streams, live | live | 20 | 5 | 20 | 4 | 20 | -| 20 streams, test | test | 20 | 5 | 10 | 2 | 10 | -| 3 streams, live | live | 3 | 3 | 20 | 6 | 18 | -| 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | -| 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | +| Scenario | Mode | Streams | `effective_streams` | `rps` | `max_segments_per_stream` | Max concurrent requests | +| ---------------- | ---- | ------- | ------------------- | ----- | ------------------------- | ----------------------- | +| 20 streams, live | live | 20 | 5 | 20 | 4 | 20 | +| 20 streams, test | test | 20 | 5 | 10 | 2 | 10 | +| 3 streams, live | live | 3 | 3 | 20 | 6 | 18 | +| 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | +| 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | When fewer streams are configured, each stream gets more segments β€” the full rate limit budget is distributed across whatever streams exist. A single-stream @@ -274,18 +275,13 @@ regardless of which stream or segment they belong to. The source does not store error state. If a range fails after all retries, the range stays in `remaining` with its cursor for the next attempt. -## Events / Incremental Sync - -After backfill completes (`remaining: []`), the source switches to -incremental mode using Stripe's `/events` API or WebSocket. This is outside -the `time_range` model β€” events are a live stream, not a bounded range. - -The global state stores an `events_cursor` for resumption: +## Events -```ts -// source.global -{ events_cursor: "2024-04-16T23:50:00Z" } -``` +The `/events` endpoint is treated as just another stream in the catalog β€” +same `time_range` model, same `remaining`-based pagination. No special +incremental mode or live polling by default. -Events and backfill can run concurrently β€” backfill covers historical data -within `time_range`, events cover real-time changes. +For experimental live event polling (using events as a webhook replacement), +an opt-in flag stores cursor state in `source.global`, which is completely +separate from the per-stream backfill cursor logic. This is not enabled by +default. From 02e6f1e5b3d02c1da039a82e9394cb25eb9c2af1 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:05:14 +0000 Subject: [PATCH 15/32] docs: keep 'remaining' for state field, 'segments' for config remaining describes the work left to do in source state. max_segments_per_stream is the config limit on fan-out. Different concepts, different names. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index 3a1d4510..295e4c24 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -248,7 +248,7 @@ type StripeSourceConfig = { | 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | | 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | -When fewer streams are configured, each stream gets more segments β€” the full +When fewer streams are configured, each stream gets more remaining β€” the full rate limit budget is distributed across whatever streams exist. A single-stream sync gets the entire budget. From 26954d336e53dce792edaf83414e096fffedfd4a Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:08:21 +0000 Subject: [PATCH 16/32] =?UTF-8?q?docs:=20fix=20wording=20=E2=80=94=20'more?= =?UTF-8?q?=20segments'=20not=20'more=20remaining'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index 295e4c24..3a1d4510 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -248,7 +248,7 @@ type StripeSourceConfig = { | 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | | 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | -When fewer streams are configured, each stream gets more remaining β€” the full +When fewer streams are configured, each stream gets more segments β€” the full rate limit budget is distributed across whatever streams exist. A single-stream sync gets the entire budget. From 19176115776f343ae797ec686c990cf172f61a32 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:18:35 +0000 Subject: [PATCH 17/32] docs: subdivision between requests, range_complete as stream_status - Subdivision happens between requests, not mid-request - Source subdivides if a range didn't complete in previous request - range_complete is a stream_status subtype (Stripe polymorphism pattern) - Full example shows actual messages emitted across 3 requests - stream_status uses 'start' not 'started' Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 159 +++++++++++--------- docs/engine/sync-lifecycle.md | 38 +++-- 2 files changed, 118 insertions(+), 79 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index 3a1d4510..958defc5 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -76,49 +76,33 @@ state: { ### 3. Subdivision (n-ary search) -If a range is too large to complete within the request budget (too many -records, approaching time limit), the source subdivides it. The remaining -unpaginated portion of the range is split into N smaller ranges: +If a range didn't complete in the previous request, the source subdivides it +at the start of the next request. The source knows the `created` timestamp of +the last record it paginated (from the cursor). It splits the unpaginated +portion into N parts (where N = `max_segments_per_stream`): ``` -Before β€” range is too large, source has paginated up to created=2020-06-15 -via cursor but there's much more: +Previous request ended with: + remaining: [{ gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_xyz" }] -state: { - remaining: [ - { gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_xyz" } - ] -} +Last record seen had created=2020-06-15. Range didn't complete β†’ subdivide. +The paginated portion [2018, 2020-06-15) keeps its cursor. +The unpaginated portion [2020-06-15, 2024-04-17) splits into N=2: -Source decides to subdivide. It knows it has paginated records up to -created=2020-06-15 (from the last record seen). It splits the remaining -unpaginated portion [2020-06-15, 2024-04-17) into N=2 parts: - -state: { remaining: [ { gte: "2018-01-01", lt: "2020-06-15", cursor: "cus_xyz" }, { gte: "2020-06-15", lt: "2022-05-16", cursor: null }, { gte: "2022-05-16", lt: "2024-04-17", cursor: null } ] -} -β†’ emit source_state ``` -The original range narrows to `[2018-01-01, 2020-06-15)` (the portion being -paginated with a cursor), and the rest becomes new ranges with `cursor: null`. - -This is the n-ary search: large ranges get split, small ranges complete -directly. The source adapts to data density without upfront probing. - -**When to subdivide:** - -- The source can track pagination speed (records/second, pages/second). -- If a range is progressing too slowly relative to the time budget, subdivide. -- Only the unpaginated portion is split β€” the cursor range continues as-is. +**When to subdivide:** At the start of a request, if any range in `remaining` +has a cursor (meaning it was in progress last request but didn't complete). +Subdivision happens between requests, not mid-request. -**Recursive subdivision:** If a subdivided range is still too large, it gets -split again on the next pass. Each iteration narrows the ranges until they're -small enough to complete. +**Recursive:** If a subdivided range still doesn't complete in one request, +it gets split again next time. Each pass narrows the ranges until they're +small enough to complete in a single request. ### 4. Resumption (existing state) @@ -139,63 +123,102 @@ Existing state: { ### 5. Completion -When `remaining` is empty: +When a sub-range is exhausted, the source removes it from `remaining` and +emits a `stream_status: range_complete`: ``` -state: { remaining: [] } -β†’ emit source_state with time_range +β†’ emit trace { stream_status: { stream: 'customers', status: 'range_complete', + range_complete: { gte: '2018-01-01', lt: '2019-06-01' } } } ``` -The engine observes empty `remaining` and marks this `time_range` as complete -in `completed_ranges`. +The engine merges this into `completed_ranges`. + +When all sub-ranges are done (`remaining: []`), the source emits +`stream_status: complete` for the stream. ## Full Example +Shows the messages emitted by the source during a two-request backfill of +`customers` with `time_range: [2018, 2024)`. + +### Request 1 β€” full range, doesn't complete + ``` -Request 1: - Engine assigns time_range [2018, 2024) to customers +Source initializes: remaining: [{ gte: "2018", lt: "2024", cursor: null }] - remaining: [{ gte: "2018", lt: "2024", cursor: null }] - β†’ paginate... got 10K records, up to created=2019-03 - remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] - β†’ still going... approaching time limit, subdivide - remaining: [ - { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // finishing current - { gte: "2019-03", lt: "2021-09", cursor: null }, // new - { gte: "2021-09", lt: "2024", cursor: null } // new - ] - β†’ first range completes - remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: null }, - { gte: "2021-09", lt: "2024", cursor: null } - ] - β†’ time limit hit, emit source_state, stop +← trace { stream_status: { stream: "customers", status: "start" } } +← record { stream: "customers", data: { id: "cus_001", ... } } +← record { stream: "customers", data: { id: "cus_002", ... } } + ... 5000 records ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] } } + ... source iterator cut off (time limit / state limit) ... - end { has_more: true } +← end { has_more: true } +``` -Request 2 (continuation, same sync_run_id): - Source resumes with remaining from state +Range didn't complete in one request β†’ source will subdivide on next request. +### Request 2 β€” source subdivides, finishes first sub-range + +``` +Source resumes, sees remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] +Last record seen was created=2019-03. Range didn't complete last request β†’ subdivide: remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: null }, - { gte: "2021-09", lt: "2024", cursor: null } - ] - β†’ paginate first range... completes - remaining: [ - { gte: "2021-09", lt: "2024", cursor: null } + { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // current work (has cursor) + { gte: "2019-03", lt: "2021-09", cursor: null }, // new + { gte: "2021-09", lt: "2024", cursor: null } // new ] - β†’ paginate second range... completes - remaining: [] - end { has_more: false } +← record { stream: "customers", data: { ... } } + ... finishes [2018, 2019-03) ... +← trace { stream_status: { stream: "customers", status: "range_complete", + range_complete: { gte: "2018", lt: "2019-03" } } } +← state { stream: "customers", data: { remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: null }, + { gte: "2021-09", lt: "2024", cursor: null } + ] } } +← record { stream: "customers", data: { ... } } + ... starts [2019-03, 2021-09), gets partway through ... +← state { stream: "customers", data: { remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: "cus_12000" }, + { gte: "2021-09", lt: "2024", cursor: null } + ] } } + ... cut off ... + +← end { has_more: true } +``` - Engine marks [2018, 2024) as completed. +### Request 3 β€” finishes remaining ranges + +``` +Source resumes: remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: "cus_12000" }, + { gte: "2021-09", lt: "2024", cursor: null } +] +These ranges completed last request partially β€” no further subdivision needed, +just resume from cursors. + +← record { stream: "customers", data: { ... } } + ... finishes [2019-03, 2021-09) ... +← trace { stream_status: { stream: "customers", status: "range_complete", + range_complete: { gte: "2019-03", lt: "2021-09" } } } +← record { stream: "customers", data: { ... } } + ... finishes [2021-09, 2024) ... +← trace { stream_status: { stream: "customers", status: "range_complete", + range_complete: { gte: "2021-09", lt: "2024" } } } +← state { stream: "customers", data: { remaining: [] } } +← trace { stream_status: { stream: "customers", status: "complete" } } + +← end { has_more: false } ``` +Engine's `completed_ranges` for customers after merging all `range_complete` messages: +`[{ gte: "2018", lt: "2024" }]` + ## State on the Wire -The source emits `source_state` messages with `time_range` so the engine -can track which range this state belongs to: +Source state is opaque to the engine. The engine learns about range completion +via `stream_status: range_complete` messages, not by inspecting source state: ```ts { diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index b9a3de46..e8e62b43 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -69,16 +69,19 @@ Sources are iterators that yield these message types: // Data record { type: 'record', record: { stream: string, data: Record, emitted_at: string } } -// Checkpoint (per-stream β€” most common) -// time_range tells the engine which range this checkpoint belongs to. -// When data has no cursor (pagination done), engine marks this range as synced. -{ type: 'source_state', source_state: { state_type: 'stream', stream: string, time_range?: { gte: string, lt: string }, data: unknown } } +// Checkpoint (per-stream β€” most common). Data is opaque to the engine. +{ type: 'source_state', source_state: { state_type: 'stream', stream: string, data: unknown } } // Checkpoint (global β€” e.g. events cursor shared across all streams) { type: 'source_state', source_state: { state_type: 'global', data: unknown } } -// Lifecycle signal -{ type: 'trace', trace: { trace_type: 'stream_status', stream_status: { stream: string, status: 'started' | 'complete' } } } +// Stream status β€” discriminated union on status (Stripe polymorphism pattern) +{ type: 'trace', trace: { trace_type: 'stream_status', stream_status: StreamStatus } } + +// where StreamStatus is: +// { stream: string, status: 'start' } +// { stream: string, status: 'range_complete', range_complete: { gte: string, lt: string } } +// { stream: string, status: 'complete' } // Error β€” discriminated union on error_level (see Error Handling) { type: 'trace', trace: { trace_type: 'error', error: SyncError & { stack_trace?: string } } } @@ -152,15 +155,28 @@ and distilled into `progress`. ## Stream Status -Source emits `stream_status` trace messages with `started` and `complete` per -stream. The engine uses these to know when a stream is active. +`stream_status` is a discriminated union on `status` (Stripe polymorphism +pattern β€” the status value names the payload key): + +```ts +type StreamStatus = + | { stream: string; status: 'start' } + | { stream: string; status: 'range_complete'; range_complete: { gte: string; lt: string } } + | { stream: string; status: 'complete' } +``` + +| Status | Emitted by | Engine action | +|---|---|---| +| `start` | Source | Stream is active | +| `range_complete` | Source | Merge range into `completed_ranges` | +| `complete` | Source (optional) | Stream is done; engine derives this if source exhausts without it | A stream's backfill is done when `completed_ranges` covers the full range `[0, started_at)`. -The source manages sub-ranges internally β€” the engine doesn't see or track them. -The engine learns about completed ranges from `source_state` messages that -include a `time_range` and have no remaining cursor. +The source manages sub-ranges internally β€” the engine doesn't see or track +them. The engine learns about completed ranges via `range_complete` status +messages. Errors are orthogonal to lifecycle. A stream can be `complete` with errors (some pages failed but the stream moved past them) or `incomplete` without From 9535f36cf306d0df1e1113507bf121c2de922f4b Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:19:16 +0000 Subject: [PATCH 18/32] docs: fix example to use 100 records per page (Stripe API limit) Each Stripe API page returns max 100 records. Example now shows page-by-page pagination with state checkpoints after each page. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index 958defc5..f0728c49 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -143,15 +143,22 @@ Shows the messages emitted by the source during a two-request backfill of ### Request 1 β€” full range, doesn't complete +Stripe returns max 100 records per page. Each page = 1 API request = 1 state +checkpoint. + ``` Source initializes: remaining: [{ gte: "2018", lt: "2024", cursor: null }] ← trace { stream_status: { stream: "customers", status: "start" } } ← record { stream: "customers", data: { id: "cus_001", ... } } -← record { stream: "customers", data: { id: "cus_002", ... } } - ... 5000 records ... + ... 100 records (page 1) ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_100" }] } } +← record { stream: "customers", data: { ... } } + ... 100 records (page 2) ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_200" }] } } + ... pages 3-50 (5000 records total) ... ← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] } } - ... source iterator cut off (time limit / state limit) ... + ... source cut off (time limit / state limit) ... ← end { has_more: true } ``` @@ -162,25 +169,26 @@ Range didn't complete in one request β†’ source will subdivide on next request. ``` Source resumes, sees remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] -Last record seen was created=2019-03. Range didn't complete last request β†’ subdivide: +Last record had created=2019-03. Range didn't complete β†’ subdivide: remaining: [ - { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // current work (has cursor) + { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // current (has cursor) { gte: "2019-03", lt: "2021-09", cursor: null }, // new { gte: "2021-09", lt: "2024", cursor: null } // new ] ← record { stream: "customers", data: { ... } } - ... finishes [2018, 2019-03) ... + ... 100 records (page) ... +← state { ... } + ... finishes [2018, 2019-03) after a few more pages ... ← trace { stream_status: { stream: "customers", status: "range_complete", range_complete: { gte: "2018", lt: "2019-03" } } } ← state { stream: "customers", data: { remaining: [ { gte: "2019-03", lt: "2021-09", cursor: null }, { gte: "2021-09", lt: "2024", cursor: null } ] } } -← record { stream: "customers", data: { ... } } - ... starts [2019-03, 2021-09), gets partway through ... + ... starts [2019-03, 2021-09), paginates several pages ... ← state { stream: "customers", data: { remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: "cus_12000" }, + { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, { gte: "2021-09", lt: "2024", cursor: null } ] } } ... cut off ... @@ -192,18 +200,15 @@ Last record seen was created=2019-03. Range didn't complete last request β†’ sub ``` Source resumes: remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: "cus_12000" }, + { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, { gte: "2021-09", lt: "2024", cursor: null } ] -These ranges completed last request partially β€” no further subdivision needed, -just resume from cursors. +These ranges made progress last request β€” no further subdivision, resume. -← record { stream: "customers", data: { ... } } - ... finishes [2019-03, 2021-09) ... + ... paginates [2019-03, 2021-09) page by page ... ← trace { stream_status: { stream: "customers", status: "range_complete", range_complete: { gte: "2019-03", lt: "2021-09" } } } -← record { stream: "customers", data: { ... } } - ... finishes [2021-09, 2024) ... + ... paginates [2021-09, 2024) page by page ... ← trace { stream_status: { stream: "customers", status: "range_complete", range_complete: { gte: "2021-09", lt: "2024" } } } ← state { stream: "customers", data: { remaining: [] } } From 60cb06d57816b76cd4b2094fd049c4f000ac3db1 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:27:45 +0000 Subject: [PATCH 19/32] docs: add engine log messages to sync lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine emits log messages (info/warn/error) alongside progress. Tolerant processing β€” anomalies are logged as warnings, not rejected. Defines the full set of engine log messages by level. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 47 ++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index e8e62b43..92f18f71 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -92,7 +92,7 @@ Sources are iterators that yield these message types: ### Engine β†’ client -The engine emits three message types: `progress`, `record`, and `end`. +The engine emits four message types: `progress`, `record`, `log`, and `end`. ```ts // Progress β€” emitted on every source_state checkpoint and stream_status change. @@ -126,6 +126,9 @@ The engine emits three message types: `progress`, `record`, and `end`. // Records β€” passed through from source { type: 'record', record: { stream: string, data: Record, emitted_at: string } } +// Log β€” engine operational messages (see Engine Logs below) +{ type: 'log', log: { level: 'info' | 'warn' | 'error', message: string } } + // Terminal β€” this request is done. // end.request has the same shape as progress but scoped to this request only. { @@ -147,9 +150,9 @@ The engine emits three message types: `progress`, `record`, and `end`. | This run (across requests)| Latest `progress` message (ProgressPayload) | | All time (across runs) | Sum of `completed_ranges` coverage + record counts | -The engine does NOT emit trace messages to the client. Errors are included -inside `progress`. Source traces and logs are consumed by the engine -and distilled into `progress`. +The engine does NOT emit trace messages to the client. Source errors are +included inside `progress`. Source traces and logs are consumed by the engine +and distilled into `progress` and `log` messages. --- @@ -530,6 +533,42 @@ retries, timeouts) are managed internally by the source. --- +## Engine Logs + +The engine emits `log` messages for operational events. These are alertable +in production. The engine processes source messages tolerantly β€” it does not +reject unexpected ordering β€” but logs warnings for anomalies. + +### info + +| Message | When | +|---|---| +| `stream started: {stream}` | Source emits `stream_status: start` | +| `stream complete: {stream}` | Source emits `stream_status: complete` | +| `range complete: {stream} [{gte}, {lt})` | Source emits `stream_status: range_complete` | +| `sync run started: {sync_run_id}` | New sync run begins | +| `sync run continued: {sync_run_id}` | Continuation of existing run | +| `request complete: {record_count} records, {state_count} states` | Request finishes | + +### warn + +| Message | When | +|---|---| +| `state before start: {stream}` | Source emitted `source_state` for a stream before `stream_status: start` | +| `state after complete: {stream}` | Source emitted `source_state` for a stream after `stream_status: complete` | +| `duplicate start: {stream}` | Source emitted `stream_status: start` for a stream that already started | +| `unknown stream: {stream}` | Source emitted a message for a stream not in the catalog | + +### error + +| Message | When | +|---|---| +| `global error: {message}` | Source emitted `error_level: global` β€” sync aborted | +| `stream error: {stream}: {message}` | Source emitted `error_level: stream` β€” stream skipped | +| `source crashed: {message}` | Source iterator threw an exception | + +--- + ## Wire Format NDJSON. One message per line. From 2c78fdda2d62be0b634fb8cb6b1aec09d0047499 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:29:00 +0000 Subject: [PATCH 20/32] docs: engine logs warn/error only, source logs rps and rate limits Engine: no redundant info logs (progress stream covers that). Only warn (anomalies) and error (failures). Source: emits info logs for real-time rps, warn logs for rate limits and retries. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 11 +++++++++++ docs/engine/sync-lifecycle.md | 18 +++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index f0728c49..ae474332 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -291,6 +291,17 @@ of all ranges. This ensures resumability if the source is cut off mid-run. The global rate limiter (`max_requests_per_second`) governs all API calls regardless of which stream or segment they belong to. +## Source Logs + +The Stripe source emits `log` messages for real-time operational visibility. +These are passed through by the engine. + +| Level | Message | When | +|---|---|---| +| info | `{stream}: {rps} requests/sec` | Periodically during pagination | +| warn | `rate limited: retrying in {n}s` | Stripe returned 429 | +| warn | `retry {n}/{max}: {status} {message}` | Request failed, retrying | + ## Error Handling - **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 92f18f71..abd25459 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -535,20 +535,12 @@ retries, timeouts) are managed internally by the source. ## Engine Logs -The engine emits `log` messages for operational events. These are alertable -in production. The engine processes source messages tolerantly β€” it does not -reject unexpected ordering β€” but logs warnings for anomalies. +The engine emits `log` messages for anomalies and failures only. Normal +progress (stream starts, completions, record counts) is already in the +`progress` stream β€” no redundant info logs. -### info - -| Message | When | -|---|---| -| `stream started: {stream}` | Source emits `stream_status: start` | -| `stream complete: {stream}` | Source emits `stream_status: complete` | -| `range complete: {stream} [{gte}, {lt})` | Source emits `stream_status: range_complete` | -| `sync run started: {sync_run_id}` | New sync run begins | -| `sync run continued: {sync_run_id}` | Continuation of existing run | -| `request complete: {record_count} records, {state_count} states` | Request finishes | +The engine processes source messages tolerantly β€” it does not reject +unexpected ordering β€” but logs warnings so they're alertable in production. ### warn From 3676ce7b0c3685b52e79401b98009d1db4153b06 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:32:30 +0000 Subject: [PATCH 21/32] docs: remove fields and system_columns from ConfiguredStream fields is redundant with json_schema (use schema to express field projection). system_columns is a destination concern, not protocol. Also made time_range.gte optional (omit for "from the beginning"). Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 168 +++++++++++++++++----------------- 1 file changed, 85 insertions(+), 83 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index abd25459..c939c612 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -31,19 +31,19 @@ range management, stream lifecycle guarantees, and stall detection to the engine CLIENT ←—start/endβ€”β†’ ENGINE ←—iteratorβ€”β†’ SOURCE ``` -| Concern | Client | Engine | Source | -| ---------------------------- | ---------------------- | ----------------------------------------- | ----------------------------------- | -| What to sync (streams) | Provides catalog | Adjusts catalog (time_range, deprioritize)| Syncs what it's given | -| When to sync (scheduling) | Decides | β€” | β€” | -| Run identity | Generates sync_run_id | Freezes bounds, tracks continuations | Unaware | -| Time range bounds | β€” | Computes, injects into catalog | Respects `time_range` if present | -| Internal pagination | β€” | β€” | Manages ranges, parallel pages | -| Stream lifecycle | Consumes progress | Guarantees terminal status | Emits `started`, optionally `complete` | -| Progress reporting | Consumes | Enriches source signals, emits progress | Emits raw stream_status + records | -| Error reporting | Decides retry policy | Passes through, tracks for stalls | Emits trace errors | -| State | Opaque round-trip | Manages engine section | Manages source section | -| Stall detection | β€” | Tracks per-stream across runs | β€” | -| `has_more` | Reads, acts | Derives from source + engine state | β€” | +| Concern | Client | Engine | Source | +| ------------------------- | --------------------- | ------------------------------------------ | -------------------------------------- | +| What to sync (streams) | Provides catalog | Adjusts catalog (time_range, deprioritize) | Syncs what it's given | +| When to sync (scheduling) | Decides | β€” | β€” | +| Run identity | Generates sync_run_id | Freezes bounds, tracks continuations | Unaware | +| Time range bounds | β€” | Computes, injects into catalog | Respects `time_range` if present | +| Internal pagination | β€” | β€” | Manages ranges, parallel pages | +| Stream lifecycle | Consumes progress | Guarantees terminal status | Emits `started`, optionally `complete` | +| Progress reporting | Consumes | Enriches source signals, emits progress | Emits raw stream_status + records | +| Error reporting | Decides retry policy | Passes through, tracks for stalls | Emits trace errors | +| State | Opaque round-trip | Manages engine section | Manages source section | +| Stall detection | β€” | Tracks per-stream across runs | β€” | +| `has_more` | Reads, acts | Derives from source + engine state | β€” | --- @@ -143,12 +143,12 @@ The engine emits four message types: `progress`, `record`, `log`, and `end`. `ProgressPayload` is used in two places with different scopes: -| Scope | Where to find it | -| ------------------------- | --------------------------------------------- | -| Between two progress msgs | Client diffs consecutive `progress` messages | -| This request | `end.request_progress` (ProgressPayload) | -| This run (across requests)| Latest `progress` message (ProgressPayload) | -| All time (across runs) | Sum of `completed_ranges` coverage + record counts | +| Scope | Where to find it | +| -------------------------- | -------------------------------------------------- | +| Between two progress msgs | Client diffs consecutive `progress` messages | +| This request | `end.request_progress` (ProgressPayload) | +| This run (across requests) | Latest `progress` message (ProgressPayload) | +| All time (across runs) | Sum of `completed_ranges` coverage + record counts | The engine does NOT emit trace messages to the client. Source errors are included inside `progress`. Source traces and logs are consumed by the engine @@ -168,11 +168,11 @@ type StreamStatus = | { stream: string; status: 'complete' } ``` -| Status | Emitted by | Engine action | -|---|---|---| -| `start` | Source | Stream is active | -| `range_complete` | Source | Merge range into `completed_ranges` | -| `complete` | Source (optional) | Stream is done; engine derives this if source exhausts without it | +| Status | Emitted by | Engine action | +| ---------------- | ----------------- | ----------------------------------------------------------------- | +| `start` | Source | Stream is active | +| `range_complete` | Source | Merge range into `completed_ranges` | +| `complete` | Source (optional) | Stream is done; engine derives this if source exhausts without it | A stream's backfill is done when `completed_ranges` covers the full range `[0, started_at)`. @@ -197,22 +197,20 @@ before passing to the source. ```ts type ConfiguredStream = { stream: { - name: string // e.g. "customers", "invoices" - primary_key: string[][] // e.g. [["id"]] + name: string // e.g. "customers", "invoices" + primary_key: string[][] // e.g. [["id"]] json_schema?: Record - metadata?: Record // e.g. { api_version, account_id, live_mode } + metadata?: Record // e.g. { api_version, account_id, live_mode } } sync_mode: 'full_refresh' | 'incremental' destination_sync_mode: 'append' | 'overwrite' | 'append_dedup' cursor_field?: string[] - fields?: string[] // field projection - backfill_limit?: number // cap backfill to N records - system_columns?: Array<{ name: string; type: string; index: boolean }> + backfill_limit?: number // cap backfill to N records - // NEW β€” set by engine, not client + // Set by engine, not client time_range?: { - gte: string // inclusive lower bound (ISO 8601) - lt: string // exclusive upper bound (ISO 8601) + gte?: string // inclusive lower bound (ISO 8601); omit for "from the beginning" + lt: string // exclusive upper bound (ISO 8601) } } @@ -225,11 +223,11 @@ type ConfiguredCatalog = { ```ts type StartPayload = { - sync_run_id: string // client-generated UUID - source_config: Record // source-specific (e.g. Stripe API key, account) + sync_run_id: string // client-generated UUID + source_config: Record // source-specific (e.g. Stripe API key, account) destination_config: Record // destination-specific (e.g. Postgres connection) configured_catalog: ConfiguredCatalog - state?: SyncState // from previous end; omit on first sync + state?: SyncState // from previous end; omit on first sync } ``` @@ -238,8 +236,8 @@ type StartPayload = { ```ts type EndPayload = { has_more: boolean - state: SyncState // round-trip into next start - request: ProgressPayload // stats for this request only (same shape as progress) + state: SyncState // round-trip into next start + request: ProgressPayload // stats for this request only (same shape as progress) } ``` @@ -252,19 +250,19 @@ a reducer. ```ts // Errors are a discriminated union on error_level. type SyncError = - | { error_level: 'global'; message: string } - | { error_level: 'stream'; message: string; stream: string } + | { error_level: 'global'; message: string } + | { error_level: 'stream'; message: string; stream: string } | { error_level: 'transient'; message: string; stream?: string } type StreamProgress = { - completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges - record_count: number // records this run (across requests) - state_count: number // checkpoints this run for this stream + completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges + record_count: number // records this run (across requests) + state_count: number // checkpoints this run for this stream } type ProgressPayload = { - elapsed_ms: number // wall-clock since run started (across requests) - global_state_count: number // total checkpoints this run (all streams) + elapsed_ms: number // wall-clock since run started (across requests) + global_state_count: number // total checkpoints this run (all streams) derived: { // Computed from the sum of all stream record_counts / (elapsed_ms / 1000). // Uses run-level totals, not windowed β€” so this is the average rate since @@ -275,8 +273,8 @@ type ProgressPayload = { // Computed from global_state_count / (elapsed_ms / 1000). states_per_second: number } - streams: Record // keyed by stream name - errors: SyncError[] // all errors accumulated this run + streams: Record // keyed by stream name + errors: SyncError[] // all errors accumulated this run } ``` @@ -284,19 +282,19 @@ type ProgressPayload = { ```ts type SyncState = { - source: SourceState // opaque to engine β€” cursor data - engine: EngineState // opaque to client β€” run progress + run identity + source: SourceState // opaque to engine β€” cursor data + engine: EngineState // opaque to client β€” run progress + run identity } type SourceState = { - streams: Record // per-stream cursor data, keyed by stream name - global: Record // source-wide data (e.g. events cursor) + streams: Record // per-stream cursor data, keyed by stream name + global: Record // source-wide data (e.g. events cursor) } type EngineState = { - sync_run_id: string // current run ID - started_at: string // ISO 8601 β€” frozen snapshot upper bound - run_progress: ProgressPayload // accumulated run-level progress + sync_run_id: string // current run ID + started_at: string // ISO 8601 β€” frozen snapshot upper bound + run_progress: ProgressPayload // accumulated run-level progress } ``` @@ -309,7 +307,7 @@ opaque; the types below are source-internal. // Per-stream state (source-internal, opaque to engine) // Minimal β€” just a pagination cursor. Range management is engine's job. type StripeStreamState = { - page_cursor: string | null // Stripe list pagination cursor + page_cursor: string | null // Stripe list pagination cursor } ``` @@ -318,17 +316,18 @@ The engine tracks which ranges are complete and which need work via `completed_ranges` and `pending_ranges` in engine state. **Example β€” two streams mid-sync:** + ```jsonc { "source": { "streams": { "customers": { "page_cursor": "cus_xyz" }, - "invoices": { "page_cursor": null } + "invoices": { "page_cursor": null }, }, "global": { - "events_cursor": "2024-04-16T23:50:00Z" - } - } + "events_cursor": "2024-04-16T23:50:00Z", + }, + }, } ``` @@ -341,13 +340,14 @@ across runs. ```ts type EngineState = { - sync_run_id: string // current run ID - started_at: string // ISO 8601 β€” frozen snapshot upper bound - run_progress: ProgressPayload // accumulated run-level progress + sync_run_id: string // current run ID + started_at: string // ISO 8601 β€” frozen snapshot upper bound + run_progress: ProgressPayload // accumulated run-level progress } ``` **Example β€” customers fully synced, invoices mid-backfill, big_table stalled:** + ```jsonc { "engine": { @@ -361,22 +361,22 @@ type EngineState = { "customers": { "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], "record_count": 45000, - "state_count": 16 + "state_count": 16, }, "invoices": { "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], "record_count": 1200, - "state_count": 8 + "state_count": 8, }, "big_table": { "completed_ranges": [], "record_count": 0, - "state_count": 0 - } + "state_count": 0, + }, }, - "errors": [] - } - } + "errors": [], + }, + }, } ``` @@ -409,6 +409,7 @@ A sync run is identified by `sync_run_id`. Within a run, the upper time bound ### Completion When `has_more: false`: + - All streams completed their ranges or were marked `incomplete`. - Engine promotes completed `pending_range` entries to `completed_ranges`. - Client should use a new `sync_run_id` for the next sync. @@ -497,11 +498,11 @@ has_more = true if any catalog stream where: Errors carry their blast radius. The `error_level` determines the engine's action: -| `error_level` | Blast radius | Engine action | Example | -|---|---|---|---| -| `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | -| `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | -| `transient` | One request | Informational | Rate limited, retried 3x in 4.2s | +| `error_level` | Blast radius | Engine action | Example | +| ------------- | ------------ | ------------------------------------ | ----------------------------------------- | +| `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | +| `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | +| `transient` | One request | Informational | Rate limited, retried 3x in 4.2s | ### Source β†’ engine error flow @@ -516,6 +517,7 @@ Errors carry their blast radius. The `error_level` determines the engine's actio ``` The source decides the `error_level`: + - **Transient**: Request failed and retried β€” emit for observability. - **Stream**: Stream-level failure (e.g. resource not enabled) β€” skip stream. - **Global**: Unrecoverable (e.g. invalid credentials) β€” stop everything. @@ -544,20 +546,20 @@ unexpected ordering β€” but logs warnings so they're alertable in production. ### warn -| Message | When | -|---|---| -| `state before start: {stream}` | Source emitted `source_state` for a stream before `stream_status: start` | +| Message | When | +| -------------------------------- | -------------------------------------------------------------------------- | +| `state before start: {stream}` | Source emitted `source_state` for a stream before `stream_status: start` | | `state after complete: {stream}` | Source emitted `source_state` for a stream after `stream_status: complete` | -| `duplicate start: {stream}` | Source emitted `stream_status: start` for a stream that already started | -| `unknown stream: {stream}` | Source emitted a message for a stream not in the catalog | +| `duplicate start: {stream}` | Source emitted `stream_status: start` for a stream that already started | +| `unknown stream: {stream}` | Source emitted a message for a stream not in the catalog | ### error -| Message | When | -|---|---| -| `global error: {message}` | Source emitted `error_level: global` β€” sync aborted | +| Message | When | +| ----------------------------------- | ----------------------------------------------------- | +| `global error: {message}` | Source emitted `error_level: global` β€” sync aborted | | `stream error: {stream}: {message}` | Source emitted `error_level: stream` β€” stream skipped | -| `source crashed: {message}` | Source iterator threw an exception | +| `source crashed: {message}` | Source iterator threw an exception | --- From b4a883d5faa59142343ddc7e4ab7d6dd5d509317 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:33:21 +0000 Subject: [PATCH 22/32] docs: flatten ConfiguredStream, remove nested stream object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConfiguredStream.stream.name β†’ ConfiguredStream.name. No more nesting of Stream inside ConfiguredStream. TODO: move metadata (api_version, account_id, live_mode) out of per-stream into source_config or destination injection. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index c939c612..6410e3c1 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -196,12 +196,9 @@ before passing to the source. ```ts type ConfiguredStream = { - stream: { - name: string // e.g. "customers", "invoices" - primary_key: string[][] // e.g. [["id"]] - json_schema?: Record - metadata?: Record // e.g. { api_version, account_id, live_mode } - } + name: string // e.g. "customers", "invoices" + primary_key: string[][] // e.g. [["id"]] + json_schema?: Record sync_mode: 'full_refresh' | 'incremental' destination_sync_mode: 'append' | 'overwrite' | 'append_dedup' cursor_field?: string[] @@ -213,6 +210,9 @@ type ConfiguredStream = { lt: string // exclusive upper bound (ISO 8601) } } +// TODO: metadata (api_version, account_id, live_mode) currently lives on +// Stream.metadata. It should move to source_config or be injected by the +// destination β€” it's per-source, not per-stream. type ConfiguredCatalog = { streams: ConfiguredStream[] From b4d3281a914c26adc3fb75ef9218e981a06a5088 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:35:41 +0000 Subject: [PATCH 23/32] =?UTF-8?q?docs:=20rename=20state=20=E2=86=92=20star?= =?UTF-8?q?ting=5Fstate/ending=5Fstate,=20clarify=20reducer=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - StartPayload.state β†’ starting_state - EndPayload.state β†’ ending_state - Round-trip is self-documenting: end.ending_state β†’ start.starting_state - Progress: client generally doesn't need a reducer, but can diff for deltas Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 6410e3c1..66292a35 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -57,9 +57,9 @@ Begins or continues a sync run. See [Types](#types) for `StartPayload`. The run is done. See [Types](#types) for `EndPayload`. -`has_more: true` β€” send another `start` with the same `sync_run_id` and the -returned `state`. `has_more: false` β€” this run is complete; use a new -`sync_run_id` for the next sync. +`has_more: true` β€” send another `start` with the same `sync_run_id` and +`ending_state` as the next `starting_state`. `has_more: false` β€” this run is +complete; use a new `sync_run_id` for the next sync. ### Source β†’ engine @@ -227,7 +227,7 @@ type StartPayload = { source_config: Record // source-specific (e.g. Stripe API key, account) destination_config: Record // destination-specific (e.g. Postgres connection) configured_catalog: ConfiguredCatalog - state?: SyncState // from previous end; omit on first sync + starting_state?: SyncState // from previous end.ending_state; omit on first sync } ``` @@ -236,16 +236,17 @@ type StartPayload = { ```ts type EndPayload = { has_more: boolean - state: SyncState // round-trip into next start - request: ProgressPayload // stats for this request only (same shape as progress) + ending_state: SyncState // round-trip into next start.starting_state + request_progress: ProgressPayload // stats for this request only } ``` ### Progress message (engine β†’ client) Emitted on every `source_state` checkpoint and `stream_status` change. Each -message is a complete snapshot of run-level progress β€” the client never needs -a reducer. +message is a complete snapshot of run-level progress β€” the client generally +doesn't need a reducer. To get real-time deltas, the client can diff two +consecutive progress messages. ```ts // Errors are a discriminated union on error_level. @@ -401,7 +402,7 @@ A sync run is identified by `sync_run_id`. Within a run, the upper time bound ### Continuation -1. Client sends `start` with the same `sync_run_id` and `state` from previous `end`. +1. Client sends `start` with the same `sync_run_id` and `starting_state` from previous `end.ending_state`. 2. Engine sees same ID β€” preserves `started_at` from engine state. 3. Engine sets the same `time_range` (same frozen upper bound). 4. Source resumes from its cursor state within the same range. @@ -574,7 +575,7 @@ NDJSON. One message per line. ← {"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[],"record_count":2500,"state_count":1}},"errors":[]}} ← {"type":"record","record":{"stream":"customers","data":{...}}} ← {"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"record_count":5000,"state_count":2}},"errors":[]}} -← {"type":"end","has_more":false,"state":{"source":{...},"engine":{...}}} +← {"type":"end","has_more":false,"ending_state":{"source":{...},"engine":{...}}} ``` Over HTTP: POST with NDJSON body (one `start` line), NDJSON response stream. @@ -594,9 +595,9 @@ do { source_config, destination_config, configured_catalog, - state, + starting_state: state, }) - state = end.state + state = end.ending_state } while (end.has_more) // Backfill complete. Schedule next sync with a new sync_run_id. From cd07ff1add71fb398707c8fc543135a90cf1b7c1 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:36:59 +0000 Subject: [PATCH 24/32] docs: add TODO for destination write_stats on StreamProgress Future: destination can report inserted/updated/deleted counts per stream (e.g. Postgres upsert). Extension point noted in StreamProgress. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 66292a35..abddc790 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -259,6 +259,8 @@ type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges record_count: number // records this run (across requests) state_count: number // checkpoints this run for this stream + // TODO: destination can enrich with write_stats: { inserted, updated, deleted } + // when supported (e.g. Postgres upsert knows which rows were new vs changed). } type ProgressPayload = { From 5e6b47008890200b87dcdd18176ca0ef0f553381 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:38:08 +0000 Subject: [PATCH 25/32] docs: add inserted/updated/deleted to StreamProgress All three are first-class fields. For now inserted = record_count, updated = 0, deleted = 0. Destination can report real values when it supports upsert/delete tracking. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index abddc790..97afcda4 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -259,8 +259,9 @@ type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges record_count: number // records this run (across requests) state_count: number // checkpoints this run for this stream - // TODO: destination can enrich with write_stats: { inserted, updated, deleted } - // when supported (e.g. Postgres upsert knows which rows were new vs changed). + inserted: number // records inserted (for now, same as record_count) + updated: number // records updated (0 until destination supports upsert tracking) + deleted: number // records deleted (0 until destination supports delete tracking) } type ProgressPayload = { From 6da7ecebad7c809c45e46af380d9facc14e8b174 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:51:57 +0000 Subject: [PATCH 26/32] docs: change_count + insert/update/delete breakdown on StreamProgress change_count = total records processed (always known by engine). insert_count + update_count + delete_count = change_count when destination reports the breakdown. All zero until then. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 97afcda4..ea9b45a1 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -257,11 +257,11 @@ type SyncError = type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges - record_count: number // records this run (across requests) state_count: number // checkpoints this run for this stream - inserted: number // records inserted (for now, same as record_count) - updated: number // records updated (0 until destination supports upsert tracking) - deleted: number // records deleted (0 until destination supports delete tracking) + change_count: number // total records processed (always known by engine) + insert_count: number // 0 until destination reports (insert_count + update_count + delete_count = change_count) + update_count: number // 0 until destination reports + delete_count: number // 0 until destination reports } type ProgressPayload = { From 82b7c664172d471a1b944fdf4afdd54955ca0408 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:53:20 +0000 Subject: [PATCH 27/32] docs: make change_count and insert/update/delete mutually exclusive Either change_count (when breakdown unavailable) or insert_count + update_count + delete_count (when destination reports). All optional. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index ea9b45a1..da08ef11 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -258,10 +258,12 @@ type SyncError = type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges state_count: number // checkpoints this run for this stream - change_count: number // total records processed (always known by engine) - insert_count: number // 0 until destination reports (insert_count + update_count + delete_count = change_count) - update_count: number // 0 until destination reports - delete_count: number // 0 until destination reports + // Either change_count (when breakdown unavailable) or the three counts (when + // destination reports). If breakdown is present, total = insert + update + delete. + change_count?: number // total records processed + insert_count?: number // records inserted + update_count?: number // records updated + delete_count?: number // records deleted } type ProgressPayload = { From 805f28f47733a2a6e133d3d2e6dc9358cb4b1e05 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:53:47 +0000 Subject: [PATCH 28/32] docs: rename change_count back to record_count It's a count of record messages, not an interpretation of what they mean. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index da08ef11..22f5d923 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -258,9 +258,9 @@ type SyncError = type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges state_count: number // checkpoints this run for this stream - // Either change_count (when breakdown unavailable) or the three counts (when + // Either record_count (when breakdown unavailable) or the three counts (when // destination reports). If breakdown is present, total = insert + update + delete. - change_count?: number // total records processed + record_count?: number // record messages processed insert_count?: number // records inserted update_count?: number // records updated delete_count?: number // records deleted From c7baa0a0f4991db698315abfbb6936b23f7ec9e8 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:54:05 +0000 Subject: [PATCH 29/32] docs: make record_count required, breakdown optional record_count is always known (engine counts record messages). insert/update/delete are optional enrichment from the destination. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 22f5d923..84a324e6 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -258,9 +258,8 @@ type SyncError = type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges state_count: number // checkpoints this run for this stream - // Either record_count (when breakdown unavailable) or the three counts (when - // destination reports). If breakdown is present, total = insert + update + delete. - record_count?: number // record messages processed + record_count: number // record messages processed (always known by engine) + // Optional breakdown from destination. When present, insert + update + delete = record_count. insert_count?: number // records inserted update_count?: number // records updated delete_count?: number // records deleted From 7a5c136e67e79d284a84279f7a813522613cf7af Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 06:54:25 +0000 Subject: [PATCH 30/32] docs: clarify insert/update/delete as future destination feature Reserved for when destinations report per-operation counts. Not implemented yet. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index 84a324e6..b14880d7 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -259,10 +259,12 @@ type StreamProgress = { completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges state_count: number // checkpoints this run for this stream record_count: number // record messages processed (always known by engine) - // Optional breakdown from destination. When present, insert + update + delete = record_count. - insert_count?: number // records inserted - update_count?: number // records updated - delete_count?: number // records deleted + // Future: destination reports breakdown. Not implemented yet β€” reserved for + // when destinations (e.g. Postgres) report per-operation counts. + // When present, insert_count + update_count + delete_count = record_count. + insert_count?: number + update_count?: number + delete_count?: number } type ProgressPayload = { From b4c3cf0b719c9ea4a4c01a41e6a77d30d5f3deff Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 07:27:53 +0000 Subject: [PATCH 31/32] docs: tighten sync lifecycle protocol Committed-By-Agent: codex Co-authored-by: codex --- docs/engine/sync-lifecycle-source-stripe.md | 376 +++++--------- docs/engine/sync-lifecycle.md | 548 ++++++++------------ 2 files changed, 335 insertions(+), 589 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index ae474332..4665f3a4 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -1,229 +1,174 @@ # Sync Lifecycle β€” Stripe Source -How the Stripe source manages pagination within a `time_range` assigned by the -engine. For the overall sync lifecycle and protocol, see -[sync-lifecycle.md](./sync-lifecycle.md). +How the Stripe source paginates finite backfills within the lifecycle described +in [sync-lifecycle.md](./sync-lifecycle.md). ## Overview -The engine assigns a `time_range` per stream via the configured catalog. The -Stripe source paginates all records within that range using an n-ary search -algorithm: start with the full range, paginate, and subdivide if the range -is too large to complete. No upfront density probing β€” the source discovers -the right granularity by doing the work. +Stripe list pagination is resumable because the API accepts an object-ID cursor: -## Source State - -```ts -type StripeStreamState = { - remaining: Array<{ - gte: string // ISO 8601 β€” inclusive lower bound - lt: string // ISO 8601 β€” exclusive upper bound - cursor: string | null // Stripe pagination cursor; null = not yet started - }> -} +```http +GET /v1/customers?limit=100&starting_after=cus_123 ``` -- `cursor: null` β†’ range planned but first page not yet fetched. -- `cursor: "cus_abc"` β†’ resume pagination after this object. -- Range removed from list β†’ complete. -- `remaining: []` β†’ source is done with the assigned `time_range`. +For the same endpoint and same filter set, `starting_after` means "continue +after this object in the current list order." -## Algorithm +Stripe list endpoints return objects in descending `created` order. That makes +`starting_after` sufficient for continuation across requests: -### 1. Initialization (no existing state) +- The source stores the last emitted object ID as `starting_after`. +- On the next request, it replays the same query shape and resumes from that ID. +- For streams that support `created` filtering, the engine can also assign a + fixed `time_range`. -The source receives `time_range` from the catalog and has no state. It starts -with the full range as a single entry: +This design does **not** split partially paginated ranges across requests. A +time-range stream resumes the same assigned range until it emits `complete`. -``` -Engine assigns: time_range { gte: "2018-01-01", lt: "2024-04-17" } +## Two Stream Modes -state: { - remaining: [ - { gte: "2018-01-01", lt: "2024-04-17", cursor: null } - ] -} -``` +### Time-range streams -### 2. Pagination +These streams support Stripe `created[gte]` / `created[lt]` filters. -The source picks a range from `remaining` and paginates it: +- The engine injects `time_range`. +- The source paginates with `created[...]` plus `starting_after`. +- The source may emit `range_complete` when the assigned range is fully read. +- The source emits `complete` when the stream is terminal for the run. -1. Call the Stripe list API with `created[gte]` and `created[lt]` filters, - plus `starting_after` if cursor is set. -2. Emit records. -3. Update cursor in state, emit `source_state`. -4. When a range is exhausted (`has_more: false`), remove it from `remaining`. +### Non-time-range streams -``` -First page fetched, got cursor: +These streams do not support `created` filtering. -state: { - remaining: [ - { gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_abc" } - ] -} -β†’ emit source_state +- The engine does not inject `time_range`. +- The source paginates with `starting_after` only. +- There is no range coverage accounting. +- The source emits `complete` when the stream is terminal for the run. -Pagination exhausted, range complete: +Not every Stripe endpoint supports every pagination feature. Streams only enter +the time-range path if the endpoint supports the necessary `created` filters. -state: { - remaining: [] +## Source State + +Stripe source state is opaque to the engine. The minimal per-stream form is: + +```ts +type StripeStreamState = { + starting_after: string | null } -β†’ emit source_state (done) ``` -### 3. Subdivision (n-ary search) +- `starting_after: null` means "start from the first page". +- `starting_after: "cus_abc"` means "resume after object `cus_abc`". -If a range didn't complete in the previous request, the source subdivides it -at the start of the next request. The source knows the `created` timestamp of -the last record it paginated (from the cursor). It splits the unpaginated -portion into N parts (where N = `max_segments_per_stream`): +The assigned `time_range`, when present, lives in the configured catalog. It is +not inferred from source state. -``` -Previous request ended with: - remaining: [{ gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_xyz" }] - -Last record seen had created=2020-06-15. Range didn't complete β†’ subdivide. -The paginated portion [2018, 2020-06-15) keeps its cursor. -The unpaginated portion [2020-06-15, 2024-04-17) splits into N=2: - - remaining: [ - { gte: "2018-01-01", lt: "2020-06-15", cursor: "cus_xyz" }, - { gte: "2020-06-15", lt: "2022-05-16", cursor: null }, - { gte: "2022-05-16", lt: "2024-04-17", cursor: null } - ] -``` +## Pagination Algorithm -**When to subdivide:** At the start of a request, if any range in `remaining` -has a cursor (meaning it was in progress last request but didn't complete). -Subdivision happens between requests, not mid-request. +### 1. Initialization -**Recursive:** If a subdivided range still doesn't complete in one request, -it gets split again next time. Each pass narrows the ranges until they're -small enough to complete in a single request. +If there is no saved state: -### 4. Resumption (existing state) +- For a time-range stream, the source receives `time_range` from the engine and + starts with `starting_after: null`. +- For a non-time-range stream, the source starts with `starting_after: null`. -If the source has existing state (from a previous request in the same sync -run), it resumes directly from `remaining`: +### 2. Page fetch -``` -Source receives time_range { gte: "2018-01-01", lt: "2024-04-17" } -Existing state: { - remaining: [ - { gte: "2022-05-16", lt: "2024-04-17", cursor: "cus_xyz" } - ] -} - -β†’ Resume paginating from cus_xyz in [2022-05-16, 2024-04-17) -β†’ No re-initialization -``` +For each page: -### 5. Completion +1. Build request params. +2. Call the Stripe list endpoint. +3. Emit records in the order returned by Stripe. +4. Save the last emitted object ID as `starting_after`. +5. Emit `source_state`. -When a sub-range is exhausted, the source removes it from `remaining` and -emits a `stream_status: range_complete`: +Time-range example: +```http +GET /v1/customers?limit=100&created[gte]=1514764800&created[lt]=1713312000 +GET /v1/customers?limit=100&created[gte]=1514764800&created[lt]=1713312000&starting_after=cus_100 ``` -β†’ emit trace { stream_status: { stream: 'customers', status: 'range_complete', - range_complete: { gte: '2018-01-01', lt: '2019-06-01' } } } + +Non-time-range example: + +```http +GET /v1/reporting/report_types?limit=100 +GET /v1/reporting/report_types?limit=100&starting_after=rpt_100 ``` -The engine merges this into `completed_ranges`. +### 3. Resumption -When all sub-ranges are done (`remaining: []`), the source emits -`stream_status: complete` for the stream. +On the next request in the same sync run: -## Full Example +- The engine reuses the same `sync_run_id`. +- For time-range streams, the engine re-injects the same fixed `time_range`. +- The source loads `starting_after` from source state. +- The source resumes the exact same query shape with that `starting_after`. -Shows the messages emitted by the source during a two-request backfill of -`customers` with `time_range: [2018, 2024)`. +This works because `starting_after` is an object ID in Stripe's stable list +ordering. It is a resume token for pagination, not a derived time boundary. -### Request 1 β€” full range, doesn't complete +### 4. Completion -Stripe returns max 100 records per page. Each page = 1 API request = 1 state -checkpoint. +When Stripe returns `has_more: false` for the current stream: -``` -Source initializes: remaining: [{ gte: "2018", lt: "2024", cursor: null }] +1. The source emits a final `source_state`. +2. If the stream had an assigned `time_range`, the source may emit + `stream_status: range_complete` for that range. +3. The source emits `stream_status: complete`. -← trace { stream_status: { stream: "customers", status: "start" } } -← record { stream: "customers", data: { id: "cus_001", ... } } - ... 100 records (page 1) ... -← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_100" }] } } -← record { stream: "customers", data: { ... } } - ... 100 records (page 2) ... -← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_200" }] } } - ... pages 3-50 (5000 records total) ... -← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] } } - ... source cut off (time limit / state limit) ... +`complete` is the terminal signal the engine trusts. `range_complete` is +progress telemetry only. -← end { has_more: true } -``` +## Message Examples -Range didn't complete in one request β†’ source will subdivide on next request. +### Time-range stream -### Request 2 β€” source subdivides, finishes first sub-range +Request 1: -``` -Source resumes, sees remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] -Last record had created=2019-03. Range didn't complete β†’ subdivide: - remaining: [ - { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // current (has cursor) - { gte: "2019-03", lt: "2021-09", cursor: null }, // new - { gte: "2021-09", lt: "2024", cursor: null } // new - ] - -← record { stream: "customers", data: { ... } } - ... 100 records (page) ... -← state { ... } - ... finishes [2018, 2019-03) after a few more pages ... -← trace { stream_status: { stream: "customers", status: "range_complete", - range_complete: { gte: "2018", lt: "2019-03" } } } -← state { stream: "customers", data: { remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: null }, - { gte: "2021-09", lt: "2024", cursor: null } - ] } } - ... starts [2019-03, 2021-09), paginates several pages ... -← state { stream: "customers", data: { remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, - { gte: "2021-09", lt: "2024", cursor: null } - ] } } - ... cut off ... +```text +Engine assigns: customers time_range [2018-01-01, 2024-04-17) +← trace { stream_status: { stream: "customers", status: "started" } } +← record { stream: "customers", data: { id: "cus_001", ... } } +← state { stream: "customers", data: { starting_after: "cus_100" } } +← record { stream: "customers", data: { id: "cus_101", ... } } +← state { stream: "customers", data: { starting_after: "cus_200" } } +... cut off ... ← end { has_more: true } ``` -### Request 3 β€” finishes remaining ranges +Request 2: -``` -Source resumes: remaining: [ - { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, - { gte: "2021-09", lt: "2024", cursor: null } -] -These ranges made progress last request β€” no further subdivision, resume. +```text +Engine reassigns the same customers time_range [2018-01-01, 2024-04-17) +Source resumes with starting_after = "cus_200" - ... paginates [2019-03, 2021-09) page by page ... +← record { stream: "customers", data: { id: "cus_201", ... } } +← state { stream: "customers", data: { starting_after: "cus_300" } } +... final page ... +← state { stream: "customers", data: { starting_after: "cus_5421" } } ← trace { stream_status: { stream: "customers", status: "range_complete", - range_complete: { gte: "2019-03", lt: "2021-09" } } } - ... paginates [2021-09, 2024) page by page ... -← trace { stream_status: { stream: "customers", status: "range_complete", - range_complete: { gte: "2021-09", lt: "2024" } } } -← state { stream: "customers", data: { remaining: [] } } + range_complete: { gte: "2018-01-01T00:00:00Z", lt: "2024-04-17T00:00:00Z" } } } ← trace { stream_status: { stream: "customers", status: "complete" } } - ← end { has_more: false } ``` -Engine's `completed_ranges` for customers after merging all `range_complete` messages: -`[{ gte: "2018", lt: "2024" }]` +### Non-time-range stream -## State on the Wire +```text +No time_range assigned -Source state is opaque to the engine. The engine learns about range completion -via `stream_status: range_complete` messages, not by inspecting source state: +← trace { stream_status: { stream: "reporting_report_types", status: "started" } } +← record { stream: "reporting_report_types", data: { id: "rpt_001", ... } } +← state { stream: "reporting_report_types", data: { starting_after: "rpt_001" } } +... final page ... +← trace { stream_status: { stream: "reporting_report_types", status: "complete" } } +``` + +## State on the Wire ```ts { @@ -231,96 +176,37 @@ via `stream_status: range_complete` messages, not by inspecting source state: source_state: { state_type: 'stream', stream: 'customers', - time_range: { gte: '2018-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' }, data: { - remaining: [ - { gte: '2022-05-16T00:00:00Z', lt: '2024-04-17T00:00:00Z', cursor: 'cus_xyz' } - ] + starting_after: 'cus_200' } } } ``` -## Concurrency - -Three controls govern how the source uses the Stripe API: - -```ts -// Source config β€” only max_concurrent_streams is user-configurable -type StripeSourceConfig = { - api_key: string - account_id?: string - max_concurrent_streams?: number // default 5 -} - -// Derived internally by the source: -// live_mode = inferred from api_key prefix (sk_live_ vs sk_test_) -// max_requests_per_second = live_mode ? 20 : 10 -// effective_streams = min(max_concurrent_streams, configured_stream_count) -// max_segments_per_stream = floor(max_requests_per_second / effective_streams) -``` - -| Control | What it controls | How it's set | -| ------------------------- | -------------------------------------------- | ------------------------------------------ | -| `max_concurrent_streams` | Streams paginating in parallel | Config (default 5), capped at catalog size | -| `max_requests_per_second` | Global rate limit across all activity | Inferred from API key mode | -| `max_segments_per_stream` | Sub-ranges per stream (n-ary search fan-out) | Derived: rps / concurrent streams | - -### Examples - -| Scenario | Mode | Streams | `effective_streams` | `rps` | `max_segments_per_stream` | Max concurrent requests | -| ---------------- | ---- | ------- | ------------------- | ----- | ------------------------- | ----------------------- | -| 20 streams, live | live | 20 | 5 | 20 | 4 | 20 | -| 20 streams, test | test | 20 | 5 | 10 | 2 | 10 | -| 3 streams, live | live | 3 | 3 | 20 | 6 | 18 | -| 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | -| 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | - -When fewer streams are configured, each stream gets more segments β€” the full -rate limit budget is distributed across whatever streams exist. A single-stream -sync gets the entire budget. +The engine persists this state opaquely and passes it back on continuation. It +does not inspect `starting_after`. -## Parallel Pagination - -The source paginates up to `max_segments_per_stream` ranges from `remaining` -concurrently per stream, and up to `effective_streams` streams in parallel. -Records from different ranges/streams are interleaved on the output stream. -State checkpoints are emitted after each page, reflecting the current state -of all ranges. This ensures resumability if the source is cut off mid-run. - -The global rate limiter (`max_requests_per_second`) governs all API calls -regardless of which stream or segment they belong to. - -## Source Logs - -The Stripe source emits `log` messages for real-time operational visibility. -These are passed through by the engine. +## Error Handling -| Level | Message | When | -|---|---|---| -| info | `{stream}: {rps} requests/sec` | Periodically during pagination | -| warn | `rate limited: retrying in {n}s` | Stripe returned 429 | -| warn | `retry {n}/{max}: {status} {message}` | Request failed, retrying | +- **Transient errors**: retry at the HTTP layer and emit a `transient` error + trace for observability. +- **Stream errors**: emit a `stream` error trace, stop this stream, then emit + `complete` for explicit terminality. +- **Global errors**: emit a `global` error trace and stop the sync. -## Error Handling +The source does not encode error semantics into source state. -- **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP - layer with exponential backoff. Emit a `transient` error trace for - observability regardless of whether the retry succeeded. -- **Stream errors** (resource not available, permission denied): Emit a - `stream` error trace, stop this stream, move to the next. -- **Global errors** (invalid API key): Emit a `global` error trace, stop. +## Exclusions -The source does not store error state. If a range fails after all retries, -the range stays in `remaining` with its cursor for the next attempt. +This lifecycle doc does not cover: -## Events +- live `/events` polling +- cross-request range subdivision +- `ending_before`-driven reverse scans +- `full_refresh` semantics -The `/events` endpoint is treated as just another stream in the catalog β€” -same `time_range` model, same `remaining`-based pagination. No special -incremental mode or live polling by default. +In protocol terms, Stripe backfill now explicitly removes: -For experimental live event polling (using events as a webhook replacement), -an opt-in flag stores cursor state in `source.global`, which is completely -separate from the per-stream backfill cursor logic. This is not enabled by -default. +- using `starting_after` to derive new time boundaries between requests +- requiring every stream to support `time_range` +- using `range_complete` to decide terminality diff --git a/docs/engine/sync-lifecycle.md b/docs/engine/sync-lifecycle.md index b14880d7..201b7641 100644 --- a/docs/engine/sync-lifecycle.md +++ b/docs/engine/sync-lifecycle.md @@ -1,27 +1,50 @@ # Sync Lifecycle -How sync runs work: run identity, state ownership, time ranges, and stall -detection. For message types and connector interfaces, see [protocol.md](./protocol.md). +How finite sync runs work: run identity, opaque state, optional time ranges, and +terminal stream status. For message types and connector interfaces, see +[protocol.md](./protocol.md). + +## Scope + +This design is intentionally narrow: + +- Incremental backfills only. +- Finite reads only. +- `full_refresh` is out of scope. +- Live `/events` polling is out of scope. +- Generic stall detection is out of scope. + +## Removed From This Protocol + +To keep lifecycle semantics tight, this protocol explicitly removes these ideas: + +- **No `full_refresh` lifecycle.** `sync_mode: 'full_refresh'` and + `destination_sync_mode: 'overwrite'` are not part of this protocol. They need + separate semantics because "done for this run" and "historical coverage" mean + different things for a full reread. +- **No `range_complete`-driven terminality.** `range_complete` remains optional + progress telemetry only. It does not drive `has_more`. +- **No cross-request range subdivision in the protocol.** The protocol does not + assume that a partially paginated time range can be split into smaller ranges + between requests. ## Motivation The base protocol treats each `read()` call as independent. The caller manages -pagination (via `state_limit`), upper bounds, and retry policy externally. This -creates several problems: - -1. **Backfill bounds shift between calls.** Each call computes `now()` as the - upper bound, so high-volume accounts chase a moving target and never converge. -2. **No run identity.** Multiple calls that form one logical backfill have no - shared context. The engine cannot distinguish "continuation" from "new sync." -3. **Source owns retry policy.** Error types are baked into source state, mixing - cursor data with skip-on-resume decisions. -4. **Engine duplicates source bookkeeping.** Stream status and errors are tracked - independently by source and engine, with divergent representations. -5. **Stalled streams are invisible.** A non-incremental stream that restarts - every run blocks other streams with no detection or mitigation. - -This design introduces **sync runs** as a first-class concept and moves time -range management, stream lifecycle guarantees, and stall detection to the engine. +pagination, upper bounds, and continuation externally. That creates three +problems: + +1. **Backfill bounds shift between calls.** A stream that derives its own upper + bound from `now()` can chase a moving target forever. +2. **No run identity.** Multiple requests that belong to one logical backfill + have no shared context. +3. **Completion is ambiguous.** If the engine inspects source-specific state to + guess whether a stream is done, protocol behavior depends on connector + internals instead of explicit source signals. + +This design introduces **sync runs** as a first-class concept. The engine owns +run identity and optional outer time bounds. The source owns pagination and +emits explicit lifecycle signals. --- @@ -31,19 +54,31 @@ range management, stream lifecycle guarantees, and stall detection to the engine CLIENT ←—start/endβ€”β†’ ENGINE ←—iteratorβ€”β†’ SOURCE ``` -| Concern | Client | Engine | Source | -| ------------------------- | --------------------- | ------------------------------------------ | -------------------------------------- | -| What to sync (streams) | Provides catalog | Adjusts catalog (time_range, deprioritize) | Syncs what it's given | -| When to sync (scheduling) | Decides | β€” | β€” | -| Run identity | Generates sync_run_id | Freezes bounds, tracks continuations | Unaware | -| Time range bounds | β€” | Computes, injects into catalog | Respects `time_range` if present | -| Internal pagination | β€” | β€” | Manages ranges, parallel pages | -| Stream lifecycle | Consumes progress | Guarantees terminal status | Emits `started`, optionally `complete` | -| Progress reporting | Consumes | Enriches source signals, emits progress | Emits raw stream_status + records | -| Error reporting | Decides retry policy | Passes through, tracks for stalls | Emits trace errors | -| State | Opaque round-trip | Manages engine section | Manages source section | -| Stall detection | β€” | Tracks per-stream across runs | β€” | -| `has_more` | Reads, acts | Derives from source + engine state | β€” | +| Concern | Client | Engine | Source | +| --- | --- | --- | --- | +| What to sync | Provides catalog | Passes catalog through, may inject `time_range` | Syncs what it's given | +| When to sync | Decides | β€” | β€” | +| Run identity | Generates `sync_run_id` | Tracks run continuity | Unaware | +| Time range bounds | β€” | Freezes `started_at`, injects `time_range` when supported | Respects `time_range` if present | +| Internal pagination | β€” | β€” | Manages `starting_after` / equivalent | +| Stream lifecycle | Consumes | Tracks terminal streams | Emits `started`, optional `range_complete`, `complete` | +| Progress reporting | Consumes | Emits run-level snapshots | Emits records, checkpoints, traces | +| Error reporting | Decides retry policy above the engine | Passes through, stops on `global` | Classifies and emits trace errors | +| State | Opaque round-trip | Manages engine section | Manages source section | +| `has_more` | Reads, acts | Derives from explicit terminal stream state | β€” | + +--- + +## Core Rule + +The engine trusts only explicit stream status messages for lifecycle: + +- `started` means the stream is active for this request. +- `range_complete` is progress telemetry only. +- `complete` is the only terminal signal. + +The engine does **not** inspect source state to infer completion. Source state is +opaque cursor data. --- @@ -55,11 +90,15 @@ Begins or continues a sync run. See [Types](#types) for `StartPayload`. ### `end` β€” engine β†’ client -The run is done. See [Types](#types) for `EndPayload`. +The request is done. See [Types](#types) for `EndPayload`. -`has_more: true` β€” send another `start` with the same `sync_run_id` and -`ending_state` as the next `starting_state`. `has_more: false` β€” this run is -complete; use a new `sync_run_id` for the next sync. +`has_more: true` means at least one configured stream has not emitted +`stream_status: complete` for this run yet. Continue by sending another `start` +with the same `sync_run_id` and the previous `ending_state` as the next +`starting_state`. + +`has_more: false` means every configured stream is terminal for this run. The +next sync should use a new `sync_run_id`. ### Source β†’ engine @@ -69,21 +108,16 @@ Sources are iterators that yield these message types: // Data record { type: 'record', record: { stream: string, data: Record, emitted_at: string } } -// Checkpoint (per-stream β€” most common). Data is opaque to the engine. +// Checkpoint. Data is opaque to the engine. { type: 'source_state', source_state: { state_type: 'stream', stream: string, data: unknown } } -// Checkpoint (global β€” e.g. events cursor shared across all streams) +// Global checkpoint for source-wide state. { type: 'source_state', source_state: { state_type: 'global', data: unknown } } -// Stream status β€” discriminated union on status (Stripe polymorphism pattern) +// Stream status { type: 'trace', trace: { trace_type: 'stream_status', stream_status: StreamStatus } } -// where StreamStatus is: -// { stream: string, status: 'start' } -// { stream: string, status: 'range_complete', range_complete: { gte: string, lt: string } } -// { stream: string, status: 'complete' } - -// Error β€” discriminated union on error_level (see Error Handling) +// Error { type: 'trace', trace: { trace_type: 'error', error: SyncError & { stack_trace?: string } } } // Diagnostic log @@ -95,25 +129,11 @@ Sources are iterators that yield these message types: The engine emits four message types: `progress`, `record`, `log`, and `end`. ```ts -// Progress β€” emitted on every source_state checkpoint and stream_status change. -// -// Each message is a complete run-level snapshot, not a delta. Run-level -// totals ("45K customers synced across 3 requests") are what clients -// typically display. Point-in-time rates are derivable by diffing two -// consecutive snapshots β€” the common case is served directly, the rare -// case is still easy. -// -// Errors are included for the same reason: separating them into their own -// event stream would force every client to accumulate errors alongside -// progress snapshots, defeating the single-message-renders-everything model. -// -// All counts are cumulative since the start of the run (across requests with -// the same sync_run_id). Client can diff consecutive messages for deltas. { type: 'progress', progress: { - elapsed_ms: number, // wall-clock since run started (across all requests) - global_state_count: number, // total checkpoints this run (all streams) + elapsed_ms: number, + global_state_count: number, derived: { records_per_second: number, states_per_second: number, @@ -123,67 +143,48 @@ The engine emits four message types: `progress`, `record`, `log`, and `end`. } } -// Records β€” passed through from source { type: 'record', record: { stream: string, data: Record, emitted_at: string } } -// Log β€” engine operational messages (see Engine Logs below) { type: 'log', log: { level: 'info' | 'warn' | 'error', message: string } } -// Terminal β€” this request is done. -// end.request has the same shape as progress but scoped to this request only. { type: 'end', end: { has_more: boolean, - state: SyncState, - request_progress: ProgressPayload, // same shape, scoped to this request + ending_state: SyncState, + request_progress: ProgressPayload, } } ``` -`ProgressPayload` is used in two places with different scopes: - -| Scope | Where to find it | -| -------------------------- | -------------------------------------------------- | -| Between two progress msgs | Client diffs consecutive `progress` messages | -| This request | `end.request_progress` (ProgressPayload) | -| This run (across requests) | Latest `progress` message (ProgressPayload) | -| All time (across runs) | Sum of `completed_ranges` coverage + record counts | - -The engine does NOT emit trace messages to the client. Source errors are -included inside `progress`. Source traces and logs are consumed by the engine -and distilled into `progress` and `log` messages. +The engine does not pass trace messages through to the client. It folds them +into `progress` and `log`. --- ## Stream Status -`stream_status` is a discriminated union on `status` (Stripe polymorphism -pattern β€” the status value names the payload key): +`stream_status` is a discriminated union on `status`: ```ts type StreamStatus = - | { stream: string; status: 'start' } + | { stream: string; status: 'started' } | { stream: string; status: 'range_complete'; range_complete: { gte: string; lt: string } } | { stream: string; status: 'complete' } ``` -| Status | Emitted by | Engine action | -| ---------------- | ----------------- | ----------------------------------------------------------------- | -| `start` | Source | Stream is active | -| `range_complete` | Source | Merge range into `completed_ranges` | -| `complete` | Source (optional) | Stream is done; engine derives this if source exhausts without it | +| Status | Meaning | Engine action | +| --- | --- | --- | +| `started` | Stream is active | Mark stream active for progress | +| `range_complete` | A time range finished | Update progress only | +| `complete` | Stream is terminal for this run | Mark stream terminal | -A stream's backfill is done when `completed_ranges` covers the full range -`[0, started_at)`. +`range_complete` is optional and only meaningful for streams that support +engine-assigned `time_range`. It is not used to derive `has_more`. -The source manages sub-ranges internally β€” the engine doesn't see or track -them. The engine learns about completed ranges via `range_complete` status -messages. - -Errors are orthogonal to lifecycle. A stream can be `complete` with errors -(some pages failed but the stream moved past them) or `incomplete` without -errors (time limit hit mid-stream). +A source that decides to stop a stream after a stream-level error should still +emit `complete` for that stream. That keeps lifecycle semantics explicit: +errors explain *why* the stream stopped; `complete` says it is terminal. --- @@ -191,28 +192,28 @@ errors (time limit hit mid-stream). ### Configured catalog (client β†’ engine β†’ source) -The client provides the catalog. The engine adjusts it (injects `time_range`) -before passing to the source. +The client provides the catalog. The engine may inject `time_range` into +streams that support it. ```ts type ConfiguredStream = { - name: string // e.g. "customers", "invoices" - primary_key: string[][] // e.g. [["id"]] + name: string + primary_key: string[][] json_schema?: Record - sync_mode: 'full_refresh' | 'incremental' - destination_sync_mode: 'append' | 'overwrite' | 'append_dedup' + sync_mode: 'incremental' + destination_sync_mode: 'append' | 'append_dedup' cursor_field?: string[] - backfill_limit?: number // cap backfill to N records + backfill_limit?: number + + // Source capability from discover/spec. + supports_time_range?: boolean - // Set by engine, not client + // Set by engine only when supports_time_range is true. time_range?: { - gte?: string // inclusive lower bound (ISO 8601); omit for "from the beginning" - lt: string // exclusive upper bound (ISO 8601) + gte?: string + lt: string } } -// TODO: metadata (api_version, account_id, live_mode) currently lives on -// Stream.metadata. It should move to source_config or be injected by the -// destination β€” it's per-source, not per-stream. type ConfiguredCatalog = { streams: ConfiguredStream[] @@ -223,11 +224,11 @@ type ConfiguredCatalog = { ```ts type StartPayload = { - sync_run_id: string // client-generated UUID - source_config: Record // source-specific (e.g. Stripe API key, account) - destination_config: Record // destination-specific (e.g. Postgres connection) + sync_run_id: string + source_config: Record + destination_config: Record configured_catalog: ConfiguredCatalog - starting_state?: SyncState // from previous end.ending_state; omit on first sync + starting_state?: SyncState } ``` @@ -236,266 +237,154 @@ type StartPayload = { ```ts type EndPayload = { has_more: boolean - ending_state: SyncState // round-trip into next start.starting_state - request_progress: ProgressPayload // stats for this request only + ending_state: SyncState + request_progress: ProgressPayload } ``` ### Progress message (engine β†’ client) -Emitted on every `source_state` checkpoint and `stream_status` change. Each -message is a complete snapshot of run-level progress β€” the client generally -doesn't need a reducer. To get real-time deltas, the client can diff two -consecutive progress messages. - ```ts -// Errors are a discriminated union on error_level. type SyncError = | { error_level: 'global'; message: string } | { error_level: 'stream'; message: string; stream: string } | { error_level: 'transient'; message: string; stream?: string } type StreamProgress = { - completed_ranges?: Array<{ gte: string; lt: string }> // merged completed time ranges - state_count: number // checkpoints this run for this stream - record_count: number // record messages processed (always known by engine) - // Future: destination reports breakdown. Not implemented yet β€” reserved for - // when destinations (e.g. Postgres) report per-operation counts. - // When present, insert_count + update_count + delete_count = record_count. - insert_count?: number - update_count?: number - delete_count?: number + state_count: number + record_count: number + completed_ranges?: Array<{ gte: string; lt: string }> + terminal: boolean } type ProgressPayload = { - elapsed_ms: number // wall-clock since run started (across requests) - global_state_count: number // total checkpoints this run (all streams) + elapsed_ms: number + global_state_count: number derived: { - // Computed from the sum of all stream record_counts / (elapsed_ms / 1000). - // Uses run-level totals, not windowed β€” so this is the average rate since - // the run started. A client that wants instantaneous rate can diff - // record_count between two consecutive progress messages and divide by - // the elapsed_ms delta. records_per_second: number - // Computed from global_state_count / (elapsed_ms / 1000). states_per_second: number } - streams: Record // keyed by stream name - errors: SyncError[] // all errors accumulated this run + streams: Record + errors: SyncError[] } ``` +`completed_ranges` is progress data only. It does not determine completion. + ### SyncState (round-tripped between start and end) ```ts type SyncState = { - source: SourceState // opaque to engine β€” cursor data - engine: EngineState // opaque to client β€” run progress + run identity + source: SourceState + engine: EngineState } type SourceState = { - streams: Record // per-stream cursor data, keyed by stream name - global: Record // source-wide data (e.g. events cursor) + streams: Record + global: Record } type EngineState = { - sync_run_id: string // current run ID - started_at: string // ISO 8601 β€” frozen snapshot upper bound - run_progress: ProgressPayload // accumulated run-level progress + sync_run_id: string + started_at: string + terminal_streams: string[] + run_progress: ProgressPayload } ``` ### Source state β€” Stripe example -Pure cursor data. No error types, no status field. The engine treats this as -opaque; the types below are source-internal. +Source state is opaque to the engine. For Stripe list endpoints, the source can +store the last emitted object ID as `starting_after`: ```ts -// Per-stream state (source-internal, opaque to engine) -// Minimal β€” just a pagination cursor. Range management is engine's job. type StripeStreamState = { - page_cursor: string | null // Stripe list pagination cursor -} -``` - -The source receives `time_range` from the catalog and paginates within it. -The engine tracks which ranges are complete and which need work via -`completed_ranges` and `pending_ranges` in engine state. - -**Example β€” two streams mid-sync:** - -```jsonc -{ - "source": { - "streams": { - "customers": { "page_cursor": "cus_xyz" }, - "invoices": { "page_cursor": null }, - }, - "global": { - "events_cursor": "2024-04-16T23:50:00Z", - }, - }, -} -``` - -### Engine state - -The engine state contains run identity and a `run_progress` field that is a -`ProgressPayload`. The client round-trips it opaquely. The engine uses it to -accumulate progress across requests within a run and to track completed ranges -across runs. - -```ts -type EngineState = { - sync_run_id: string // current run ID - started_at: string // ISO 8601 β€” frozen snapshot upper bound - run_progress: ProgressPayload // accumulated run-level progress + starting_after: string | null } ``` -**Example β€” customers fully synced, invoices mid-backfill, big_table stalled:** - -```jsonc -{ - "engine": { - "sync_run_id": "sr_abc", - "started_at": "2024-04-17T00:00:00Z", - "run_progress": { - "elapsed_ms": 8400, - "global_state_count": 24, - "derived": { "records_per_second": 5500, "states_per_second": 2.9 }, - "streams": { - "customers": { - "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2024-04-17T00:00:00Z" }], - "record_count": 45000, - "state_count": 16, - }, - "invoices": { - "completed_ranges": [{ "gte": "2018-01-01T00:00:00Z", "lt": "2021-06-01T00:00:00Z" }], - "record_count": 1200, - "state_count": 8, - }, - "big_table": { - "completed_ranges": [], - "record_count": 0, - "state_count": 0, - }, - }, - "errors": [], - }, - }, -} -``` +For time-range streams, the assigned `time_range` lives in the catalog, not in +source state. --- ## Sync Runs -A sync run is identified by `sync_run_id`. Within a run, the upper time bound -(`started_at`) is frozen. +A sync run is identified by `sync_run_id`. Within a run, `started_at` is frozen. ### New run 1. Client sends `start` with a new `sync_run_id`. 2. Engine freezes `started_at = now()` and stores it in engine state. -3. Engine computes `time_range` for each stream: - - Upper bound: `started_at` - - Lower bound: end of last `completed_ranges` entry (or account creation for - first backfill) -4. Engine injects `time_range` into configured catalog before passing to source. -5. Source syncs within the given range, yields messages, exhausts. -6. Engine emits progress, pipes records to destination, returns `end`. +3. For each configured stream where `supports_time_range` is true, the engine + injects `time_range.lt = started_at`. +4. Source runs, emits records, checkpoints, and explicit stream statuses. +5. Engine emits progress, forwards records to the destination, and returns + `end`. ### Continuation -1. Client sends `start` with the same `sync_run_id` and `starting_state` from previous `end.ending_state`. -2. Engine sees same ID β€” preserves `started_at` from engine state. -3. Engine sets the same `time_range` (same frozen upper bound). -4. Source resumes from its cursor state within the same range. +1. Client sends `start` with the same `sync_run_id` and previous `ending_state`. +2. Engine preserves `started_at` from engine state. +3. The engine re-injects the same `time_range` into streams that support it. +4. Source resumes from its opaque cursor state. ### Completion When `has_more: false`: -- All streams completed their ranges or were marked `incomplete`. -- Engine promotes completed `pending_range` entries to `completed_ranges`. -- Client should use a new `sync_run_id` for the next sync. +- Every configured stream is present in `engine.terminal_streams`. +- The client should start the next sync with a new `sync_run_id`. ### Example ``` sync_run_id: "sr_1" - request 1: customers [2018, 2024) β†’ timed out β†’ end { has_more: true } - request 2: customers [2018, 2021) β†’ completed - customers [2021, 2024) β†’ timed out β†’ end { has_more: true } - request 3: customers [2021, 2022.5) β†’ completed - customers [2022.5, 2024) β†’ completed β†’ end { has_more: false } - completed_ranges merges to [2018, 2024) βœ“ + request 1: customers [2018, 2024) β†’ timed out β†’ end { has_more: true } + request 2: customers [2018, 2024) β†’ complete β†’ end { has_more: false } ``` -Each run's upper bound is frozen. Ranges that don't complete get subdivided. -Completed adjacent ranges merge. +The range is stable across requests. The source resumes within that range using +its own cursor state. --- ## Time Ranges -Time is a first-class concept. The engine sets the outer bounds; the source -manages pagination and subdivision within them. - -### Flow +Time range support is optional per stream. -``` -Client catalog: { stream: "customers", sync_mode: "incremental" } - (no time_range β€” client doesn't set this) - ↓ -Engine sets range: { stream: "customers", sync_mode: "incremental", - time_range: { gte: "2021-01-01T00:00:00Z", lt: "2024-04-17T00:00:00Z" } } - (computed from completed_ranges + started_at) - ↓ -Source receives: time_range on configured stream. - Manages its own subdivision/parallelism within it. - Emits source_state with time_range to report range completion. -``` +### Streams with `supports_time_range: true` -### How the engine tracks completed ranges +- The engine injects `time_range`. +- `time_range.lt` is frozen to `started_at` for the duration of the run. +- The source resumes within that range using opaque source state. +- The source may emit `range_complete` for progress reporting. -The engine observes `source_state` messages. When a state message includes -`time_range` and has no remaining cursor, the engine knows that range is done -and adds it to `completed_ranges`, merging adjacent ranges: +### Streams with `supports_time_range: false` -``` -completed_ranges: [{ gte: "2018", lt: "2021" }] -+ source_state with time_range [2021, 2024), cursor: null -= completed_ranges: [{ gte: "2018", lt: "2024" }] (merged) -``` - -### Engine range computation - -On each request, the engine computes the `time_range` to assign: - -1. Upper bound: `started_at` (frozen for the run) -2. Lower bound: end of last `completed_ranges` entry (or account start for first backfill) -3. If `completed_ranges` has gaps, fill the first gap +- The engine does not inject `time_range`. +- The source paginates using its own cursor semantics only. +- No coverage accounting is implied. ### Why this matters -- **Frozen upper bounds.** `started_at` does not move within a run. -- **Adaptive parallelism.** Dense ranges get subdivided; sparse ranges complete in one shot. -- **Visibility.** Engine knows what fraction of history is synced. -- **Compact state.** Merged ranges keep state O(active ranges), not O(total history). +- Frozen upper bounds prevent moving-target backfills for eligible streams. +- Streams without time filtering still fit the same continuation contract. +- The engine never needs to understand source-specific pagination tokens. --- ## `has_more` Derivation -The engine derives `has_more` at end of run: +The engine derives `has_more` from explicit terminal stream state: +```ts +has_more = configured_catalog.streams.some( + (stream) => !engine.terminal_streams.includes(stream.name) +) ``` -has_more = true if any catalog stream where: - - source state has a page_cursor (mid-pagination), OR - - completed_ranges don't cover [0, started_at) -``` + +`completed_ranges` and source-state shape do not participate in this decision. --- @@ -503,70 +392,48 @@ has_more = true if any catalog stream where: ### Error levels -Errors carry their blast radius. The `error_level` determines the engine's action: - -| `error_level` | Blast radius | Engine action | Example | -| ------------- | ------------ | ------------------------------------ | ----------------------------------------- | -| `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key, bad source config | -| `stream` | One stream | Skip stream, continue others | Resource not available, permission denied | -| `transient` | One request | Informational | Rate limited, retried 3x in 4.2s | +| `error_level` | Blast radius | Engine action | Example | +| --- | --- | --- | --- | +| `global` | Entire sync | Abort all streams, `has_more: false` | Invalid API key | +| `stream` | One stream | Keep processing other streams | Resource unavailable | +| `transient` | One request or page | Informational | Rate limited, retried | ### Source β†’ engine error flow ```ts -// Source emits trace errors β€” discriminated union on error_level: { type: 'trace', trace: { trace_type: 'error', error: SyncError } } - -// Examples: -{ error: { error_level: 'global', message: 'Invalid API key' } } -{ error: { error_level: 'stream', message: 'Not available in test mode', stream: 'invoices' } } -{ error: { error_level: 'transient', message: 'Rate limited, retried 3x', stream: 'customers' } } ``` -The source decides the `error_level`: - -- **Transient**: Request failed and retried β€” emit for observability. -- **Stream**: Stream-level failure (e.g. resource not enabled) β€” skip stream. -- **Global**: Unrecoverable (e.g. invalid credentials) β€” stop everything. - ### Engine behavior -The engine accumulates errors into `progress.errors[]` and acts on them: +- `global`: stop immediately and emit `end { has_more: false }` +- `stream`: record the error and continue with other streams +- `transient`: record the error only -- **`global`**: Stop the source, emit `end { has_more: false }`. -- **`stream`**: Skip that stream, continue others. -- **`transient`**: No action. Included in `progress.errors` for observability. - -Errors are NOT stored in source state. Range-level concerns (subdivision, -retries, timeouts) are managed internally by the source. +Errors are not stored in source state. They are separate from lifecycle. --- ## Engine Logs -The engine emits `log` messages for anomalies and failures only. Normal -progress (stream starts, completions, record counts) is already in the -`progress` stream β€” no redundant info logs. - -The engine processes source messages tolerantly β€” it does not reject -unexpected ordering β€” but logs warnings so they're alertable in production. +The engine emits `log` messages for anomalies and failures only. ### warn -| Message | When | -| -------------------------------- | -------------------------------------------------------------------------- | -| `state before start: {stream}` | Source emitted `source_state` for a stream before `stream_status: start` | -| `state after complete: {stream}` | Source emitted `source_state` for a stream after `stream_status: complete` | -| `duplicate start: {stream}` | Source emitted `stream_status: start` for a stream that already started | -| `unknown stream: {stream}` | Source emitted a message for a stream not in the catalog | +| Message | When | +| --- | --- | +| `state before started: {stream}` | Source emitted `source_state` before `stream_status: started` | +| `state after complete: {stream}` | Source emitted `source_state` after `stream_status: complete` | +| `duplicate started: {stream}` | Source emitted `stream_status: started` twice | +| `unknown stream: {stream}` | Source emitted a message for a stream not in the catalog | ### error -| Message | When | -| ----------------------------------- | ----------------------------------------------------- | -| `global error: {message}` | Source emitted `error_level: global` β€” sync aborted | -| `stream error: {stream}: {message}` | Source emitted `error_level: stream` β€” stream skipped | -| `source crashed: {message}` | Source iterator threw an exception | +| Message | When | +| --- | --- | +| `global error: {message}` | Source emitted `error_level: global` | +| `stream error: {stream}: {message}` | Source emitted `error_level: stream` | +| `source crashed: {message}` | Source iterator threw | --- @@ -574,18 +441,14 @@ unexpected ordering β€” but logs warnings so they're alertable in production. NDJSON. One message per line. +```json +{"type":"start","sync_run_id":"sr_abc","source_config":{},"configured_catalog":{"streams":[{"name":"customers","sync_mode":"incremental","supports_time_range":true}]}} +{"type":"progress","progress":{"elapsed_ms":100,"global_state_count":0,"derived":{"records_per_second":0,"states_per_second":0},"streams":{"customers":{"state_count":0,"record_count":0,"completed_ranges":[],"terminal":false}},"errors":[]}} +{"type":"record","record":{"stream":"customers","data":{"id":"cus_123"}}} +{"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"derived":{"records_per_second":1562,"states_per_second":0.6},"streams":{"customers":{"state_count":1,"record_count":2500,"completed_ranges":[],"terminal":false}},"errors":[]}} +{"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"derived":{"records_per_second":1562,"states_per_second":0.6},"streams":{"customers":{"state_count":2,"record_count":5000,"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"terminal":true}},"errors":[]}} +{"type":"end","end":{"has_more":false,"ending_state":{"source":{"streams":{"customers":{"starting_after":null}},"global":{}},"engine":{"sync_run_id":"sr_abc","started_at":"2024-04-17T00:00:00Z","terminal_streams":["customers"],"run_progress":{"elapsed_ms":3200,"global_state_count":2,"derived":{"records_per_second":1562,"states_per_second":0.6},"streams":{"customers":{"state_count":2,"record_count":5000,"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"terminal":true}},"errors":[]}}},"request_progress":{"elapsed_ms":3200,"global_state_count":2,"derived":{"records_per_second":1562,"states_per_second":0.6},"streams":{"customers":{"state_count":2,"record_count":5000,"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"terminal":true}},"errors":[]}}} ``` -β†’ {"type":"start","sync_run_id":"sr_abc","source_config":{...},"configured_catalog":{...}} -← {"type":"progress","progress":{"elapsed_ms":100,"global_state_count":0,"records_per_second":0,"states_per_second":0,"streams":{"customers":{"completed_ranges":[],"record_count":0,"state_count":0}},"errors":[]}} -← {"type":"record","record":{"stream":"customers","data":{...}}} -← {"type":"progress","progress":{"elapsed_ms":1600,"global_state_count":1,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[],"record_count":2500,"state_count":1}},"errors":[]}} -← {"type":"record","record":{"stream":"customers","data":{...}}} -← {"type":"progress","progress":{"elapsed_ms":3200,"global_state_count":2,"records_per_second":1562,"states_per_second":0.6,"streams":{"customers":{"completed_ranges":[{"gte":"2018-01-01T00:00:00Z","lt":"2024-04-17T00:00:00Z"}],"record_count":5000,"state_count":2}},"errors":[]}} -← {"type":"end","has_more":false,"ending_state":{"source":{...},"engine":{...}}} -``` - -Over HTTP: POST with NDJSON body (one `start` line), NDJSON response stream. -Clients that don't want streaming buffer until `end`. --- @@ -605,10 +468,7 @@ do { }) state = end.ending_state } while (end.has_more) - -// Backfill complete. Schedule next sync with a new sync_run_id. ``` -The client does not need to understand stream statuses, error classification, -time ranges, or stall detection. It sends config + state, gets back -`has_more` + state. Everything else is in the progress stream for observability. +The client does not need to interpret source state. It only needs to round-trip +`ending_state` and continue until `has_more` is false. From 3ba817d7e87e8dff8f21c6c3b70111f52649ae79 Mon Sep 17 00:00:00 2001 From: tx Date: Fri, 17 Apr 2026 07:52:25 +0000 Subject: [PATCH 32/32] docs: sync Stripe source lifecycle with latest formatting Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- docs/engine/sync-lifecycle-source-stripe.md | 376 +++++++++++++------- 1 file changed, 245 insertions(+), 131 deletions(-) diff --git a/docs/engine/sync-lifecycle-source-stripe.md b/docs/engine/sync-lifecycle-source-stripe.md index 4665f3a4..e14bb1a5 100644 --- a/docs/engine/sync-lifecycle-source-stripe.md +++ b/docs/engine/sync-lifecycle-source-stripe.md @@ -1,212 +1,326 @@ # Sync Lifecycle β€” Stripe Source -How the Stripe source paginates finite backfills within the lifecycle described -in [sync-lifecycle.md](./sync-lifecycle.md). +How the Stripe source manages pagination within a `time_range` assigned by the +engine. For the overall sync lifecycle and protocol, see +[sync-lifecycle.md](./sync-lifecycle.md). ## Overview -Stripe list pagination is resumable because the API accepts an object-ID cursor: +The engine assigns a `time_range` per stream via the configured catalog. The +Stripe source paginates all records within that range using an n-ary search +algorithm: start with the full range, paginate, and subdivide if the range +takes more than one request to complete. No upfront density probing β€” the source discovers +the right granularity by doing the work. -```http -GET /v1/customers?limit=100&starting_after=cus_123 -``` - -For the same endpoint and same filter set, `starting_after` means "continue -after this object in the current list order." +## Source State -Stripe list endpoints return objects in descending `created` order. That makes -`starting_after` sufficient for continuation across requests: +```ts +type StripeStreamState = { + remaining: Array<{ + gte: string // ISO 8601 β€” inclusive lower bound + lt: string // ISO 8601 β€” exclusive upper bound + cursor: string | null // Stripe pagination cursor; null = not yet started + }> +} +``` -- The source stores the last emitted object ID as `starting_after`. -- On the next request, it replays the same query shape and resumes from that ID. -- For streams that support `created` filtering, the engine can also assign a - fixed `time_range`. +- `cursor: null` β†’ range planned but first page not yet fetched. +- `cursor: "cus_abc"` β†’ resume pagination after this object. +- Range removed from list β†’ complete. +- `remaining: []` β†’ source is done with the assigned `time_range`. -This design does **not** split partially paginated ranges across requests. A -time-range stream resumes the same assigned range until it emits `complete`. +## Algorithm -## Two Stream Modes +### 1. Initialization (no existing state) -### Time-range streams +The source receives `time_range` from the catalog and has no state. It starts +with the full range as a single entry: -These streams support Stripe `created[gte]` / `created[lt]` filters. +``` +Engine assigns: time_range { gte: "2018-01-01", lt: "2024-04-17" } -- The engine injects `time_range`. -- The source paginates with `created[...]` plus `starting_after`. -- The source may emit `range_complete` when the assigned range is fully read. -- The source emits `complete` when the stream is terminal for the run. +state: { + remaining: [ + { gte: "2018-01-01", lt: "2024-04-17", cursor: null } + ] +} +``` -### Non-time-range streams +### 2. Pagination -These streams do not support `created` filtering. +The source picks a range from `remaining` and paginates it: -- The engine does not inject `time_range`. -- The source paginates with `starting_after` only. -- There is no range coverage accounting. -- The source emits `complete` when the stream is terminal for the run. +1. Call the Stripe list API with `created[gte]` and `created[lt]` filters, + plus `starting_after` if cursor is set. +2. Emit records. +3. Update cursor in state, emit `source_state`. +4. When a range is exhausted (`has_more: false`), remove it from `remaining`. -Not every Stripe endpoint supports every pagination feature. Streams only enter -the time-range path if the endpoint supports the necessary `created` filters. +``` +First page fetched, got cursor: -## Source State +state: { + remaining: [ + { gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_abc" } + ] +} +β†’ emit source_state -Stripe source state is opaque to the engine. The minimal per-stream form is: +Pagination exhausted, range complete: -```ts -type StripeStreamState = { - starting_after: string | null +state: { + remaining: [] } +β†’ emit source_state (done) ``` -- `starting_after: null` means "start from the first page". -- `starting_after: "cus_abc"` means "resume after object `cus_abc`". - -The assigned `time_range`, when present, lives in the configured catalog. It is -not inferred from source state. +### 3. Subdivision (n-ary search) -## Pagination Algorithm +If a range didn't complete in the previous request, the source subdivides it +at the start of the next request. The source knows the `created` timestamp of +the last record it paginated (from the cursor). It splits the unpaginated +portion into N parts (where N = `max_segments_per_stream`): -### 1. Initialization - -If there is no saved state: +``` +Previous request ended with: + remaining: [{ gte: "2018-01-01", lt: "2024-04-17", cursor: "cus_xyz" }] + +Last record seen had created=2020-06-15. Range didn't complete β†’ subdivide. +The paginated portion [2018, 2020-06-15) keeps its cursor. +The unpaginated portion [2020-06-15, 2024-04-17) splits into N=2: + + remaining: [ + { gte: "2018-01-01", lt: "2020-06-15", cursor: "cus_xyz" }, + { gte: "2020-06-15", lt: "2022-05-16", cursor: null }, + { gte: "2022-05-16", lt: "2024-04-17", cursor: null } + ] +``` -- For a time-range stream, the source receives `time_range` from the engine and - starts with `starting_after: null`. -- For a non-time-range stream, the source starts with `starting_after: null`. +**When to subdivide:** At the start of a request, if any range in `remaining` +has a cursor (meaning it was in progress last request but didn't complete). +Subdivision happens between requests, not mid-request. -### 2. Page fetch +**Recursive:** If a subdivided range still doesn't complete in one request, +it gets split again next time. Each pass narrows the ranges until they're +small enough to complete in a single request. -For each page: +### 4. Resumption (existing state) -1. Build request params. -2. Call the Stripe list endpoint. -3. Emit records in the order returned by Stripe. -4. Save the last emitted object ID as `starting_after`. -5. Emit `source_state`. +If the source has existing state (from a previous request in the same sync +run), it resumes directly from `remaining`: -Time-range example: +``` +Source receives time_range { gte: "2018-01-01", lt: "2024-04-17" } +Existing state: { + remaining: [ + { gte: "2022-05-16", lt: "2024-04-17", cursor: "cus_xyz" } + ] +} -```http -GET /v1/customers?limit=100&created[gte]=1514764800&created[lt]=1713312000 -GET /v1/customers?limit=100&created[gte]=1514764800&created[lt]=1713312000&starting_after=cus_100 +β†’ Resume paginating from cus_xyz in [2022-05-16, 2024-04-17) +β†’ No re-initialization ``` -Non-time-range example: +### 5. Completion -```http -GET /v1/reporting/report_types?limit=100 -GET /v1/reporting/report_types?limit=100&starting_after=rpt_100 +When a sub-range is exhausted, the source removes it from `remaining` and +emits a `stream_status: range_complete`: + +``` +β†’ emit trace { stream_status: { stream: 'customers', status: 'range_complete', + range_complete: { gte: '2018-01-01', lt: '2019-06-01' } } } ``` -### 3. Resumption +The engine merges this into `completed_ranges`. -On the next request in the same sync run: +When all sub-ranges are done (`remaining: []`), the source emits +`stream_status: complete` for the stream. -- The engine reuses the same `sync_run_id`. -- For time-range streams, the engine re-injects the same fixed `time_range`. -- The source loads `starting_after` from source state. -- The source resumes the exact same query shape with that `starting_after`. +## Full Example -This works because `starting_after` is an object ID in Stripe's stable list -ordering. It is a resume token for pagination, not a derived time boundary. +Shows the messages emitted by the source during a two-request backfill of +`customers` with `time_range: [2018, 2024)`. -### 4. Completion +### Request 1 β€” full range, doesn't complete -When Stripe returns `has_more: false` for the current stream: +Stripe returns max 100 records per page. Each page = 1 API request = 1 state +checkpoint. -1. The source emits a final `source_state`. -2. If the stream had an assigned `time_range`, the source may emit - `stream_status: range_complete` for that range. -3. The source emits `stream_status: complete`. +``` +Source initializes: remaining: [{ gte: "2018", lt: "2024", cursor: null }] -`complete` is the terminal signal the engine trusts. `range_complete` is -progress telemetry only. +← trace { stream_status: { stream: "customers", status: "start" } } +← record { stream: "customers", data: { id: "cus_001", ... } } + ... 100 records (page 1) ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_100" }] } } +← record { stream: "customers", data: { ... } } + ... 100 records (page 2) ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_200" }] } } + ... pages 3-50 (5000 records total) ... +← state { stream: "customers", data: { remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] } } + ... source cut off (time limit / state limit) ... -## Message Examples +← end { has_more: true } +``` -### Time-range stream +Range didn't complete in one request β†’ source will subdivide on next request. -Request 1: +### Request 2 β€” source subdivides, finishes first sub-range -```text -Engine assigns: customers time_range [2018-01-01, 2024-04-17) +``` +Source resumes, sees remaining: [{ gte: "2018", lt: "2024", cursor: "cus_5000" }] +Last record had created=2019-03. Range didn't complete β†’ subdivide: + remaining: [ + { gte: "2018", lt: "2019-03", cursor: "cus_5000" }, // current (has cursor) + { gte: "2019-03", lt: "2021-09", cursor: null }, // new + { gte: "2021-09", lt: "2024", cursor: null } // new + ] + +← record { stream: "customers", data: { ... } } + ... 100 records (page) ... +← state { ... } + ... finishes [2018, 2019-03) after a few more pages ... +← trace { stream_status: { stream: "customers", status: "range_complete", + range_complete: { gte: "2018", lt: "2019-03" } } } +← state { stream: "customers", data: { remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: null }, + { gte: "2021-09", lt: "2024", cursor: null } + ] } } + ... starts [2019-03, 2021-09), paginates several pages ... +← state { stream: "customers", data: { remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, + { gte: "2021-09", lt: "2024", cursor: null } + ] } } + ... cut off ... -← trace { stream_status: { stream: "customers", status: "started" } } -← record { stream: "customers", data: { id: "cus_001", ... } } -← state { stream: "customers", data: { starting_after: "cus_100" } } -← record { stream: "customers", data: { id: "cus_101", ... } } -← state { stream: "customers", data: { starting_after: "cus_200" } } -... cut off ... ← end { has_more: true } ``` -Request 2: +### Request 3 β€” finishes remaining ranges -```text -Engine reassigns the same customers time_range [2018-01-01, 2024-04-17) -Source resumes with starting_after = "cus_200" +``` +Source resumes: remaining: [ + { gte: "2019-03", lt: "2021-09", cursor: "cus_8000" }, + { gte: "2021-09", lt: "2024", cursor: null } +] +These ranges made progress last request β€” no further subdivision, resume. -← record { stream: "customers", data: { id: "cus_201", ... } } -← state { stream: "customers", data: { starting_after: "cus_300" } } -... final page ... -← state { stream: "customers", data: { starting_after: "cus_5421" } } + ... paginates [2019-03, 2021-09) page by page ... ← trace { stream_status: { stream: "customers", status: "range_complete", - range_complete: { gte: "2018-01-01T00:00:00Z", lt: "2024-04-17T00:00:00Z" } } } + range_complete: { gte: "2019-03", lt: "2021-09" } } } + ... paginates [2021-09, 2024) page by page ... +← trace { stream_status: { stream: "customers", status: "range_complete", + range_complete: { gte: "2021-09", lt: "2024" } } } +← state { stream: "customers", data: { remaining: [] } } ← trace { stream_status: { stream: "customers", status: "complete" } } + ← end { has_more: false } ``` -### Non-time-range stream - -```text -No time_range assigned - -← trace { stream_status: { stream: "reporting_report_types", status: "started" } } -← record { stream: "reporting_report_types", data: { id: "rpt_001", ... } } -← state { stream: "reporting_report_types", data: { starting_after: "rpt_001" } } -... final page ... -← trace { stream_status: { stream: "reporting_report_types", status: "complete" } } -``` +Engine's `completed_ranges` for customers after merging all `range_complete` messages: +`[{ gte: "2018", lt: "2024" }]` ## State on the Wire +Source state is opaque to the engine. The engine learns about range completion +via `stream_status: range_complete` messages, not by inspecting source state: + ```ts { type: 'source_state', source_state: { state_type: 'stream', stream: 'customers', + time_range: { gte: '2018-01-01T00:00:00Z', lt: '2024-04-17T00:00:00Z' }, data: { - starting_after: 'cus_200' + remaining: [ + { gte: '2022-05-16T00:00:00Z', lt: '2024-04-17T00:00:00Z', cursor: 'cus_xyz' } + ] } } } ``` -The engine persists this state opaquely and passes it back on continuation. It -does not inspect `starting_after`. +## Concurrency -## Error Handling +Three controls govern how the source uses the Stripe API: + +```ts +// Source config β€” only max_concurrent_streams is user-configurable +type StripeSourceConfig = { + api_key: string + account_id?: string + max_concurrent_streams?: number // default 5 +} + +// Derived internally by the source: +// live_mode = inferred from api_key prefix (sk_live_ vs sk_test_) +// max_requests_per_second = live_mode ? 20 : 10 +// effective_streams = min(max_concurrent_streams, configured_stream_count) +// max_segments_per_stream = floor(max_requests_per_second / effective_streams) +``` + +| Control | What it controls | How it's set | +| ------------------------- | -------------------------------------------- | ------------------------------------------ | +| `max_concurrent_streams` | Streams paginating in parallel | Config (default 5), capped at catalog size | +| `max_requests_per_second` | Global rate limit across all activity | Inferred from API key mode | +| `max_segments_per_stream` | Sub-ranges per stream (n-ary search fan-out) | Derived: rps / concurrent streams | + +### Examples + +| Scenario | Mode | Streams | `effective_streams` | `rps` | `max_segments_per_stream` | Max concurrent requests | +| ---------------- | ---- | ------- | ------------------- | ----- | ------------------------- | ----------------------- | +| 20 streams, live | live | 20 | 5 | 20 | 4 | 20 | +| 20 streams, test | test | 20 | 5 | 10 | 2 | 10 | +| 3 streams, live | live | 3 | 3 | 20 | 6 | 18 | +| 1 stream, live | live | 1 | 1 | 20 | 20 | 20 | +| 1 stream, test | test | 1 | 1 | 10 | 10 | 10 | + +When fewer streams are configured, each stream gets more segments β€” the full +rate limit budget is distributed across whatever streams exist. A single-stream +sync gets the entire budget. -- **Transient errors**: retry at the HTTP layer and emit a `transient` error - trace for observability. -- **Stream errors**: emit a `stream` error trace, stop this stream, then emit - `complete` for explicit terminality. -- **Global errors**: emit a `global` error trace and stop the sync. +## Parallel Pagination -The source does not encode error semantics into source state. +The source paginates up to `max_segments_per_stream` ranges from `remaining` +concurrently per stream, and up to `effective_streams` streams in parallel. +Records from different ranges/streams are interleaved on the output stream. +State checkpoints are emitted after each page, reflecting the current state +of all ranges. This ensures resumability if the source is cut off mid-run. + +The global rate limiter (`max_requests_per_second`) governs all API calls +regardless of which stream or segment they belong to. + +## Source Logs + +The Stripe source emits `log` messages for real-time operational visibility. +These are passed through by the engine. + +| Level | Message | When | +|---|---|---| +| info | `{stream}: {rps} requests/sec` | Periodically during pagination | +| warn | `rate limited: retrying in {n}s` | Stripe returned 429 | +| warn | `retry {n}/{max}: {status} {message}` | Request failed, retrying | + +## Error Handling -## Exclusions +- **Transient errors** (rate limits, 5xx, timeouts): Retried at the HTTP + layer with exponential backoff. Emit a `transient` error trace for + observability regardless of whether the retry succeeded. +- **Stream errors** (resource not available, permission denied): Emit a + `stream` error trace, stop this stream, move to the next. +- **Global errors** (invalid API key): Emit a `global` error trace, stop. -This lifecycle doc does not cover: +The source does not store error state. If a range fails after all retries, +the range stays in `remaining` with its cursor for the next attempt. -- live `/events` polling -- cross-request range subdivision -- `ending_before`-driven reverse scans -- `full_refresh` semantics +## Events -In protocol terms, Stripe backfill now explicitly removes: +The `/events` endpoint is treated as just another stream in the catalog β€” +same `time_range` model, same `remaining`-based pagination. No special +incremental mode or live polling by default. -- using `starting_after` to derive new time boundaries between requests -- requiring every stream to support `time_range` -- using `range_complete` to decide terminality +For experimental live event polling (using events as a webhook replacement), +an opt-in flag stores cursor state in `source.global`, which is completely +separate from the per-stream backfill cursor logic. This is not enabled by +default.