feat(output): add transactional record spool

CONTRIBUTING.md

@@ -84,10 +84,10 @@ Every change must preserve these project contracts:
 The `internal/archguard` tests enforce the package and dependency boundaries:
 
 - **Core** (`applypatch`, `model`, `redact`, `rule`, `sequence`, `finding`,
-  `pipeline`, `output`, `winfile`) must not import a plane package.
+  `pipeline`, `output`, `spool`, `winfile`) must not import a plane package.
 - **Forensics** (`extract`, `discover`, `casebundle`) and **monitoring** (`hook`,
   `otel`, `state`) remain independent; neither imports the other.
-- Only `state` and `sequence` may import `bbolt`.
+- Only `state`, `sequence`, and `spool` may import `bbolt`.
 - In `cmd/numbat`, scan, timeline, and case files must not import the monitoring
   plane or bbolt. Hook, collect, and ship files must not import the forensics
   plane. `agents.go` is the sole cross-plane reporting bridge.

README.md

@@ -6,8 +6,8 @@ pre-action blocking, and forensic reconstruction.
 numbat observes supported desktop, CLI, IDE, and gateway agents through local
 hooks and plugins, OTLP/HTTP logs, and on-disk session artifacts. Live and
 at-rest activity is normalized into one event model and evaluated by the same
-CEL rule engine. Detection runs locally; records can be written to stdout or a
-local file and optionally delivered over HTTP.
+CEL rule engine. Detection runs locally. Records can use stdout, a local file,
+a durable on-disk queue, or HTTP delivery.
 
 The [coverage matrix](docs/agent-coverage.md#matrix) is authoritative for each
 host and surface. Blocking is off by default and limited to supported

cmd/numbat/collect.go

@@ -65,6 +65,7 @@ func runCollect(args []string, stdout, stderr io.Writer) int {
 	var outputValues multiFlag
 	fs.Var(&outputValues, "output", outputFlagHelp(outputModeStdout))
 	outputFile := fs.String("output-file", "", "destination path (required when --output includes file)")
+	spoolFile := fs.String("spool-file", "", "durable queue path (required when --output includes spool)")
 	httpURL := fs.String("http-url", "", "ingest URL (required when --output includes http)")
 	httpBatch := fs.Int("http-batch-size", 500, "records per HTTP POST")
 	httpTimeout := fs.Duration("http-timeout", 30*time.Second, "HTTP request timeout")
@@ -129,6 +130,7 @@ func runCollect(args []string, stdout, stderr io.Writer) int {
 		modes:         outputValues,
 		defaultMode:   outputModeStdout,
 		file:          *outputFile,
+		spool:         *spoolFile,
 		httpURL:       *httpURL,
 		httpBatch:     *httpBatch,
 		httpTimeout:   *httpTimeout,

cmd/numbat/hook.go

@@ -90,8 +90,9 @@ func runHookEvent(event string, args []string, stdin io.Reader, stdout, stderr i
 	contentFlag := fs.String("content", "preview", contentFlagHelp())
 	includeReasoning := fs.Bool("include-reasoning", false, "include source-recorded reasoning events when the integration exposes them")
 	var outputValues multiFlag
-	fs.Var(&outputValues, "output", outputFlagHelp(outputModeStdout)+"; stdout mode writes records to hook stderr and is unavailable in enforce mode")
+	fs.Var(&outputValues, "output", outputFlagHelp(outputModeStdout)+". stdout mode writes records to hook stderr and is unavailable in enforce mode")
 	outputFile := fs.String("output-file", "", "destination path (required when --output includes file)")
+	spoolFile := fs.String("spool-file", "", "durable queue path (required when --output includes spool)")
 	httpURL := fs.String("http-url", "", "ingest URL (required when --output includes http)")
 	httpBatch := fs.Int("http-batch-size", 500, "records per HTTP POST")
 	httpTimeout := fs.Duration("http-timeout", defaultHookHTTPTimeout, "HTTP request timeout")
@@ -199,6 +200,7 @@ func runHookEvent(event string, args []string, stdin io.Reader, stdout, stderr i
 		stateDB:          *stateDB,
 		modes:            outputValues,
 		file:             *outputFile,
+		spool:            *spoolFile,
 		httpURL:          *httpURL,
 		httpBatch:        *httpBatch,
 		httpTO:           *httpTimeout,
@@ -253,6 +255,7 @@ type hookOptions struct {
 	stateDB          string
 	modes            []string
 	file             string
+	spool            string
 	httpURL          string
 	httpBatch        int
 	httpTO           time.Duration
@@ -291,10 +294,22 @@ func handleHook(event string, lc hook.Lifecycle, agent, sourceAgent string, stdi
 	if err != nil {
 		return false, "", "", err
 	}
+	spoolFile, err := expandHookPath(opts.spool)
+	if err != nil {
+		return false, "", "", err
+	}
+	var statePath string
+	if sinks.spool {
+		statePath, err = hookStatePath(opts.stateDB, spoolFile)
+		if err != nil {
+			return false, "", "", err
+		}
+	}
 	sink, err := buildSink(sinkConfig{
 		modes:         opts.modes,
 		defaultMode:   outputModeStdout,
 		file:          outputFile,
+		spool:         spoolFile,
 		httpURL:       opts.httpURL,
 		httpBatch:     opts.httpBatch,
 		httpTimeout:   opts.httpTO,
@@ -353,7 +368,12 @@ func handleHook(event string, lc hook.Lifecycle, agent, sourceAgent string, stdi
 	var stateErr error
 	if (opts.sel.findings || opts.enforce) && eng != nil {
 		if seqs := eng.SequenceRules(); len(seqs) > 0 {
-			stateDB, stateErr = openHookState(opts.stateDB)
+			if statePath == "" {
+				statePath, stateErr = hookStatePath(opts.stateDB, spoolFile)
+			}
+			if stateErr == nil {
+				stateDB, stateErr = state.Open(statePath, 250*time.Millisecond)
+			}
 			if stateErr != nil {
 				em.Diag("warn", fmt.Sprintf("state database unavailable, %d sequence rule(s) skipped for this event: %v", len(seqs), stateErr))
 			} else {
@@ -596,7 +616,7 @@ func validateEnforceIO(enforce bool, sel emitSelection, sinks outputSinks) error
 		return fmt.Errorf("--enforce requires --emit findings (or --emit all)")
 	}
 	if sinks.stdout {
-		return fmt.Errorf("--enforce does not support --output stdout; use file and/or http for operator findings")
+		return fmt.Errorf("--enforce does not support --output stdout; use file, spool, and/or http for operator findings")
 	}
 	return nil
 }
@@ -687,18 +707,103 @@ func hookEventID(run string) string {
 // findingOptions pins one detection time across findings from the same callback.
 func findingOptions() finding.Options { return finding.Options{Now: time.Now()} }
 
-// openHookState opens the shared state database for one hook invocation: the
-// --state-db override, or state.db under a numbat-owned directory in the
-// user's home. The lock timeout is short because the agent is blocked while
-// the hook runs — a contended lock fails open rather than stalling it.
-func openHookState(override string) (*state.DB, error) {
+// hookStatePath resolves the shared sequence state path and prevents it from
+// sharing a bbolt file with the record spool.
+func hookStatePath(override, spoolPath string) (string, error) {
 	path := override
 	if path == "" {
 		home, err := os.UserHomeDir()
 		if err != nil {
-			return nil, fmt.Errorf("resolve home for state db: %w", err)
+			return "", fmt.Errorf("resolve home for state db: %w", err)
 		}
 		path = filepath.Join(home, ".numbat", "state.db")
 	}
-	return state.Open(path, 250*time.Millisecond)
+	if spoolPath != "" {
+		same, err := sameHookDataPath(spoolPath, path)
+		if err != nil {
+			return "", fmt.Errorf("compare --spool-file and --state-db: %w", err)
+		}
+		if same {
+			return "", errors.New("--spool-file and --state-db must name different files")
+		}
+	}
+	return path, nil
+}
+
+func sameHookDataPath(a, b string) (bool, error) { return sameShipPath(a, b) }
+
+func pathWithResolvedParent(path string) (string, error) {
+	path, err := filepath.Abs(filepath.Clean(path))
+	if err != nil {
+		return "", err
+	}
+	for {
+		resolved, err := filepath.EvalSymlinks(path)
+		if err == nil {
+			return resolved, nil
+		}
+		if !errors.Is(err, os.ErrNotExist) {
+			return "", err
+		}
+		info, err := os.Lstat(path)
+		if errors.Is(err, os.ErrNotExist) || err == nil && info.Mode()&os.ModeSymlink == 0 {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		path, err = resolvedSymlinkTarget(path)
+		if err != nil {
+			return "", err
+		}
+	}
+	parent := filepath.Dir(path)
+	var missing []string
+	for {
+		resolved, err := filepath.EvalSymlinks(parent)
+		if err == nil {
+			for i := len(missing) - 1; i >= 0; i-- {
+				resolved = filepath.Join(resolved, missing[i])
+			}
+			return filepath.Join(resolved, filepath.Base(path)), nil
+		}
+		if !errors.Is(err, os.ErrNotExist) {
+			return "", err
+		}
+		info, lstatErr := os.Lstat(parent)
+		if lstatErr == nil && info.Mode()&os.ModeSymlink != 0 {
+			target, err := resolvedSymlinkTarget(parent)
+			if err != nil {
+				return "", err
+			}
+			for i := len(missing) - 1; i >= 0; i-- {
+				target = filepath.Join(target, missing[i])
+			}
+			return pathWithResolvedParent(filepath.Join(target, filepath.Base(path)))
+		}
+		if lstatErr != nil && !errors.Is(lstatErr, os.ErrNotExist) {
+			return "", lstatErr
+		}
+		next := filepath.Dir(parent)
+		if next == parent {
+			return "", err
+		}
+		missing = append(missing, filepath.Base(parent))
+		parent = next
+	}
+}
+
+func resolvedSymlinkTarget(path string) (string, error) {
+	target, err := os.Readlink(path)
+	if err != nil {
+		return "", err
+	}
+	if !filepath.IsAbs(target) {
+		parent, err := filepath.EvalSymlinks(filepath.Dir(path))
+		if err != nil {
+			return "", err
+		}
+		target = filepath.Join(parent, target)
+	}
+	return filepath.Abs(filepath.Clean(target))
 }

cmd/numbat/hook_install.go

@@ -43,6 +43,7 @@ func runHookAdmin(action string, args []string, stdout, stderr io.Writer) int {
 		includeReasoning  bool
 		outputValues      multiFlag
 		outputFileValue   string
+		spoolFileValue    string
 		httpURL           string
 		httpBatch         = 500
 		httpTimeout       = defaultHookHTTPTimeout
@@ -59,8 +60,9 @@ func runHookAdmin(action string, args []string, stdout, stderr io.Writer) int {
 		fs.Var(&emitValues, "emit", "records emitted by live integrations: findings, events, indicators, or all (repeatable; default findings; enforce mode requires findings)")
 		fs.StringVar(&contentValue, "content", "preview", contentFlagHelp())
 		fs.BoolVar(&includeReasoning, "include-reasoning", false, "include source-recorded reasoning events when an integration exposes them")
-		fs.Var(&outputValues, "output", outputFlagHelp(outputModeFile)+"; stdout mode writes records to hook stderr and is unavailable in enforce mode")
+		fs.Var(&outputValues, "output", outputFlagHelp(outputModeFile)+". stdout mode writes records to hook stderr and is unavailable in enforce mode")
 		fs.StringVar(&outputFileValue, "output-file", "", "destination path when --output includes file (default findings.ndjson, or records.ndjson when --emit includes events/indicators)")
+		fs.StringVar(&spoolFileValue, "spool-file", "", "durable queue path when --output includes spool (default findings.spool, or records.spool when --emit includes events/indicators)")
 		fs.StringVar(&httpURL, "http-url", "", "ingest URL (required when --output includes http)")
 		fs.IntVar(&httpBatch, "http-batch-size", 500, "records per HTTP POST")
 		fs.DurationVar(&httpTimeout, "http-timeout", defaultHookHTTPTimeout, "HTTP request timeout")
@@ -189,6 +191,7 @@ func runHookAdmin(action string, args []string, stdout, stderr io.Writer) int {
 			includeReasoning: includeReasoning,
 			modes:            outputValues,
 			file:             outputFileValue,
+			spool:            spoolFileValue,
 			httpURL:          httpURL,
 			httpBatch:        httpBatch,
 			httpTimeout:      httpTimeout,
@@ -289,6 +292,7 @@ type installRuntimeConfig struct {
 	includeReasoning bool
 	modes            []string
 	file             string
+	spool            string
 	httpURL          string
 	httpBatch        int
 	httpTimeout      time.Duration
@@ -377,6 +381,9 @@ func installRuntimeArgs(cfg installRuntimeConfig, home string) ([]string, error)
 	if !sinks.file && cfg.file != "" {
 		return nil, fmt.Errorf("--output-file is only valid when --output includes file")
 	}
+	if !sinks.spool && cfg.spool != "" {
+		return nil, fmt.Errorf("--spool-file is only valid when --output includes spool")
+	}
 	if !sinks.http && cfg.httpURL != "" {
 		return nil, fmt.Errorf("--http-url is only valid when --output includes http")
 	}
@@ -388,6 +395,9 @@ func installRuntimeArgs(cfg installRuntimeConfig, home string) ([]string, error)
 		if cfg.httpURL != "" {
 			return nil, fmt.Errorf("--http-url is only valid when --output includes http")
 		}
+		if cfg.spool != "" {
+			return nil, fmt.Errorf("--spool-file is only valid when --output includes spool")
+		}
 		return append(args, "--output=stdout"), nil
 	}
 
@@ -418,6 +428,30 @@ func installRuntimeArgs(cfg installRuntimeConfig, home string) ([]string, error)
 		}
 		args = append(args, "--output-file", file)
 	}
+	if sinks.spool {
+		if set["spool-file"] && strings.TrimSpace(cfg.spool) == "" {
+			return nil, fmt.Errorf("--spool-file must not be empty")
+		}
+		path := cfg.spool
+		if path == "" {
+			findingsOnly := emitSel.defaultFindingsOnly()
+			if cfg.managed && findingsOnly {
+				path = "$HOME/.numbat/findings.spool"
+			} else if cfg.managed {
+				path = "$HOME/.numbat/records.spool"
+			} else if findingsOnly {
+				path = hook.DefaultFindingsSpoolPath(home)
+			} else {
+				path = hook.DefaultRecordsSpoolPath(home)
+			}
+		} else if !filepath.IsAbs(path) && !runtimeExpandedPath(path) {
+			path, err = filepath.Abs(path)
+			if err != nil {
+				return nil, fmt.Errorf("resolve --spool-file %q: %w", cfg.spool, err)
+			}
+		}
+		args = append(args, "--spool-file", path)
+	}
 	if sinks.http {
 		if cfg.httpURL == "" {
 			return nil, fmt.Errorf("--output including http requires --http-url URL")

cmd/numbat/main.go

@@ -13,7 +13,7 @@
 //	numbat scan [--agent NAME ... | --path FILE|DIR ...]      scan agent artifacts; emit selected NDJSON records
 //	numbat timeline [--agent NAME ... | --path FILE|DIR ...]  reconstruct a per-session chronological view
 //	numbat collect [--addr 127.0.0.1:4318]   receive OTLP/HTTP protobuf logs; emit records
-//	numbat ship --input-file F --http-url U  tail a local record file to an HTTP endpoint
+//	numbat ship (--spool-file S | --input-file F) --http-url U  send local records to an HTTP endpoint
 //	numbat hook (install|uninstall|status)   manage live agent integrations
 //	numbat agents                            report detected and supported local agents
 //	numbat rules check                       validate and compile rules; run companion tests
@@ -203,7 +203,7 @@ usage:
   numbat scan [--agent NAME ... | --path FILE|DIR ...]      scan agent artifacts; emit records (NDJSON)
   numbat timeline [--agent NAME ... | --path FILE|DIR ...]  reconstruct a per-session chronological view (text|json)
   numbat collect [--addr 127.0.0.1:4318]   receive OTLP/HTTP protobuf logs; emit records
-  numbat ship --input-file F --http-url U  tail a local record file to an HTTP endpoint
+  numbat ship (--spool-file S | --input-file F) --http-url U  send local records to an HTTP endpoint
   numbat hook EVENT --agent NAME           live integration callback (normally not run by hand)
   numbat hook install --agent NAME|all     install numbat's live integrations
   numbat hook uninstall --agent NAME|all   remove numbat-owned live integrations

cmd/numbat/scan.go

@@ -41,6 +41,7 @@ func runScan(args []string, stdout, stderr io.Writer) int {
 	var outputValues multiFlag
 	fs.Var(&outputValues, "output", outputFlagHelp(outputModeStdout))
 	outputFile := fs.String("output-file", "", "destination path (required when --output includes file)")
+	spoolFile := fs.String("spool-file", "", "durable queue path (required when --output includes spool)")
 	httpURL := fs.String("http-url", "", "ingest URL (required when --output includes http)")
 	httpBatch := fs.Int("http-batch-size", 500, "records per HTTP POST")
 	httpTimeout := fs.Duration("http-timeout", 30*time.Second, "HTTP request timeout")
@@ -128,6 +129,7 @@ func runScan(args []string, stdout, stderr io.Writer) int {
 		modes:         outputValues,
 		defaultMode:   outputModeStdout,
 		file:          *outputFile,
+		spool:         *spoolFile,
 		httpURL:       *httpURL,
 		httpBatch:     *httpBatch,
 		httpTimeout:   *httpTimeout,