fix(eval): surface batch-evaluation result truncation as a warning

src/core/batchEvaluationResults.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "bun:test";
 import type { CloudWatchLogsClient, OutputLogEvent } from "@aws-sdk/client-cloudwatch-logs";
+import { ResultTruncationError } from "../errors";
 import { createSilentLogger } from "../testing";
 import {
   isTerminalStatus,
@@ -156,6 +157,33 @@ test("readEvaluationResults skips lines without an evaluation name", async () =>
   expect(results).toEqual([]);
 });
 
+test("readEvaluationResults throws (not silently truncates) when it hits the page cap", async () => {
+  // Token advances on every call, so the loop never detects exhaustion and runs
+  // into MAX_RESULT_PAGES. It must throw so the caller surfaces truncation, rather
+  // than returning the accumulated partial list as if it were complete.
+  let call = 0;
+  const everAdvancing = {
+    send: async () => ({
+      events: [
+        {
+          message: JSON.stringify({
+            attributes: { "gen_ai.evaluation.name": "Builtin.Correctness" },
+          }),
+        },
+      ],
+      nextForwardToken: `t-${call++}`, // always changes → never exhausts
+    }),
+  } as unknown as CloudWatchLogsClient;
+
+  const err = await readEvaluationResults(everAdvancing, "lg", "ls", createSilentLogger()).then(
+    () => undefined,
+    (e) => e as ResultTruncationError,
+  );
+  expect(err).toBeInstanceOf(ResultTruncationError);
+  expect(err?.message).toMatch(/incomplete/);
+  expect(err?.source).toBe("internal"); // our page cap, not a user or service fault
+});
+
 test("parseEvaluationLogEvent warns on and skips an unparseable line", () => {
   const warnings: string[] = [];
   const logger = createSilentLogger();

src/core/batchEvaluationResults.tsx

@@ -1,4 +1,5 @@
 import { GetLogEventsCommand, type CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs";
+import { ResultTruncationError } from "../errors";
 import type { BatchEvaluationResultEntry } from "../handlers/eval/types";
 import type { Logger } from "../logging";
 
@@ -20,15 +21,20 @@ export function isTerminalStatus(status?: string): boolean {
 
 // GetLogEvents returns at most 1 MB / 10,000 events per call, so a job with many
 // results spans multiple pages. This caps the page loop as a safety valve against
-// a non-advancing token (see below); at 10k events/page it allows ~1M results,
-// far beyond the 500-session job limit.
+// a non-advancing token (see below). At 10k events/page it allows ~1M results, but
+// the 1 MB limit binds first — large explanations can cap a page well under 10k, so
+// this is not a "far beyond any real job" ceiling. Hitting it means the results are
+// truncated, which we surface as an error (see below) rather than silently
+// returning a partial list as if complete.
 const MAX_RESULT_PAGES = 100;
 
 // readEvaluationResults reads and parses the per-session/-trace/-tool scores from
 // a completed batch evaluation's CloudWatch result stream, following pagination to
-// completion. The caller supplies the log group and stream name from the job's
-// GetBatchEvaluation outputConfig (the service-selected values — we do not derive
-// the stream name, since its format is not part of the SDK contract).
+// completion. Throws if the stream exceeds MAX_RESULT_PAGES (results would be
+// truncated) — see the throw site. The caller supplies the log group and stream
+// name from the job's GetBatchEvaluation outputConfig (the service-selected values
+// — we do not derive the stream name, since its format is not part of the SDK
+// contract).
 export async function readEvaluationResults(
   logs: CloudWatchLogsClient,
   logGroupName: string,
@@ -63,10 +69,15 @@ export async function readEvaluationResults(
     token = next;
   }
 
-  logger.warn(
-    `stopped reading batch-evaluation results after ${MAX_RESULT_PAGES} pages; results may be truncated`,
+  // Cap reached with the token still advancing: the stream has more pages than we
+  // read, so `results` is truncated. Throw rather than return the partial list —
+  // getBatchEvaluation catches this into `resultsError`, which the CLI surfaces as
+  // a stderr warning (stdout metadata stays clean), the same customer-visible path
+  // as any other CloudWatch read failure. A silent partial list would read as
+  // complete.
+  throw new ResultTruncationError(
+    `batch-evaluation results exceed ${MAX_RESULT_PAGES} CloudWatch pages; retrieved ${results.length} results are incomplete`,
   );
-  return results;
 }
 
 // parseEvaluationLogEvent turns one CloudWatch result-log message into a result

src/errors/errors.tsx

@@ -166,3 +166,14 @@ export class FileWriteError extends AgentCoreCLIError {
     super(message, { source: ERROR_SOURCE.USER, ...options });
   }
 }
+
+/**
+ * Thrown when a paginated read is cut short by a client-side page cap, so the
+ * returned data is incomplete rather than the full result set. INTERNAL: the
+ * service and the user are both fine — the limit is ours.
+ */
+export class ResultTruncationError extends AgentCoreCLIError {
+  constructor(message: string, options?: Omit<AgentCoreCLIErrorOptions, "source">) {
+    super(message, { ...options, source: ERROR_SOURCE.INTERNAL });
+  }
+}

src/errors/index.tsx

@@ -9,6 +9,7 @@ export {
   NetworkingError,
   NotImplementedError,
   ProjectFileExistsError,
+  ResultTruncationError,
   RuntimeInvokeInterruptedError,
   RuntimeInvokeResponseError,
   SourceResolutionError,