diff --git a/apps/report/src/components/detail-panel/index.tsx b/apps/report/src/components/detail-panel/index.tsx
index 48bf6198ec..45bc6fc42a 100644
--- a/apps/report/src/components/detail-panel/index.tsx
+++ b/apps/report/src/components/detail-panel/index.tsx
@@ -16,6 +16,10 @@ import type {
IExecutionDump,
} from '@midscene/core';
import { executionToMarkdown, getTaskSearchArea } from '@midscene/core';
+import {
+ buildDeepAssertScreenshots,
+ isCurrentScreenshotFallback,
+} from '@midscene/core/agent';
import {
Blackboard,
Player,
@@ -253,7 +257,13 @@ const DetailPanel = ({
activeTask.uiContext?.screenshot?.capturedAt,
);
- contextLocatorView = activeTask.uiContext ? (
+ const evidenceScreenshots = buildDeepAssertScreenshots(activeTask);
+ const allowCurrentScreenshot =
+ evidenceScreenshots === undefined ||
+ isCurrentScreenshotFallback(activeTask);
+
+ contextLocatorView =
+ allowCurrentScreenshot && activeTask.uiContext ? (
@@ -268,7 +278,16 @@ const DetailPanel = ({
) : null;
- if (activeTask.recorder?.length) {
+ if (evidenceScreenshots && evidenceScreenshots.length > 0) {
+ for (const [index, item] of evidenceScreenshots.entries()) {
+ screenshotItems.push({
+ timestamp: item.screenshotTimestamp ?? index,
+ screenshotTimestamp: item.screenshotTimestamp,
+ screenshot: item.screenshot,
+ timing: item.timing,
+ });
+ }
+ } else if (evidenceScreenshots === undefined && activeTask.recorder?.length) {
for (const item of activeTask.recorder) {
const screenshot = item.screenshot?.base64;
if (screenshot) {
diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts
index 4f4c80b323..ec49d3e87a 100644
--- a/packages/core/src/agent/agent.ts
+++ b/packages/core/src/agent/agent.ts
@@ -125,6 +125,8 @@ export type AiActOptions = {
deepLocate?: boolean;
abortSignal?: AbortSignal;
context?: string;
+ AfterActPictures?: number;
+ Interval?: number;
};
type AiActInternalOptions = AiActOptions & {
@@ -400,6 +402,7 @@ export class Agent
waitAfterAction: this.opts.waitAfterAction,
useDeviceTime: this.opts.useDeviceTime,
actionSpace: this.fullActionSpace,
+ getAssertionExecutions: () => this.dump.executions,
hooks: {
onSnapshotChange: async (runner) => {
const executionDump = runner.dump();
@@ -1234,6 +1237,10 @@ export class Agent
deepLocate,
abortSignal,
internalReportDisplay,
+ {
+ AfterActPictures: opt?.AfterActPictures,
+ Interval: opt?.Interval,
+ },
);
// update cache
@@ -1399,7 +1406,7 @@ export class Agent
async aiAssert(
assertion: TUserPrompt,
- msg?: string,
+ msg?: string | AssertOptions,
opt?: AssertOptions,
): Promise {
return this.createInsight().aiAssert(assertion, msg, opt);
diff --git a/packages/core/src/agent/assertion-evidence.ts b/packages/core/src/agent/assertion-evidence.ts
new file mode 100644
index 0000000000..db753311bb
--- /dev/null
+++ b/packages/core/src/agent/assertion-evidence.ts
@@ -0,0 +1,560 @@
+import type {
+ AssertionBoundary,
+ AssertionEvaluationContext,
+ AssertionEvidenceImage,
+ ExecutionDump,
+ ExecutionTask,
+ IExecutionDump,
+} from '@/types';
+
+export const DEFAULT_AFTER_FRAMES = 1;
+export const DEFAULT_FRAME_INTERVAL_MS = 50;
+export const DEFAULT_BEFORE_EXECUTIONS = 1;
+export const DEFAULT_BEFORE_TASKS = 1;
+export const DEFAULT_MAX_PICTURES = 2;
+
+export const ANALYSIS_SECTIONS = [
+ '当前界面判断',
+ '关联 task 分析',
+ '截图证据分析',
+ '最终结论',
+] as const;
+
+const ACTION_EVIDENCE_PHASE_RE = /(before-calling|after-calling-(\d+))$/;
+
+const STATUS_TEXT: Record = {
+ finished: '已完成',
+ failed: '失败',
+ running: '执行中',
+ pending: '待执行',
+};
+
+export type ActionEvidenceOptions = {
+ AfterActPictures: number;
+ Interval: number;
+};
+
+export type NormalizedAssertEvidenceOptions = {
+ deepAssert: boolean;
+ AssertionContextBoundary: AssertionBoundary;
+ BeforeExecutions: number;
+ BeforeTasks: number;
+ MaxPictures: number;
+};
+
+export type AssertCallArgs = {
+ message?: string;
+ options?: {
+ keepRawResponse?: boolean;
+ context?: string;
+ abortSignal?: AbortSignal;
+ deepAssert?: boolean;
+ AssertionContextBoundary?: AssertionBoundary;
+ BeforeExecutions?: number;
+ BeforeTasks?: number;
+ MaxPictures?: number;
+ [key: string]: unknown;
+ };
+};
+
+export function nonNegative(
+ value: number | undefined,
+ fallback: number,
+ integer = true,
+): number {
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+ return fallback;
+ }
+ return integer ? Math.floor(value) : value;
+}
+
+export function normalizeActionEvidenceOptions(options?: {
+ AfterActPictures?: number;
+ Interval?: number;
+}): ActionEvidenceOptions {
+ return {
+ AfterActPictures: nonNegative(
+ options?.AfterActPictures,
+ DEFAULT_AFTER_FRAMES,
+ ),
+ Interval: nonNegative(options?.Interval, DEFAULT_FRAME_INTERVAL_MS),
+ };
+}
+
+export function normalizeAssertEvidenceOptions(
+ options?: Record | null,
+): NormalizedAssertEvidenceOptions {
+ const boundary = options?.AssertionContextBoundary;
+ return {
+ deepAssert: options?.deepAssert !== false,
+ AssertionContextBoundary:
+ boundary === 'session' || boundary === 'lastAssert'
+ ? boundary
+ : 'lastAssert',
+ BeforeExecutions: nonNegative(
+ typeof options?.BeforeExecutions === 'number'
+ ? options.BeforeExecutions
+ : undefined,
+ DEFAULT_BEFORE_EXECUTIONS,
+ ),
+ BeforeTasks: nonNegative(
+ typeof options?.BeforeTasks === 'number' ? options.BeforeTasks : undefined,
+ DEFAULT_BEFORE_TASKS,
+ ),
+ MaxPictures: nonNegative(
+ typeof options?.MaxPictures === 'number' ? options.MaxPictures : undefined,
+ DEFAULT_MAX_PICTURES,
+ ),
+ };
+}
+
+export function resolveAssertCallArgs(
+ message?: string | object,
+ options?: AssertCallArgs['options'],
+): AssertCallArgs {
+ if (message && typeof message === 'object' && !Array.isArray(message)) {
+ return {
+ options: {
+ ...(options || {}),
+ ...(message as AssertCallArgs['options']),
+ },
+ };
+ }
+ return {
+ ...(typeof message === 'string' ? { message } : {}),
+ ...(options ? { options } : {}),
+ };
+}
+
+export function isActionSpaceTask(
+ task: Pick | undefined,
+): boolean {
+ return task?.type === 'Action Space';
+}
+
+export function isAssertTask(
+ task: Pick | undefined,
+): boolean {
+ if (!task) {
+ return false;
+ }
+ if (task.type === 'Assert' || task.type === 'Assertion') {
+ return true;
+ }
+ return task.type === 'Insight' && task.subType === 'Assert';
+}
+
+export function isFinalPlanningSummary(
+ task: Pick | undefined,
+): boolean {
+ if (!task || task.type !== 'Planning' || task.status !== 'finished') {
+ return false;
+ }
+ const output = task.output as
+ | { shouldContinuePlanning?: boolean; actions?: unknown[] }
+ | undefined;
+ return output?.shouldContinuePlanning === false && !output?.actions?.length;
+}
+
+export function actionEvidencePhase(value: string | undefined): string {
+ if (!value) {
+ return '';
+ }
+ const match = ACTION_EVIDENCE_PHASE_RE.exec(value);
+ return match?.[1] ?? '';
+}
+
+function statusText(status: string | undefined): string {
+ if (!status) {
+ return '';
+ }
+ return STATUS_TEXT[status] ?? status;
+}
+
+function taskPrompt(task: ExecutionTask): string {
+ const param = task.param as Record | undefined;
+ if (typeof param?.assertion === 'string' && param.assertion.trim()) {
+ return param.assertion;
+ }
+ if (typeof param?.thought === 'string' && param.thought.trim()) {
+ return param.thought;
+ }
+ if (typeof task.thought === 'string' && task.thought.trim()) {
+ return task.thought;
+ }
+ if (typeof param?.description === 'string' && param.description.trim()) {
+ return param.description;
+ }
+ const locate = param?.locate as { prompt?: unknown; description?: unknown };
+ if (typeof locate?.description === 'string' && locate.description.trim()) {
+ return locate.description;
+ }
+ if (typeof locate?.prompt === 'string' && locate.prompt.trim()) {
+ return locate.prompt;
+ }
+ if (typeof param?.userInstructionDisplay === 'string') {
+ return param.userInstructionDisplay;
+ }
+ if (typeof param?.userInstruction === 'string') {
+ return param.userInstruction;
+ }
+ if (typeof param?.dataDemand === 'string') {
+ return param.dataDemand;
+ }
+ return '';
+}
+
+function failureReason(task: ExecutionTask): string | undefined {
+ if (typeof task.errorMessage === 'string' && task.errorMessage.trim()) {
+ return task.errorMessage;
+ }
+ const error = task.error as { message?: string } | string | undefined;
+ if (typeof error === 'string' && error.trim()) {
+ return error;
+ }
+ if (error && typeof error === 'object' && typeof error.message === 'string') {
+ return error.message;
+ }
+ return undefined;
+}
+
+export function recentAssertionExecutions(
+ allExecutions: Array,
+ maxExecutions: number,
+ boundary: AssertionBoundary,
+): Array {
+ const nonEmpty = allExecutions.filter(
+ (item) => Array.isArray(item.tasks) && item.tasks.length > 0,
+ );
+ let start = 0;
+ if (boundary === 'lastAssert') {
+ for (let index = nonEmpty.length - 1; index >= 0; index--) {
+ if (nonEmpty[index].tasks.some((task) => isAssertTask(task))) {
+ start = index + 1;
+ break;
+ }
+ }
+ }
+ return maxExecutions > 0 ? nonEmpty.slice(start).slice(-maxExecutions) : [];
+}
+
+function eligibleTasksFromExecutions(
+ executions: Array,
+): Array<{ execution: ExecutionDump | IExecutionDump; task: ExecutionTask }> {
+ const selected: Array<{
+ execution: ExecutionDump | IExecutionDump;
+ task: ExecutionTask;
+ }> = [];
+ for (const execution of executions) {
+ for (const task of execution.tasks || []) {
+ if (task.type === 'Log' || isFinalPlanningSummary(task)) {
+ continue;
+ }
+ selected.push({ execution, task });
+ }
+ }
+ return selected;
+}
+
+export function buildEvaluationContext(
+ executions: Array,
+ beforeTasks: number,
+): AssertionEvaluationContext {
+ const eligible = eligibleTasksFromExecutions(executions);
+ const window = beforeTasks > 0 ? eligible.slice(-beforeTasks) : [];
+ const contextExecutions = executions.map((execution) => ({
+ executionId: execution.id,
+ title: execution.name || '',
+ }));
+ const tasks = window.map(({ execution, task }) => ({
+ executionTitle: execution.name || '',
+ taskId: task.taskId,
+ type: task.type,
+ ...(task.subType ? { subType: task.subType } : {}),
+ prompt: taskPrompt(task) || '(无描述)',
+ status: task.status,
+ ...(failureReason(task) ? { failureReason: failureReason(task) } : {}),
+ }));
+
+ const summaryParts = contextExecutions.map((item) => item.title).filter(Boolean);
+ return {
+ summary:
+ summaryParts.length > 0
+ ? `关联执行：${summaryParts.join('；')}`
+ : '无关联执行',
+ executions: contextExecutions,
+ tasks,
+ };
+}
+
+function screenshotIdentity(screenshot: {
+ id?: string;
+ base64?: string;
+}): { id?: string; content?: string } {
+ return {
+ ...(screenshot.id ? { id: screenshot.id } : {}),
+ ...(screenshot.base64 ? { content: screenshot.base64 } : {}),
+ };
+}
+
+export function selectAssertionEvidenceImages(
+ executions: Array,
+ maxPictures: number,
+): AssertionEvidenceImage[] {
+ if (maxPictures <= 0) {
+ return [];
+ }
+
+ const candidates: AssertionEvidenceImage[] = [];
+ for (const execution of executions) {
+ for (const task of execution.tasks || []) {
+ if (!isActionSpaceTask(task)) {
+ continue;
+ }
+ for (const record of task.recorder || []) {
+ const phase = actionEvidencePhase(record.timing);
+ if (!phase) {
+ continue;
+ }
+ const screenshot = record.screenshot;
+ const url = screenshot?.base64;
+ if (!url) {
+ continue;
+ }
+ const subtype = task.subType || task.type;
+ candidates.push({
+ name: `${execution.name || ''} / ${subtype} / ${phase}`,
+ url,
+ ...(typeof screenshot.capturedAt === 'number'
+ ? { capturedAt: screenshot.capturedAt }
+ : {}),
+ ...(screenshot.id ? { id: screenshot.id } : {}),
+ } as AssertionEvidenceImage & { id?: string });
+ }
+ }
+ }
+
+ const selected: AssertionEvidenceImage[] = [];
+ const seenIds = new Set();
+ const seenContent = new Set();
+
+ for (
+ let index = candidates.length - 1;
+ index >= 0 && selected.length < maxPictures;
+ index--
+ ) {
+ const image = candidates[index] as AssertionEvidenceImage & { id?: string };
+ const identity = screenshotIdentity({ id: image.id, base64: image.url });
+ if (identity.id && seenIds.has(identity.id)) {
+ continue;
+ }
+ if (identity.content && seenContent.has(identity.content)) {
+ continue;
+ }
+ if (identity.id) {
+ seenIds.add(identity.id);
+ }
+ if (identity.content) {
+ seenContent.add(identity.content);
+ }
+ selected.push({
+ name: image.name,
+ url: image.url,
+ ...(image.capturedAt === undefined
+ ? {}
+ : { capturedAt: image.capturedAt }),
+ });
+ }
+
+ return selected;
+}
+
+export function citationForContextTask(
+ task: AssertionEvaluationContext['tasks'][number],
+ index: number,
+): string {
+ const label = task.subType ? `${task.type}/${task.subType}` : task.type;
+ const failure = task.failureReason
+ ? `；失败原因：${task.failureReason}`
+ : '';
+ return `Task ${index}: [${task.executionTitle}] ${label} - ${task.prompt || '(无描述)'} -> ${statusText(task.status)}${failure}`;
+}
+
+function analysisSection(thought: string, section: string): string {
+ const start = thought.indexOf(section);
+ if (start < 0) {
+ return '';
+ }
+ const contentStart = start + section.length;
+ const following = ANALYSIS_SECTIONS.map((candidate) =>
+ candidate === section ? -1 : thought.indexOf(candidate, contentStart),
+ ).filter((index) => index >= 0);
+ const end = following.length > 0 ? Math.min(...following) : thought.length;
+ return thought.slice(contentStart, end).trim();
+}
+
+function stripForbiddenTerms(value: string): string {
+ return value
+ .replace(/构建响应/g, '')
+ .replace(/响应构建/g, '')
+ .replace(/DATA_DEMAND/g, '')
+ .replace(/StatementIsTruthy/g, '')
+ .trim();
+}
+
+export function composeAssertThought(input: {
+ modelThought?: string;
+ assertion: string;
+ passed: boolean;
+ evaluationContext?: AssertionEvaluationContext;
+ evidenceImages?: AssertionEvidenceImage[];
+}): string {
+ const raw = stripForbiddenTerms(input.modelThought || '');
+ const current =
+ analysisSection(raw, '当前界面判断') ||
+ (raw && !ANALYSIS_SECTIONS.some((section) => raw.includes(section))
+ ? raw
+ : '');
+ const screenshot =
+ analysisSection(raw, '截图证据分析') ||
+ (input.evidenceImages && input.evidenceImages.length > 0
+ ? '对比动作前后界面可见内容，结合断言目标判断状态是否成立。'
+ : '当前没有可用的动作前后截图，依据最终界面判断。');
+ const conclusion =
+ analysisSection(raw, '最终结论') ||
+ (input.passed
+ ? `断言「${input.assertion}」成立。`
+ : `断言「${input.assertion}」不成立。`);
+
+ const tasks = input.evaluationContext?.tasks ?? [];
+ const relatedLines =
+ tasks.length === 0
+ ? ['没有可引用的关联任务。']
+ : tasks.flatMap((task, index) => {
+ const citation = citationForContextTask(task, index + 1);
+ return [
+ citation,
+ `该任务执行了「${task.prompt || '(无描述)'}」，状态为${statusText(task.status)}，据此影响断言判断。`,
+ ];
+ });
+
+ return [
+ '当前界面判断',
+ current ||
+ (input.passed
+ ? `最终界面满足断言「${input.assertion}」。`
+ : `最终界面未满足断言「${input.assertion}」。`),
+ '关联 task 分析',
+ relatedLines.join('\n'),
+ '截图证据分析',
+ screenshot,
+ '最终结论',
+ conclusion,
+ ].join('\n');
+}
+
+export function deepAssertEvidence(
+ task: Pick | undefined,
+): AssertionEvidenceImage[] | undefined {
+ if (!task || !isAssertTask(task) || task.param?.deepAssert === false) {
+ return undefined;
+ }
+ const param = task.param as
+ | { assertionEvidenceImages?: AssertionEvidenceImage[]; deepAssert?: boolean }
+ | undefined;
+ if (
+ param?.assertionEvidenceImages === undefined &&
+ param?.deepAssert !== true
+ ) {
+ return undefined;
+ }
+ return param?.assertionEvidenceImages ?? [];
+}
+
+export function buildDeepAssertScreenshots(
+ task: Pick | undefined,
+):
+ | Array<{
+ screenshot: string;
+ timing: string;
+ screenshotTimestamp?: number;
+ }>
+ | null
+ | undefined {
+ const evidence = deepAssertEvidence(task);
+ if (evidence === undefined) {
+ return undefined;
+ }
+ if (evidence.length === 0) {
+ return null;
+ }
+ return evidence
+ .slice()
+ .reverse()
+ .map((image, index) => ({
+ screenshot: image.url,
+ timing: `参考图${index + 1} / ${image.name}`,
+ ...(image.capturedAt === undefined
+ ? {}
+ : { screenshotTimestamp: image.capturedAt }),
+ }));
+}
+
+export function asEvaluationContext(
+ value: unknown,
+): AssertionEvaluationContext | undefined {
+ if (!value || typeof value !== 'object') {
+ return undefined;
+ }
+ const candidate = value as AssertionEvaluationContext;
+ if (
+ typeof candidate.summary !== 'string' ||
+ !Array.isArray(candidate.executions) ||
+ !Array.isArray(candidate.tasks)
+ ) {
+ return undefined;
+ }
+ return candidate;
+}
+
+export function asEvidenceImages(
+ value: unknown,
+): AssertionEvidenceImage[] | undefined {
+ return Array.isArray(value) ? (value as AssertionEvidenceImage[]) : undefined;
+}
+
+export function isCurrentScreenshotFallback(
+ task: Pick | undefined,
+): boolean {
+ return task?.param?.assertionEvidenceFallback === 'currentScreenshot';
+}
+
+export function buildAssertionEvidenceModelContext(input: {
+ assertion: string;
+ evaluationContext?: AssertionEvaluationContext;
+ evidenceImages: AssertionEvidenceImage[];
+}): string {
+ const taskLines = (input.evaluationContext?.tasks ?? []).map((task, index) =>
+ citationForContextTask(task, index + 1),
+ );
+ const imageLines = input.evidenceImages.map(
+ (image, index) => `图${index + 1}: ${image.name}`,
+ );
+
+ return [
+ `断言目标：${input.assertion}`,
+ input.evaluationContext?.summary
+ ? `所选执行摘要：${input.evaluationContext.summary}`
+ : '',
+ taskLines.length > 0
+ ? `所选任务证据：\n${taskLines.join('\n')}`
+ : '所选任务证据：无',
+ imageLines.length > 0
+ ? `具名证据图（与 assertionEvidenceImages 一一对应，最新在前）：\n${imageLines.join('\n')}`
+ : '具名证据图：无',
+ '判断规则：先看最终是否成功；不要只因为路径或中间动作不同就判失败。只有断言明确要求动态过程时，才用证据链评过程。',
+ '请用中文按以下四段作答，且不要出现 DATA_DEMAND、StatementIsTruthy、构建响应、响应构建：当前界面判断；关联 task 分析（逐条引用上述 Task N）；截图证据分析（比较 before-calling 与 after-calling-* 的可见变化，不要罗列图名）；最终结论。',
+ ]
+ .filter(Boolean)
+ .join('\n');
+}
diff --git a/packages/core/src/agent/execution-session.ts b/packages/core/src/agent/execution-session.ts
index 687b9e0a7c..79fdbe07d4 100644
--- a/packages/core/src/agent/execution-session.ts
+++ b/packages/core/src/agent/execution-session.ts
@@ -13,6 +13,8 @@ import type {
type ExecutionSessionOptions = ExecutionTaskProgressOptions & {
tasks?: ExecutionTaskApply[];
referenceImages?: readonly ExecutionReferenceImage[];
+ actionEvidenceAfterFrameCount?: number;
+ actionEvidenceFrameIntervalMs?: number;
onSnapshotChange?: (
runner: TaskRunner,
error?: TaskExecutionError,
diff --git a/packages/core/src/agent/index.ts b/packages/core/src/agent/index.ts
index 986baaaacf..0bb61af1bf 100644
--- a/packages/core/src/agent/index.ts
+++ b/packages/core/src/agent/index.ts
@@ -18,6 +18,11 @@ export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
export { cacheFileExt } from './task-cache';
export { TaskExecutor } from './tasks';
+export {
+ buildDeepAssertScreenshots,
+ deepAssertEvidence,
+ isCurrentScreenshotFallback,
+} from './assertion-evidence';
export type { MidsceneUsageMetrics, UsageBucket } from './metrics';
export type {
GherkinStepKeyword,
diff --git a/packages/core/src/agent/insight.ts b/packages/core/src/agent/insight.ts
index c1debc3637..4e7e828a90 100644
--- a/packages/core/src/agent/insight.ts
+++ b/packages/core/src/agent/insight.ts
@@ -8,6 +8,7 @@ import type {
ServiceExtractParam,
UIContext,
} from '@/types';
+import { resolveAssertCallArgs } from './assertion-evidence';
import { TaskExecutionError, type TaskExecutor } from './tasks';
import { parsePrompt } from './utils';
@@ -113,14 +114,37 @@ export class Insight implements InsightAPI {
async aiAssert(
assertion: TUserPrompt,
- message?: string,
+ message?: string | AssertOptions,
options?: AssertOptions,
): Promise {
+ const resolved = resolveAssertCallArgs(message, options);
+ const assertOptions = resolved.options as AssertOptions | undefined;
const serviceOptions: QueryOptions = {
- domIncluded: options?.domIncluded ?? defaultQueryOptions.domIncluded,
+ domIncluded:
+ assertOptions?.domIncluded ?? defaultQueryOptions.domIncluded,
screenshotIncluded:
- options?.screenshotIncluded ?? defaultQueryOptions.screenshotIncluded,
- ...(options?.context !== undefined ? { context: options.context } : {}),
+ assertOptions?.screenshotIncluded ??
+ defaultQueryOptions.screenshotIncluded,
+ ...(assertOptions?.context !== undefined
+ ? { context: assertOptions.context }
+ : {}),
+ ...(assertOptions?.deepAssert === undefined
+ ? {}
+ : { deepAssert: assertOptions.deepAssert }),
+ ...(assertOptions?.AssertionContextBoundary === undefined
+ ? {}
+ : {
+ AssertionContextBoundary: assertOptions.AssertionContextBoundary,
+ }),
+ ...(assertOptions?.BeforeExecutions === undefined
+ ? {}
+ : { BeforeExecutions: assertOptions.BeforeExecutions }),
+ ...(assertOptions?.BeforeTasks === undefined
+ ? {}
+ : { BeforeTasks: assertOptions.BeforeTasks }),
+ ...(assertOptions?.MaxPictures === undefined
+ ? {}
+ : { MaxPictures: assertOptions.MaxPictures }),
};
const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
const assertionText =
@@ -134,15 +158,15 @@ export class Insight implements InsightAPI {
this.resolveModelRuntime(),
serviceOptions,
multimodalPrompt,
- this.executionOptions(options, true),
+ this.executionOptions(assertOptions, true),
);
const pass = Boolean(output);
const failureMessage = pass
? undefined
- : `Assertion failed: ${message || assertionText}\nReason: ${thought || '(no_reason)'}`;
+ : `Assertion failed: ${resolved.message || assertionText}\nReason: ${thought || '(no_reason)'}`;
- if (options?.keepRawResponse) {
+ if (assertOptions?.keepRawResponse) {
return { pass, thought, message: failureMessage };
}
if (!pass) {
@@ -154,9 +178,9 @@ export class Insight implements InsightAPI {
const diagnosticMessage =
error.task?.errorMessage || error.cause.message;
const reason = thought || diagnosticMessage || '(no_reason)';
- const failureMessage = `Assertion failed: ${message || assertionText}\nReason: ${reason}`;
+ const failureMessage = `Assertion failed: ${resolved.message || assertionText}\nReason: ${reason}`;
- if (options?.keepRawResponse) {
+ if (assertOptions?.keepRawResponse) {
return { pass: false, thought, message: failureMessage };
}
throw new Error(failureMessage, { cause: error.cause });
diff --git a/packages/core/src/agent/tasks.ts b/packages/core/src/agent/tasks.ts
index 125e6014a6..08cd397895 100644
--- a/packages/core/src/agent/tasks.ts
+++ b/packages/core/src/agent/tasks.ts
@@ -22,14 +22,17 @@ import type {
AiActEffort,
AiActProgressData,
AiActProgressPhase,
+ AssertionEvidenceImage,
DetailedLocateParam,
DeviceAction,
+ ExecutionDump,
ExecutionRecorderItem,
ExecutionTask,
ExecutionTaskApply,
ExecutionTaskInsightQueryApply,
ExecutionTaskPlanningApply,
ExecutionTaskProgressOptions,
+ IExecutionDump,
MidsceneYamlFlowItem,
PlanningAIResponse,
PlanningAction,
@@ -42,6 +45,17 @@ import type {
import { ServiceError, aiActProgressScope } from '@/types';
import { getDebug } from '@midscene/shared/logger';
import { assert } from '@midscene/shared/utils';
+import {
+ asEvaluationContext,
+ asEvidenceImages,
+ buildAssertionEvidenceModelContext,
+ buildEvaluationContext,
+ composeAssertThought,
+ normalizeActionEvidenceOptions,
+ normalizeAssertEvidenceOptions,
+ recentAssertionExecutions,
+ selectAssertionEvidenceImages,
+} from './assertion-evidence';
import { ExecutionSession } from './execution-session';
import { withFileChooser } from './file-chooser';
import {
@@ -122,6 +136,10 @@ export class TaskExecutor {
useDeviceTime?: boolean;
+ private readonly getAssertionExecutions?: () => Array<
+ ExecutionDump | IExecutionDump
+ >;
+
// @deprecated use .interface instead
get page() {
return this.interface;
@@ -138,6 +156,7 @@ export class TaskExecutor {
useDeviceTime?: boolean;
hooks?: TaskExecutorHooks;
actionSpace: DeviceAction[];
+ getAssertionExecutions?: () => Array;
},
) {
this.interface = interfaceInstance;
@@ -147,6 +166,7 @@ export class TaskExecutor {
this.replanningCycleLimit = opts.replanningCycleLimit;
this.waitAfterAction = opts.waitAfterAction;
this.useDeviceTime = opts.useDeviceTime;
+ this.getAssertionExecutions = opts.getAssertionExecutions;
this.hooks = opts.hooks;
this.providedActionSpace = opts.actionSpace;
this.taskBuilder = new TaskBuilder({
@@ -164,6 +184,8 @@ export class TaskExecutor {
tasks?: ExecutionTaskApply[];
uiContext?: UIContext;
referenceImages?: readonly ExecutionReferenceImage[];
+ actionEvidenceAfterFrameCount?: number;
+ actionEvidenceFrameIntervalMs?: number;
onSnapshotChange?: (
runner: TaskRunner,
error?: TaskExecutionError,
@@ -181,6 +203,18 @@ export class TaskExecutor {
onTaskStart: this.onTaskStartCallback,
tasks: options?.tasks,
referenceImages: options?.referenceImages,
+ ...(options?.actionEvidenceAfterFrameCount === undefined
+ ? {}
+ : {
+ actionEvidenceAfterFrameCount:
+ options.actionEvidenceAfterFrameCount,
+ }),
+ ...(options?.actionEvidenceFrameIntervalMs === undefined
+ ? {}
+ : {
+ actionEvidenceFrameIntervalMs:
+ options.actionEvidenceFrameIntervalMs,
+ }),
onSnapshotChange: async (runner, error) => {
await this.hooks?.onSnapshotChange?.(runner, error);
await options?.onSnapshotChange?.(runner, error);
@@ -380,6 +414,10 @@ export class TaskExecutor {
deepLocate?: boolean,
abortSignal?: AbortSignal,
reportOptions?: ActionReportOptions,
+ actionEvidence?: {
+ AfterActPictures?: number;
+ Interval?: number;
+ },
): Promise<
ExecutionResult<
| {
@@ -401,6 +439,7 @@ export class TaskExecutor {
deepLocate,
abortSignal,
reportOptions,
+ actionEvidence,
);
});
}
@@ -450,6 +489,10 @@ export class TaskExecutor {
deepLocate?: boolean,
abortSignal?: AbortSignal,
reportOptions?: ActionReportOptions,
+ actionEvidence?: {
+ AfterActPictures?: number;
+ Interval?: number;
+ },
): Promise<
ExecutionResult<
| {
@@ -462,6 +505,8 @@ export class TaskExecutor {
const conversationHistory = new ConversationHistory();
const promptDisplay =
reportOptions?.prompt || userPromptToString(userPrompt);
+ const normalizedActionEvidence =
+ normalizeActionEvidenceOptions(actionEvidence);
// Per-call reporter that maps the runner's native task events to aiAct
// action progress for the action batch currently running. Kept local (not
@@ -475,6 +520,9 @@ export class TaskExecutor {
taskTitleStr(reportOptions?.type || 'Act', promptDisplay),
{
referenceImages: userPromptToMultimodalPrompt(userPrompt)?.images,
+ actionEvidenceAfterFrameCount:
+ normalizedActionEvidence.AfterActPictures,
+ actionEvidenceFrameIntervalMs: normalizedActionEvidence.Interval,
onTaskEvent: async (event) => {
await activeActionReporter?.(event);
},
@@ -878,6 +926,14 @@ export class TaskExecutor {
abortSignal?: AbortSignal;
},
) {
+ const assertEvidence =
+ type === 'Assert' ? normalizeAssertEvidenceOptions(opt) : undefined;
+ const assertionText =
+ type === 'Assert'
+ ? typeof demand === 'string'
+ ? demand
+ : JSON.stringify(demand)
+ : undefined;
const queryTask: ExecutionTaskInsightQueryApply = {
type: 'Insight',
subType: type,
@@ -890,6 +946,36 @@ export class TaskExecutor {
multimodalPrompt,
} as never)
: demand, // for user param presentation in report right sidebar
+ ...(type === 'Assert' && assertionText !== undefined && assertEvidence
+ ? {
+ assertion: assertionText,
+ deepAssert: assertEvidence.deepAssert,
+ AssertionContextBoundary:
+ assertEvidence.AssertionContextBoundary,
+ BeforeExecutions: assertEvidence.BeforeExecutions,
+ BeforeTasks: assertEvidence.BeforeTasks,
+ MaxPictures: assertEvidence.MaxPictures,
+ ...(asEvaluationContext(opt?.evaluationContext)
+ ? {
+ evaluationContext: asEvaluationContext(
+ opt?.evaluationContext,
+ ),
+ }
+ : {}),
+ ...(asEvidenceImages(opt?.assertionEvidenceImages)
+ ? {
+ assertionEvidenceImages: asEvidenceImages(
+ opt?.assertionEvidenceImages,
+ ),
+ }
+ : {}),
+ ...(opt?.assertionEvidenceFallback === 'currentScreenshot'
+ ? {
+ assertionEvidenceFallback: 'currentScreenshot' as const,
+ }
+ : {}),
+ }
+ : {}),
},
executor: async (taskContext) => {
const { task } = taskContext;
@@ -942,11 +1028,63 @@ export class TaskExecutor {
);
}
+ let evidenceImages = Array.isArray(opt?.assertionEvidenceImages)
+ ? ([...opt.assertionEvidenceImages] as AssertionEvidenceImage[])
+ : [];
+ let evidenceFallback = opt?.assertionEvidenceFallback as
+ | 'currentScreenshot'
+ | undefined;
+ if (
+ type === 'Assert' &&
+ assertEvidence?.deepAssert &&
+ assertEvidence.MaxPictures !== 0 &&
+ evidenceImages.length === 0
+ ) {
+ evidenceFallback = 'currentScreenshot';
+ if (task.param) {
+ task.param.assertionEvidenceFallback = 'currentScreenshot';
+ task.param.assertionEvidenceImages = [];
+ }
+ } else if (
+ type === 'Assert' &&
+ assertEvidence?.deepAssert &&
+ task.param
+ ) {
+ task.param.assertionEvidenceImages = evidenceImages;
+ }
+
+ const extractOpt =
+ type === 'Assert' && assertEvidence?.deepAssert
+ ? {
+ ...opt,
+ deepAssert: true,
+ assertionEvidenceImages: evidenceImages,
+ assertionEvidenceFallback: evidenceFallback,
+ screenshotIncluded:
+ evidenceFallback === 'currentScreenshot'
+ ? opt?.screenshotIncluded
+ : false,
+ context: [
+ opt?.context,
+ buildAssertionEvidenceModelContext({
+ assertion: assertionText || String(demand),
+ evaluationContext: task.param?.evaluationContext,
+ evidenceImages,
+ }),
+ ]
+ .filter(
+ (item): item is string =>
+ typeof item === 'string' && item.trim().length > 0,
+ )
+ .join('\n\n'),
+ }
+ : opt;
+
try {
extractResult = await this.service.extract(
demandInput,
modelRuntime,
- opt,
+ extractOpt,
extraPageDescription,
multimodalPrompt,
uiContext,
@@ -963,8 +1101,9 @@ export class TaskExecutor {
recordAndReleaseScreenshotSequence(task, uiContext);
}
- const { data, thought, dump } = extractResult;
+ const { data, thought: rawThought, dump } = extractResult;
applyDump(dump);
+ let thought = rawThought;
let outputResult = data;
if (ifTypeRestricted) {
@@ -995,6 +1134,18 @@ export class TaskExecutor {
// the model dump before aiAssert turns the result into a thrown error.
if (type === 'Assert') {
outputResult = Boolean(outputResult);
+ if (assertEvidence?.deepAssert) {
+ thought = composeAssertThought({
+ modelThought: thought,
+ assertion: assertionText || String(demand),
+ passed: outputResult,
+ evaluationContext: task.param?.evaluationContext,
+ evidenceImages:
+ (task.param?.assertionEvidenceImages as
+ | AssertionEvidenceImage[]
+ | undefined) || evidenceImages,
+ });
+ }
}
return {
@@ -1018,6 +1169,50 @@ export class TaskExecutor {
uiContext?: UIContext;
},
): Promise> {
+ const assertEvidence =
+ type === 'Assert' ? normalizeAssertEvidenceOptions(opt) : undefined;
+ const assertionText =
+ type === 'Assert'
+ ? typeof demand === 'string'
+ ? demand
+ : JSON.stringify(demand)
+ : undefined;
+ let assertOpt = opt;
+ if (type === 'Assert' && assertEvidence?.deepAssert) {
+ const selectedExecutions = recentAssertionExecutions(
+ this.getAssertionExecutions?.() ?? [],
+ assertEvidence.BeforeExecutions,
+ assertEvidence.AssertionContextBoundary,
+ );
+ const evaluationContext = buildEvaluationContext(
+ selectedExecutions,
+ assertEvidence.BeforeTasks,
+ );
+ const assertionEvidenceImages = selectAssertionEvidenceImages(
+ selectedExecutions,
+ assertEvidence.MaxPictures,
+ );
+ assertOpt = {
+ ...opt,
+ deepAssert: true,
+ AssertionContextBoundary: assertEvidence.AssertionContextBoundary,
+ BeforeExecutions: assertEvidence.BeforeExecutions,
+ BeforeTasks: assertEvidence.BeforeTasks,
+ MaxPictures: assertEvidence.MaxPictures,
+ evaluationContext,
+ assertionEvidenceImages,
+ };
+ } else if (type === 'Assert' && assertEvidence) {
+ assertOpt = {
+ ...opt,
+ deepAssert: false,
+ AssertionContextBoundary: assertEvidence.AssertionContextBoundary,
+ BeforeExecutions: assertEvidence.BeforeExecutions,
+ BeforeTasks: assertEvidence.BeforeTasks,
+ MaxPictures: assertEvidence.MaxPictures,
+ };
+ }
+
const session = this.createExecutionSession(
taskTitleStr(
type,
@@ -1033,11 +1228,39 @@ export class TaskExecutor {
const runner = session.getRunner();
const executionModelRuntime = { ...modelRuntime, executionId: runner.id };
+
+ if (
+ type === 'Assert' &&
+ assertEvidence?.deepAssert &&
+ assertEvidence.MaxPictures === 0
+ ) {
+ const failedAssertTask: ExecutionTaskInsightQueryApply = {
+ type: 'Insight',
+ subType: 'Assert',
+ param: {
+ assertion: assertionText,
+ dataDemand: demand,
+ deepAssert: true,
+ AssertionContextBoundary: assertEvidence.AssertionContextBoundary,
+ BeforeExecutions: assertEvidence.BeforeExecutions,
+ BeforeTasks: assertEvidence.BeforeTasks,
+ MaxPictures: 0,
+ evaluationContext: asEvaluationContext(assertOpt?.evaluationContext),
+ assertionEvidenceImages: [],
+ },
+ executor: async () => {
+ throw new Error('MaxPictures is 0');
+ },
+ };
+ await session.appendAndRun(failedAssertTask);
+ throw new Error('MaxPictures is 0');
+ }
+
const queryTask = await this.createTypeQueryTask(
type,
demand,
executionModelRuntime,
- opt,
+ assertOpt,
multimodalPrompt,
executionOptions,
);
diff --git a/packages/core/src/agent/ui-observer.ts b/packages/core/src/agent/ui-observer.ts
index 55e339c2d7..503fa4daac 100644
--- a/packages/core/src/agent/ui-observer.ts
+++ b/packages/core/src/agent/ui-observer.ts
@@ -156,7 +156,7 @@ export class UIObservationImpl implements UIObservation {
async aiAssert(
assertion: TUserPrompt,
- message?: string,
+ message?: string | ObservationAssertOptions,
options?: ObservationAssertOptions,
): Promise {
this.ensureUsable();
diff --git a/packages/core/src/ai-model/workflows/insight/extraction.ts b/packages/core/src/ai-model/workflows/insight/extraction.ts
index 2e0013f1b9..31b0d15b7d 100644
--- a/packages/core/src/ai-model/workflows/insight/extraction.ts
+++ b/packages/core/src/ai-model/workflows/insight/extraction.ts
@@ -1,4 +1,8 @@
-import type { ServiceExtractOption, UIContext } from '@/types';
+import type {
+ AssertionEvidenceImage,
+ ServiceExtractOption,
+ UIContext,
+} from '@/types';
import { getDebug } from '@midscene/shared/logger';
import type {
ChatCompletionSystemMessageParam,
@@ -36,10 +40,17 @@ export async function AiExtractElementInfo(options: {
}) {
const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =
options;
+ const evidenceImages = Array.isArray(extractOption?.assertionEvidenceImages)
+ ? (extractOption.assertionEvidenceImages as AssertionEvidenceImage[])
+ : [];
+ const useEvidenceImages =
+ extractOption?.deepAssert !== false && evidenceImages.length > 0;
const insightProtocol = modelRuntime.adapter.insight.protocol;
const systemPrompt = buildInsightSystemPrompt({
- screenshotIncluded: extractOption?.screenshotIncluded !== false,
- referenceImagesIncluded: !!multimodalPrompt?.images?.length,
+ screenshotIncluded:
+ extractOption?.screenshotIncluded !== false && !useEvidenceImages,
+ referenceImagesIncluded:
+ !!multimodalPrompt?.images?.length || useEvidenceImages,
insightProtocol,
});
const screenshotBase64 = context.screenshot.base64;
@@ -51,7 +62,25 @@ export async function AiExtractElementInfo(options: {
const userContent: ChatCompletionUserMessageParam['content'] = [];
- if (extractOption?.screenshotIncluded !== false) {
+ if (useEvidenceImages) {
+ userContent.push({
+ type: 'text',
+ text: `The following ${evidenceImages.length} images are assertion evidence-chain screenshots, ordered newest first. Image names match assertionEvidenceImages. Use them as the primary visual evidence.`,
+ });
+ evidenceImages.forEach((image, index) => {
+ userContent.push({
+ type: 'text',
+ text: `Evidence ${index + 1}/${evidenceImages.length}: ${image.name}`,
+ });
+ userContent.push({
+ type: 'image_url',
+ image_url: {
+ url: image.url,
+ detail: 'high',
+ },
+ });
+ });
+ } else if (extractOption?.screenshotIncluded !== false) {
const screenshotSequence = context.screenshotSequence;
if (screenshotSequence && screenshotSequence.length > 1) {
userContent.push({
diff --git a/packages/core/src/task-runner.ts b/packages/core/src/task-runner.ts
index 0d8bb50245..de5f586b52 100644
--- a/packages/core/src/task-runner.ts
+++ b/packages/core/src/task-runner.ts
@@ -1,3 +1,9 @@
+import {
+ DEFAULT_AFTER_FRAMES,
+ DEFAULT_FRAME_INTERVAL_MS,
+ isActionSpaceTask,
+ nonNegative,
+} from '@/agent/assertion-evidence';
import type { ScreenshotItem } from '@/screenshot-item';
import { setTimingFieldOnce } from '@/task-timing';
import {
@@ -13,6 +19,7 @@ import {
type PlanningActionParamError,
type UIContext,
} from '@/types';
+import { sleep } from '@/utils';
import {
type SerializedError,
serializeError,
@@ -95,6 +102,8 @@ export interface ExecutionReferenceImage {
type TaskRunnerInitOptions = ExecutionTaskProgressOptions & {
tasks?: ExecutionTaskApply[];
referenceImages?: readonly ExecutionReferenceImage[];
+ actionEvidenceAfterFrameCount?: number;
+ actionEvidenceFrameIntervalMs?: number;
/**
* Coarse "the execution snapshot changed" signal. Fires on any state change
* (append, status flips, completion) with the whole runner, so consumers can
@@ -135,6 +144,10 @@ export class TaskRunner {
private readonly referenceImageUrls = new Set();
+ private readonly actionEvidenceAfterFrameCount: number;
+
+ private readonly actionEvidenceFrameIntervalMs: number;
+
constructor(
name: string,
uiContextBuilder: () => Promise,
@@ -152,6 +165,14 @@ export class TaskRunner {
this.onSnapshotChange = options?.onSnapshotChange;
this.onTaskEvent = options?.onTaskEvent;
this.executionLogTime = Date.now();
+ this.actionEvidenceAfterFrameCount = nonNegative(
+ options?.actionEvidenceAfterFrameCount,
+ DEFAULT_AFTER_FRAMES,
+ );
+ this.actionEvidenceFrameIntervalMs = nonNegative(
+ options?.actionEvidenceFrameIntervalMs,
+ DEFAULT_FRAME_INTERVAL_MS,
+ );
for (const image of options?.referenceImages ?? []) {
this.referenceImageUrls.add(image.url);
}
@@ -229,7 +250,7 @@ export class TaskRunner {
private attachRecorderItem(
task: ExecutionTask,
screenshot: ScreenshotItem | undefined,
- phase: 'after-calling',
+ phase: string,
): void {
if (!phase || !screenshot) {
return;
@@ -249,6 +270,20 @@ export class TaskRunner {
task.recorder.push(recorderItem);
}
+ private async capturePostActionFrames(task: ExecutionTask): Promise {
+ for (
+ let index = 0;
+ index < this.actionEvidenceAfterFrameCount;
+ index++
+ ) {
+ if (index > 0 && this.actionEvidenceFrameIntervalMs > 0) {
+ await sleep(this.actionEvidenceFrameIntervalMs);
+ }
+ const screenshot = await this.captureScreenshot();
+ this.attachRecorderItem(task, screenshot, `after-calling-${index + 1}`);
+ }
+ }
+
private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {
return {
taskId: uuid(),
@@ -370,6 +405,9 @@ export class TaskRunner {
setTimingFieldOnce(task.timing, 'getUiContextEnd');
task.uiContext = uiContext;
+ if (isActionSpaceTask(task)) {
+ this.attachRecorderItem(task, uiContext?.screenshot, 'before-calling');
+ }
const executorContext: ExecutorContext = {
task,
element: previousFindOutput?.element,
@@ -410,9 +448,15 @@ export class TaskRunner {
returnValue = await task.executor(executorContext);
}
+ if (isActionSpaceTask(task)) {
+ setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');
+ await this.capturePostActionFrames(task);
+ setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');
+ }
+
const isLastTask = taskIndex === this.tasks.length - 1;
- if (isLastTask) {
+ if (isLastTask && !isActionSpaceTask(task)) {
setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');
const screenshot = await this.captureScreenshot();
this.attachRecorderItem(task, screenshot, 'after-calling');
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index 2f4b6dadd7..9aad757cfe 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -300,10 +300,37 @@ export interface AgentWaitForOpt extends ServiceExtractOption {
timeoutMs?: number;
}
+export type AssertionBoundary = 'lastAssert' | 'session';
+
+export interface AssertionEvidenceImage {
+ name: string;
+ url: string;
+ capturedAt?: number;
+}
+
+export interface AssertionEvaluationContext {
+ summary: string;
+ executions: Array<{ executionId?: string; title: string }>;
+ tasks: Array<{
+ executionTitle: string;
+ taskId?: string;
+ type: string;
+ subType?: string;
+ prompt: string;
+ status: string;
+ failureReason?: string;
+ }>;
+}
+
export interface AgentAssertOpt {
keepRawResponse?: boolean;
context?: string;
abortSignal?: AbortSignal;
+ deepAssert?: boolean;
+ AssertionContextBoundary?: AssertionBoundary;
+ BeforeExecutions?: number;
+ BeforeTasks?: number;
+ MaxPictures?: number;
}
export interface AgentAssertResult {
@@ -340,7 +367,7 @@ export interface InsightAPI<
aiAsk(prompt: TUserPrompt, options?: QueryOpt): Promise;
aiAssert(
assertion: TUserPrompt,
- message?: string,
+ message?: string | AssertOpt,
options?: AssertOpt,
): Promise;
}
@@ -685,8 +712,17 @@ task - service-query
*/
export interface ExecutionTaskInsightQueryParam {
dataDemand: ServiceExtractParam;
+ assertion?: string;
domIncluded?: boolean | 'visible-only';
context?: string;
+ deepAssert?: boolean;
+ AssertionContextBoundary?: AssertionBoundary;
+ BeforeExecutions?: number;
+ BeforeTasks?: number;
+ MaxPictures?: number;
+ evaluationContext?: AssertionEvaluationContext;
+ assertionEvidenceFallback?: 'currentScreenshot';
+ assertionEvidenceImages?: AssertionEvidenceImage[];
}
export interface ExecutionTaskInsightQueryOutput {
@@ -708,6 +744,14 @@ task - assertion
*/
export interface ExecutionTaskInsightAssertionParam {
assertion: string;
+ deepAssert?: boolean;
+ AssertionContextBoundary?: AssertionBoundary;
+ BeforeExecutions?: number;
+ BeforeTasks?: number;
+ MaxPictures?: number;
+ evaluationContext?: AssertionEvaluationContext;
+ assertionEvidenceFallback?: 'currentScreenshot';
+ assertionEvidenceImages?: AssertionEvidenceImage[];
}
export type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<
diff --git a/packages/core/tests/unit-test/assertion-evidence-execution.test.ts b/packages/core/tests/unit-test/assertion-evidence-execution.test.ts
new file mode 100644
index 0000000000..81677e8c81
--- /dev/null
+++ b/packages/core/tests/unit-test/assertion-evidence-execution.test.ts
@@ -0,0 +1,240 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { TaskExecutor } from '@/agent/tasks';
+import { getModelRuntime } from '@/ai-model/models';
+import { ScreenshotItem } from '@/screenshot-item';
+import type { ExecutionTask, IExecutionDump, ServiceDump } from '@/types';
+import { describe, expect, it, rs } from '@rstest/core';
+
+const runtimeDumpPath = resolve(
+ process.cwd(),
+ 'tests/unit-test/fixtures/deep-assert-runtime-dump.json',
+);
+
+const createMockUIContext = async (screenshotData = 'mock-screenshot') => ({
+ screenshot: ScreenshotItem.create(screenshotData, Date.now()),
+ shotSize: { width: 1920, height: 1080 },
+ shrunkShotToLogicalRatio: 1,
+});
+
+const createMockDump = (data: unknown, thought?: string): ServiceDump => ({
+ logTime: Date.now(),
+ type: 'extract',
+ logId: 'mock-log-id',
+ userQuery: {},
+ data,
+ taskInfo: {
+ durationMs: 100,
+ rawResponse: JSON.stringify(data),
+ reasoning_content: thought,
+ },
+});
+
+const shot = (value: string, capturedAt: number) =>
+ ScreenshotItem.create(`data:image/png;base64,${value}`, capturedAt);
+
+const actionTask = (
+ id: string,
+ frames: Array<{ timing: string; screenshot: ScreenshotItem }>,
+): ExecutionTask =>
+ ({
+ taskId: id,
+ type: 'Action Space',
+ subType: 'Tap',
+ status: 'finished',
+ thought: 'tap submit',
+ param: { description: 'submit button' },
+ recorder: frames.map((frame) => ({
+ type: 'screenshot',
+ ts: frame.screenshot.capturedAt,
+ screenshot: frame.screenshot,
+ timing: frame.timing,
+ })),
+ }) as ExecutionTask;
+
+const planningTask = (id: string): ExecutionTask =>
+ ({
+ taskId: id,
+ type: 'Planning',
+ subType: 'Plan',
+ status: 'finished',
+ param: { userInstructionDisplay: 'submit the form' },
+ output: {
+ shouldContinuePlanning: false,
+ actions: [{ type: 'Tap' }],
+ },
+ }) as ExecutionTask;
+
+const modelRuntime = getModelRuntime({
+ modelName: 'mock-model',
+ modelDescription: 'mock-model-description',
+ intent: 'insight',
+ slot: 'insight',
+});
+
+const createExecutor = (
+ history: IExecutionDump[],
+ thought =
+ '当前界面判断\n表单已提交。\n关联 task 分析\n占位\n截图证据分析\n前后界面发生变化。\n最终结论\n断言成立。',
+) => {
+ const extract = rs.fn(async () => ({
+ data: { StatementIsTruthy: true },
+ thought,
+ dump: createMockDump({ StatementIsTruthy: true }, thought),
+ }));
+ const taskExecutor = new TaskExecutor({} as never, {
+ contextRetrieverFn: rs.fn(async () => createMockUIContext('current')),
+ extract,
+ } as never, {
+ actionSpace: [],
+ getAssertionExecutions: () => history,
+ });
+ return { taskExecutor, extract };
+};
+
+describe('assertion evidence execution', () => {
+ it('accepts evidence-chain options as the second argument and persists them', async () => {
+ const { taskExecutor } = createExecutor([]);
+ const { runner } = await taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'state is correct',
+ modelRuntime,
+ {
+ deepAssert: true,
+ BeforeTasks: 2,
+ MaxPictures: 2,
+ },
+ );
+ expect(runner.tasks[0]?.param).toMatchObject({
+ assertion: 'state is correct',
+ deepAssert: true,
+ BeforeTasks: 2,
+ MaxPictures: 2,
+ });
+ });
+
+ it('passes newest unique action evidence to the model', async () => {
+ const beforeShot = shot('before', 1);
+ const afterShot = shot('after', 2);
+ const history = [
+ {
+ id: 'act-1',
+ name: 'Act - submit',
+ tasks: [
+ planningTask('plan-1'),
+ actionTask('tap-1', [
+ { timing: 'before-calling', screenshot: beforeShot },
+ { timing: 'after-calling-1', screenshot: afterShot },
+ ]),
+ ],
+ },
+ ];
+ const { taskExecutor, extract } = createExecutor(history);
+ const { runner } = await taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'state is correct',
+ modelRuntime,
+ { deepAssert: true, MaxPictures: 2 },
+ );
+ expect(
+ runner.tasks[0]?.param?.assertionEvidenceImages?.map((image) => image.url),
+ ).toEqual([afterShot.base64, beforeShot.base64]);
+ expect(extract.mock.calls[0]?.[2]?.assertionEvidenceImages?.[0]?.url).toBe(
+ afterShot.base64,
+ );
+ });
+
+ it('does not build an evidence chain when deepAssert is false', async () => {
+ const history = [
+ {
+ id: 'act-1',
+ name: 'Act - submit',
+ tasks: [
+ actionTask('tap-1', [
+ { timing: 'before-calling', screenshot: shot('before', 1) },
+ ]),
+ ],
+ },
+ ];
+ const { taskExecutor } = createExecutor(history);
+ const { runner } = await taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'state is correct',
+ modelRuntime,
+ { deepAssert: false },
+ );
+ expect(runner.tasks[0]?.param?.deepAssert).toBe(false);
+ expect(runner.tasks[0]?.param?.assertionEvidenceImages).toBeUndefined();
+ });
+
+ it('fails the assert task when MaxPictures is 0', async () => {
+ const { taskExecutor } = createExecutor([]);
+ await expect(
+ taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'state is correct',
+ modelRuntime,
+ { deepAssert: true, MaxPictures: 0 },
+ ),
+ ).rejects.toThrow(/MaxPictures is 0/);
+ });
+
+ it('falls back to the current screenshot when there is no history', async () => {
+ const { taskExecutor } = createExecutor([]);
+ const { runner } = await taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'state is correct',
+ modelRuntime,
+ { deepAssert: true, MaxPictures: 2 },
+ );
+ expect(runner.tasks[0]?.param?.assertionEvidenceFallback).toBe(
+ 'currentScreenshot',
+ );
+ expect(runner.tasks[0]?.param?.assertionEvidenceImages).toEqual([]);
+ });
+
+ it('writes a full AfterActPictures=3 MaxPictures=4 dump for report verification', async () => {
+ const frames = [1, 2, 3, 4].map((index) =>
+ shot(`frame-${index}`, index * 10),
+ );
+ const history: IExecutionDump[] = [
+ {
+ id: 'act-1',
+ name: 'Act - submit form',
+ tasks: [
+ planningTask('plan-1'),
+ actionTask('tap-1', [
+ { timing: 'before-calling', screenshot: frames[0] },
+ { timing: 'after-calling-1', screenshot: frames[1] },
+ { timing: 'after-calling-2', screenshot: frames[2] },
+ { timing: 'after-calling-3', screenshot: frames[3] },
+ ]),
+ ],
+ },
+ ];
+ const { taskExecutor } = createExecutor(history);
+ const { runner } = await taskExecutor.createTypeQueryExecution(
+ 'Assert',
+ 'the form is submitted',
+ modelRuntime,
+ {
+ deepAssert: true,
+ BeforeTasks: 2,
+ MaxPictures: 4,
+ AssertionContextBoundary: 'session',
+ },
+ );
+
+ const dump = {
+ executions: [history[0], runner.dump().toJSON()],
+ };
+ mkdirSync(dirname(runtimeDumpPath), { recursive: true });
+ writeFileSync(runtimeDumpPath, JSON.stringify(dump, null, 2), 'utf8');
+
+ expect(runner.tasks[0]?.param?.BeforeTasks).toBe(2);
+ expect(runner.tasks[0]?.param?.assertionEvidenceImages?.length).toBe(4);
+ expect(runner.tasks[0]?.thought).toContain('当前界面判断');
+ expect(runner.tasks[0]?.thought).toContain('Task 1:');
+ expect(runner.tasks[0]?.thought).toContain('Task 2:');
+ });
+});
diff --git a/packages/core/tests/unit-test/assertion-evidence.test.ts b/packages/core/tests/unit-test/assertion-evidence.test.ts
new file mode 100644
index 0000000000..79a967667a
--- /dev/null
+++ b/packages/core/tests/unit-test/assertion-evidence.test.ts
@@ -0,0 +1,269 @@
+import {
+ buildDeepAssertScreenshots,
+ buildEvaluationContext,
+ citationForContextTask,
+ composeAssertThought,
+ deepAssertEvidence,
+ normalizeActionEvidenceOptions,
+ normalizeAssertEvidenceOptions,
+ recentAssertionExecutions,
+ resolveAssertCallArgs,
+ selectAssertionEvidenceImages,
+} from '@/agent/assertion-evidence';
+import { ScreenshotItem } from '@/screenshot-item';
+import type { ExecutionTask, IExecutionDump } from '@/types';
+import { describe, expect, it } from '@rstest/core';
+
+const shot = (base64: string, capturedAt: number) =>
+ ScreenshotItem.create(base64, capturedAt);
+
+const actionTask = (
+ recorder: Array<{ timing: string; screenshot: ScreenshotItem }>,
+ extra?: Partial,
+): ExecutionTask =>
+ ({
+ taskId: extra?.taskId || 'action-1',
+ type: 'Action Space',
+ subType: extra?.subType || 'Tap',
+ status: extra?.status || 'finished',
+ thought: extra?.thought || 'tap submit',
+ param: extra?.param || { description: 'submit button' },
+ recorder: recorder.map((item) => ({
+ type: 'screenshot' as const,
+ ts: item.screenshot.capturedAt,
+ screenshot: item.screenshot,
+ timing: item.timing,
+ })),
+ ...extra,
+ }) as ExecutionTask;
+
+const planningTask = (extra?: Partial): ExecutionTask =>
+ ({
+ taskId: extra?.taskId || 'plan-1',
+ type: 'Planning',
+ subType: extra?.subType || 'Plan',
+ status: extra?.status || 'finished',
+ param: extra?.param || { userInstructionDisplay: 'open settings' },
+ output: extra?.output || {
+ shouldContinuePlanning: false,
+ actions: [{ type: 'Tap' }],
+ },
+ ...extra,
+ }) as ExecutionTask;
+
+const assertTask = (extra?: Partial): ExecutionTask =>
+ ({
+ taskId: extra?.taskId || 'assert-1',
+ type: 'Insight',
+ subType: 'Assert',
+ status: extra?.status || 'finished',
+ param: extra?.param || { assertion: 'done' },
+ ...extra,
+ }) as ExecutionTask;
+
+const execution = (
+ id: string,
+ name: string,
+ tasks: ExecutionTask[],
+): IExecutionDump => ({
+ id,
+ name,
+ tasks,
+});
+
+describe('assertion evidence options', () => {
+ it('accepts evidence-chain options as the second argument', () => {
+ const resolved = resolveAssertCallArgs({
+ deepAssert: true,
+ BeforeTasks: 2,
+ MaxPictures: 2,
+ });
+ expect(resolved.message).toBeUndefined();
+ expect(resolved.options).toMatchObject({
+ deepAssert: true,
+ BeforeTasks: 2,
+ MaxPictures: 2,
+ });
+ });
+
+ it('keeps the legacy three-argument form', () => {
+ const resolved = resolveAssertCallArgs('failed', {
+ deepAssert: false,
+ });
+ expect(resolved.message).toBe('failed');
+ expect(resolved.options).toMatchObject({ deepAssert: false });
+ });
+
+ it('normalizes illegal numbers to defaults', () => {
+ expect(
+ normalizeActionEvidenceOptions({
+ AfterActPictures: -1,
+ Interval: Number.NaN,
+ }),
+ ).toEqual({ AfterActPictures: 1, Interval: 50 });
+ expect(
+ normalizeAssertEvidenceOptions({
+ BeforeExecutions: -3,
+ BeforeTasks: Number.POSITIVE_INFINITY,
+ MaxPictures: -8,
+ AssertionContextBoundary: 'other' as never,
+ }),
+ ).toEqual({
+ deepAssert: true,
+ AssertionContextBoundary: 'lastAssert',
+ BeforeExecutions: 1,
+ BeforeTasks: 1,
+ MaxPictures: 2,
+ });
+ });
+});
+
+describe('assertion evidence history and images', () => {
+ it('passes newest unique action evidence to the model', () => {
+ const beforeShot = shot('data:image/png;base64,before', 1);
+ const afterShot = shot('data:image/png;base64,after', 2);
+ const planningShot = shot('data:image/png;base64,planning', 3);
+ const history = [
+ execution('e1', 'Act - submit', [
+ {
+ ...planningTask(),
+ recorder: [
+ {
+ type: 'screenshot',
+ ts: 3,
+ screenshot: planningShot,
+ timing: 'after-calling',
+ },
+ ],
+ },
+ actionTask([
+ { timing: 'before-calling', screenshot: beforeShot },
+ { timing: 'after-calling-1', screenshot: afterShot },
+ ]),
+ ]),
+ ];
+
+ const images = selectAssertionEvidenceImages(history, 2);
+ expect(images.map((image) => image.url)).toEqual([
+ afterShot.base64,
+ beforeShot.base64,
+ ]);
+ expect(images.every((image) => /before-calling|after-calling-1$/.test(image.name))).toBe(
+ true,
+ );
+ });
+
+ it('uses lastAssert and session boundaries', () => {
+ const older = execution('old', 'Act - older', [
+ actionTask([{ timing: 'before-calling', screenshot: shot('old', 1) }], {
+ taskId: 'old-action',
+ }),
+ ]);
+ const previousAssert = execution('asserted', 'Assert - previous', [
+ assertTask({ taskId: 'prev-assert' }),
+ ]);
+ const newer = execution('new', 'Act - newer', [
+ actionTask([{ timing: 'before-calling', screenshot: shot('new', 2) }], {
+ taskId: 'new-action',
+ }),
+ ]);
+
+ const lastAssert = recentAssertionExecutions(
+ [older, previousAssert, newer],
+ 2,
+ 'lastAssert',
+ );
+ expect(lastAssert.map((item) => item.id)).toEqual(['new']);
+
+ const session = recentAssertionExecutions(
+ [older, previousAssert, newer],
+ 2,
+ 'session',
+ );
+ expect(session.map((item) => item.id)).toEqual(['asserted', 'new']);
+ });
+
+ it('keeps a contiguous recent task window and skips final planning summaries', () => {
+ const history = [
+ execution('e1', 'Act - flow', [
+ planningTask({
+ taskId: 'final-plan',
+ output: { shouldContinuePlanning: false, actions: [] },
+ }),
+ planningTask({ taskId: 'locate', subType: 'Locate' }),
+ actionTask(
+ [{ timing: 'before-calling', screenshot: shot('a', 1) }],
+ { taskId: 'tap' },
+ ),
+ ]),
+ ];
+ const context = buildEvaluationContext(history, 2);
+ expect(context.tasks.map((task) => task.taskId)).toEqual(['locate', 'tap']);
+ });
+});
+
+describe('assertion evidence report helpers', () => {
+ it('returns undefined for disabled or legacy asserts', () => {
+ expect(
+ deepAssertEvidence(
+ assertTask({ param: { assertion: 'x', deepAssert: false } }),
+ ),
+ ).toBeUndefined();
+ expect(deepAssertEvidence(assertTask({ param: { assertion: 'x' } }))).toBeUndefined();
+ });
+
+ it('reverses model images for report display', () => {
+ const screenshots = buildDeepAssertScreenshots(
+ assertTask({
+ param: {
+ assertion: 'state is correct',
+ deepAssert: true,
+ assertionEvidenceImages: [
+ { name: 'Act / Tap / after-calling-1', url: 'after', capturedAt: 2 },
+ { name: 'Act / Tap / before-calling', url: 'before', capturedAt: 1 },
+ ],
+ },
+ }),
+ );
+ expect(screenshots).toEqual([
+ {
+ screenshot: 'before',
+ timing: '参考图1 / Act / Tap / before-calling',
+ screenshotTimestamp: 1,
+ },
+ {
+ screenshot: 'after',
+ timing: '参考图2 / Act / Tap / after-calling-1',
+ screenshotTimestamp: 2,
+ },
+ ]);
+ });
+
+ it('writes exact task citations into the four analysis sections', () => {
+ const context = buildEvaluationContext(
+ [
+ execution('e1', 'Act - submit', [
+ actionTask(
+ [{ timing: 'before-calling', screenshot: shot('a', 1) }],
+ { taskId: 'tap', thought: 'tap submit' },
+ ),
+ ]),
+ ],
+ 1,
+ );
+ const thought = composeAssertThought({
+ assertion: 'state is correct',
+ passed: true,
+ evaluationContext: context,
+ evidenceImages: [
+ { name: 'Act - submit / Tap / before-calling', url: 'a', capturedAt: 1 },
+ ],
+ });
+ expect(thought).toContain('当前界面判断');
+ expect(thought).toContain('关联 task 分析');
+ expect(thought).toContain('截图证据分析');
+ expect(thought).toContain('最终结论');
+ expect(thought).toContain(citationForContextTask(context.tasks[0], 1));
+ expect(thought).not.toContain('DATA_DEMAND');
+ });
+});
diff --git a/packages/core/tests/unit-test/insight-extract-prompt.test.ts b/packages/core/tests/unit-test/insight-extract-prompt.test.ts
index 67d17bc03b..7bd795ee93 100644
--- a/packages/core/tests/unit-test/insight-extract-prompt.test.ts
+++ b/packages/core/tests/unit-test/insight-extract-prompt.test.ts
@@ -145,6 +145,44 @@ describe('insight extraction prompt assembly', () => {
});
});
+ it('sends evidence-chain images newest first instead of the current screenshot', async () => {
+ const context = createFakeContext();
+
+ await AiExtractElementInfo<{ result: boolean }>({
+ context,
+ dataQuery: {
+ StatementIsTruthy: 'Boolean, whether the form is submitted',
+ },
+ extractOption: {
+ deepAssert: true,
+ assertionEvidenceImages: [
+ {
+ name: 'Act / Tap / after-calling-1',
+ url: 'data:image/png;base64,after',
+ capturedAt: 2,
+ },
+ {
+ name: 'Act / Tap / before-calling',
+ url: 'data:image/png;base64,before',
+ capturedAt: 1,
+ },
+ ],
+ },
+ modelRuntime: getModelRuntime(modelConfig),
+ });
+
+ const msgs = rs.mocked(callAI).mock.calls[0]?.[0];
+ const userContent = msgs?.[1]?.content as Array>;
+ const imageUrls = userContent
+ .filter((item) => item.type === 'image_url')
+ .map((item) => item.image_url.url);
+ expect(imageUrls).toEqual([
+ 'data:image/png;base64,after',
+ 'data:image/png;base64,before',
+ ]);
+ expect(imageUrls).not.toContain(context.screenshot.base64);
+ });
+
it('passes abortSignal to the AI caller', async () => {
const abortController = new AbortController();
diff --git a/packages/core/tests/unit-test/task-runner/index.test.ts b/packages/core/tests/unit-test/task-runner/index.test.ts
index 97c9ded894..f8460f0ca2 100644
--- a/packages/core/tests/unit-test/task-runner/index.test.ts
+++ b/packages/core/tests/unit-test/task-runner/index.test.ts
@@ -667,5 +667,37 @@ describe(
expect(caughtError?.task?.thought).toMatch(/… \[truncated\]$/);
expect(caughtError?.task?.errorMessage).toMatch(/… \[truncated\]$/);
});
+
+ it('records before and numbered post-action frames only for action tasks', async () => {
+ const actionTask: ExecutionTaskActionApply = {
+ type: 'Action Space',
+ subType: 'Tap',
+ param: { description: 'submit' },
+ executor: rs.fn(),
+ };
+ const planningTask: ExecutionTaskApply = {
+ type: 'Planning',
+ subType: 'Plan',
+ param: { userInstruction: 'do it' },
+ executor: rs.fn(async () => ({
+ output: { actions: [{ type: 'Tap' }], shouldContinuePlanning: false },
+ })),
+ };
+ const runner = new TaskRunner('test', fakeUIContextBuilder, {
+ tasks: [planningTask, actionTask],
+ actionEvidenceAfterFrameCount: 2,
+ actionEvidenceFrameIntervalMs: 0,
+ });
+ await runner.flush();
+
+ expect(
+ runner.tasks[1].recorder?.map((item) => item.timing),
+ ).toEqual(['before-calling', 'after-calling-1', 'after-calling-2']);
+ expect(
+ runner.tasks[0].recorder?.some((item) =>
+ String(item.timing).startsWith('after-calling-'),
+ ),
+ ).toBeFalsy();
+ });
},
);
diff --git a/packages/core/tests/unit-test/tasks-null-data.test.ts b/packages/core/tests/unit-test/tasks-null-data.test.ts
index b7e954b337..ee1af64707 100644
--- a/packages/core/tests/unit-test/tasks-null-data.test.ts
+++ b/packages/core/tests/unit-test/tasks-null-data.test.ts
@@ -246,7 +246,7 @@ describe('TaskExecutor - Null Data Handling', () => {
} as any),
).resolves.toMatchObject({
output: false,
- thought: 'Could not verify assertion',
+ thought: expect.stringContaining('Could not verify assertion'),
});
expect(queryTask.log).toMatchObject({
@@ -264,7 +264,10 @@ describe('TaskExecutor - Null Data Handling', () => {
'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, whether the following statement is true: Page title is correct',
},
getModelRuntime(mockModelConfig),
- {},
+ expect.objectContaining({
+ deepAssert: true,
+ assertionEvidenceFallback: 'currentScreenshot',
+ }),
'',
undefined,
expectEmptyUIContext(),
diff --git a/packages/test/src/midscene/index.ts b/packages/test/src/midscene/index.ts
index 7b9e448272..e9634afdeb 100644
--- a/packages/test/src/midscene/index.ts
+++ b/packages/test/src/midscene/index.ts
@@ -18,6 +18,8 @@ export interface MidsceneAiActOptions {
deepLocate?: boolean;
context?: string;
abortSignal?: AbortSignal;
+ AfterActPictures?: number;
+ Interval?: number;
}
export interface MidsceneAiAssertOptions {
@@ -26,6 +28,11 @@ export interface MidsceneAiAssertOptions {
context?: string;
abortSignal?: AbortSignal;
keepRawResponse?: boolean;
+ deepAssert?: boolean;
+ AssertionContextBoundary?: 'lastAssert' | 'session';
+ BeforeExecutions?: number;
+ BeforeTasks?: number;
+ MaxPictures?: number;
}
export interface MidscenePromptImage {
@@ -144,6 +151,14 @@ const aiActOptionsInputSchema = z.strictObject({
.string()
.optional()
.describe('Additional context supplied to the UI Agent.'),
+ AfterActPictures: z
+ .number()
+ .optional()
+ .describe('How many post-action screenshots to record.'),
+ Interval: z
+ .number()
+ .optional()
+ .describe('Interval in milliseconds between post-action screenshots.'),
});
export const aiActInputSchema = z.strictObject({
@@ -166,6 +181,26 @@ const aiAssertOptionsInputSchema = z.strictObject({
.string()
.optional()
.describe('Additional context supplied to the UI Agent.'),
+ deepAssert: z
+ .boolean()
+ .optional()
+ .describe('Whether the assertion uses the action evidence chain.'),
+ AssertionContextBoundary: z
+ .enum(['lastAssert', 'session'])
+ .optional()
+ .describe('How far back the assertion looks for historical tasks.'),
+ BeforeExecutions: z
+ .number()
+ .optional()
+ .describe('How many previous executions are included.'),
+ BeforeTasks: z
+ .number()
+ .optional()
+ .describe('How many previous tasks are included.'),
+ MaxPictures: z
+ .number()
+ .optional()
+ .describe('Maximum number of evidence images sent to the model.'),
});
export const aiAssertInputSchema = z.strictObject({
diff --git a/packages/visualizer/src/utils/replay-scripts.ts b/packages/visualizer/src/utils/replay-scripts.ts
index 6987967e4f..48e34cb9ee 100644
--- a/packages/visualizer/src/utils/replay-scripts.ts
+++ b/packages/visualizer/src/utils/replay-scripts.ts
@@ -1,6 +1,12 @@
'use client';
import { mousePointer } from '@/utils';
-import { paramStr, typeStr } from '@midscene/core/agent';
+import {
+ buildDeepAssertScreenshots,
+ deepAssertEvidence,
+ isCurrentScreenshotFallback,
+ paramStr,
+ typeStr,
+} from '@midscene/core/agent';
import { getTaskSearchArea } from '@midscene/core/dump/task-service-dump';
import {
getCenterHighlightBox,
@@ -354,6 +360,44 @@ export const allScriptsFromDump = (
};
};
+const buildDeepAssertReplayScripts = (
+ task: ExecutionTask,
+ imageWidth: number,
+ imageHeight: number,
+): AnimationScript[] | null => {
+ const evidence = deepAssertEvidence(task);
+ if (evidence === undefined) {
+ return null;
+ }
+ const screenshots = buildDeepAssertScreenshots(task);
+ const images =
+ screenshots && screenshots.length > 0
+ ? screenshots
+ : isCurrentScreenshotFallback(task) && task.uiContext?.screenshot?.base64
+ ? [
+ {
+ screenshot: task.uiContext.screenshot.base64,
+ timing: '参考图1 / current screenshot',
+ screenshotTimestamp: task.uiContext.screenshot.capturedAt,
+ },
+ ]
+ : [];
+ if (images.length === 0) {
+ return [];
+ }
+ return images.map((image) => ({
+ type: 'img' as const,
+ img: image.screenshot,
+ duration: stillDuration,
+ camera: createFullPageCameraState(imageWidth, imageHeight),
+ title: typeStr(task),
+ subTitle: image.timing,
+ imageWidth,
+ imageHeight,
+ taskId: task.taskId,
+ }));
+};
+
export const generateAnimationScripts = (
execution: ExecutionDump | IExecutionDump | null,
task: ExecutionTask | number,
@@ -366,6 +410,16 @@ export const generateAnimationScripts = (
return null;
}
+ const selectedTask =
+ task !== -1 && typeof task !== 'number'
+ ? task
+ : typeof task === 'number' && task >= 0
+ ? execution.tasks[task]
+ : undefined;
+ if (selectedTask && deepAssertEvidence(selectedTask) !== undefined) {
+ return buildDeepAssertReplayScripts(selectedTask, imageWidth, imageHeight);
+ }
+
let tasksIncluded: ExecutionTask[] = [];
let taskStartIndex = 0;
if (task === -1) {
@@ -463,6 +517,18 @@ export const generateAnimationScripts = (
initSubTitle = paramStr(task);
}
+ if (deepAssertEvidence(task) !== undefined) {
+ const evidenceScripts = buildDeepAssertReplayScripts(
+ task,
+ imageWidth,
+ imageHeight,
+ );
+ if (evidenceScripts?.length) {
+ scripts.push(...evidenceScripts);
+ }
+ return;
+ }
+
if (task.type === 'Planning') {
let locateElements: LocateResultElement[] = [];
if (task.subType === 'Plan') {
diff --git a/packages/visualizer/tests/replay-scripts-camera.test.ts b/packages/visualizer/tests/replay-scripts-camera.test.ts
index f666649f66..640d0b8a27 100644
--- a/packages/visualizer/tests/replay-scripts-camera.test.ts
+++ b/packages/visualizer/tests/replay-scripts-camera.test.ts
@@ -9,6 +9,9 @@ import { describe, expect, it, rs } from '@rstest/core';
rs.mock('@midscene/core/agent', () => ({
paramStr: () => '',
typeStr: (task: { type: string }) => task.type,
+ deepAssertEvidence: () => undefined,
+ buildDeepAssertScreenshots: () => undefined,
+ isCurrentScreenshotFallback: () => false,
}));
import {