diff --git a/packages/core/src/ai-model/models/doubao.ts b/packages/core/src/ai-model/models/doubao.ts deleted file mode 100644 index 64d0cdee68..0000000000 --- a/packages/core/src/ai-model/models/doubao.ts +++ /dev/null @@ -1,171 +0,0 @@ -import type { TModelFamily } from '@midscene/shared/env'; -import type { - ChatCompletionCallContext, - ChatCompletionParamsResult, - ModelAdapterDefinition, -} from '../model-adapter/types'; -import { parseModelResponseJson } from '../shared/json'; -import { - type LocateResultValue, - createLocateResultValue, - unwrapCoordinateListLikeInput, -} from '../shared/model-locate-result'; - -const doubaoBboxCoordinatesMeta = { - shape: 'bbox', - order: 'xy', - normalizedBy: 1000, - rounding: 'round', -} as const; -const doubaoPointCoordinatesMeta = { - shape: 'point', - order: 'xy', - normalizedBy: 1000, - rounding: 'round', -} as const; - -/** - * Finds a sequence of numbers separated by punctuation or whitespace in a - * string. Separators cannot be letters, which prevents extracting the `2` in - * a mixed alphanumeric token such as `bbox_2d` as a coordinate. - */ -const coordinateSequencePattern = - /(?:^|[^a-zA-Z0-9])(\d+(?:[^a-zA-Z0-9]+\d+)+)(?=$|[^a-zA-Z0-9])/g; - -function isFourFiniteNumberArray(input: unknown): input is number[] { - return ( - Array.isArray(input) && - input.length === 4 && - input.every((value) => typeof value === 'number' && Number.isFinite(value)) - ); -} - -function parseNumbersFromUnexpectedBboxStructure(input: unknown): number[] { - const serialized = JSON.stringify(input); - if (!serialized) { - return []; - } - - const sequences = Array.from( - serialized.matchAll(coordinateSequencePattern), - (match) => match[1].match(/\d+/g)?.map(Number) ?? [], - ); - const longestLength = Math.max( - 0, - ...sequences.map((sequence) => sequence.length), - ); - const longestSequences = sequences.filter( - (sequence) => sequence.length === longestLength, - ); - - if (longestSequences.length !== 1) { - return []; - } - - return longestSequences[0]; -} - -export function parseDoubaoRawLocateValue(input: unknown): LocateResultValue { - const bbox = unwrapCoordinateListLikeInput(input as any); - const bboxList = isFourFiniteNumberArray(bbox) - ? bbox - : parseNumbersFromUnexpectedBboxStructure(bbox); - - if (bboxList.length === 4 || bboxList.length === 5) { - return createLocateResultValue(doubaoBboxCoordinatesMeta, [ - bboxList[0], - bboxList[1], - bboxList[2], - bboxList[3], - ]); - } - - if ( - bboxList.length === 6 || - bboxList.length === 2 || - bboxList.length === 3 || - bboxList.length === 7 - ) { - return createLocateResultValue(doubaoPointCoordinatesMeta, [ - bboxList[0], - bboxList[1], - ]); - } - - if (bboxList.length === 8) { - return createLocateResultValue(doubaoBboxCoordinatesMeta, [ - bboxList[0], - bboxList[1], - bboxList[4], - bboxList[5], - ]); - } - - const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `; - throw new Error(msg); -} - -const buildDoubaoChatCompletionParams = ( - input: ChatCompletionCallContext, -): ChatCompletionParamsResult => { - const { midsceneDefaults, userConfig } = input; - const { reasoningEnabled, reasoningEffort } = userConfig; - const commonOverrideConfig: Record = {}; - - if (userConfig.temperature !== undefined) { - commonOverrideConfig.temperature = userConfig.temperature; - } - - // Doubao Chat Completions JSON mode: - // https://docs.volcengine.com/docs/82379/1568221?lang=zh - if ( - userConfig.responseFormat !== 'none' && - input.expectedJsonObjectResponse - ) { - commonOverrideConfig.response_format = { type: 'json_object' }; - } - - const modelSpecificConfig: Record = {}; - - if (reasoningEnabled !== 'default') { - modelSpecificConfig.thinking = { - type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled', - }; - if (reasoningEffort) { - modelSpecificConfig.reasoning_effort = reasoningEffort; - } - } - - return { - config: { - ...midsceneDefaults, - ...commonOverrideConfig, - ...modelSpecificConfig, - }, - }; -}; - -const doubaoVisionAdapter: ModelAdapterDefinition = { - jsonParser: parseModelResponseJson, - chatCompletion: { - unsupportedUserConfig: ['reasoningBudget'], - buildChatCompletionParams: buildDoubaoChatCompletionParams, - useReasoningAsContentFallback: true, - }, - locate: { - element: { - resultFormat: { - coordinates: doubaoBboxCoordinatesMeta, - parseRawLocateValue: parseDoubaoRawLocateValue, - }, - }, - }, -}; - -export const doubaoAdapters = { - 'doubao-vision': doubaoVisionAdapter, - 'doubao-seed': doubaoVisionAdapter, -} satisfies Pick< - Record, - 'doubao-vision' | 'doubao-seed' ->; diff --git a/packages/core/src/ai-model/models/doubao/action-output-parser.ts b/packages/core/src/ai-model/models/doubao/action-output-parser.ts new file mode 100644 index 0000000000..b362d7893c --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/action-output-parser.ts @@ -0,0 +1,167 @@ +import type { DeviceAction, PlanningAction } from '@/types'; +import { + isMidsceneLocatorField, + unwrapZodField, +} from '@midscene/shared/zod-schema-utils'; +import type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol'; +import type { JsonParser } from '../../shared/json'; +import { parseDoubaoToolCall } from './tool-call-parser'; + +const getActionParameterSchema = ( + actionSpace: DeviceAction[], + actionName: string, + parameterName: string, +) => { + const action = actionSpace.find(({ name }) => name === actionName); + if (!action?.paramSchema) { + return undefined; + } + + const schema = unwrapZodField(action.paramSchema) as { + _def?: { + typeName?: string; + shape?: () => Record; + }; + shape?: Record; + }; + if (schema?._def?.typeName !== 'ZodObject') { + return undefined; + } + + const shape = + typeof schema._def.shape === 'function' + ? schema._def.shape() + : schema.shape; + return shape?.[parameterName]; +}; + +const schemaUsesStringValue = (schema: unknown): boolean => { + if (!schema) { + return false; + } + + if (isMidsceneLocatorField(schema)) { + return true; + } + + const actualSchema = unwrapZodField(schema) as { + _def?: { + typeName?: string; + options?: unknown[]; + value?: unknown; + }; + }; + if ( + actualSchema._def?.typeName === 'ZodString' || + actualSchema._def?.typeName === 'ZodEnum' + ) { + return true; + } + + if (actualSchema._def?.typeName === 'ZodLiteral') { + return typeof actualSchema._def.value === 'string'; + } + + return ( + actualSchema._def?.typeName === 'ZodUnion' && + (actualSchema._def.options ?? []).some(schemaUsesStringValue) + ); +}; + +const parseDoubaoParameterValue = ( + content: string, + { + isString, + parameterSchema, + jsonParser, + }: { + isString: boolean | undefined; + parameterSchema: unknown; + jsonParser: JsonParser; + }, +) => { + if ( + isString === true || + (isString === undefined && schemaUsesStringValue(parameterSchema)) + ) { + return content; + } + + if (isString === undefined && !parameterSchema) { + throw new Error('cannot infer parameter type without an action schema'); + } + + return jsonParser(content, { + source: 'planning-action-param', + }); +}; + +export const parseDoubaoRawLocateParameter = ( + value: unknown, +): ParsedPlanningLocateParameter => { + if (typeof value !== 'string') { + throw new Error('Seed planning locator parameter must be a string'); + } + + const promptMatch = value.match(/([\s\S]*?)<\/prompt>/i); + const pointMatches = Array.from( + value.matchAll(/\s*\d+\s+\d+\s*<\/point>/gi), + ); + if (!promptMatch && pointMatches.length === 0) { + throw new Error( + 'Seed planning locator parameter requires or ', + ); + } + + const locateParameter: { + prompt?: string; + point?: string; + } = {}; + + if (promptMatch) { + locateParameter.prompt = promptMatch[1]; + } + if (pointMatches.length > 0) { + locateParameter.point = pointMatches.map((match) => match[0]).join(''); + } + + return locateParameter; +}; + +export const createDoubaoPlanningActionOutputParser = + (jsonParser: JsonParser) => + ( + content: string, + actionSpace: DeviceAction[], + ): PlanningAction | null => { + const toolCall = parseDoubaoToolCall(content); + if (!toolCall) { + return null; + } + + const param = Object.fromEntries( + toolCall.parameters.map(({ name, isString, rawValue }) => { + try { + return [ + name, + parseDoubaoParameterValue(rawValue, { + isString, + parameterSchema: getActionParameterSchema( + actionSpace, + toolCall.functionName, + name, + ), + jsonParser, + }), + ] as const; + } catch (error) { + throw new Error(`Failed to parse Seed parameter "${name}": ${error}`); + } + }), + ); + + return { + type: toolCall.functionName, + ...(Object.keys(param).length > 0 ? { param } : {}), + }; + }; diff --git a/packages/core/src/ai-model/models/doubao/action-space.ts b/packages/core/src/ai-model/models/doubao/action-space.ts new file mode 100644 index 0000000000..6a8f2adafe --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/action-space.ts @@ -0,0 +1,185 @@ +import { + getZodDescription, + isMidsceneLocatorField, + unwrapZodField, +} from '@midscene/shared/zod-schema-utils'; +import type { z } from 'zod'; +import type { PlanningActionDescriptionBuildInput } from '../../model-adapter/planning-protocol'; +import type { LocateResultPromptSpec } from '../../shared/model-locate-result'; + +type DoubaoJsonSchema = { + type?: string; + description?: string; + enum?: string[]; + items?: DoubaoJsonSchema; + properties?: Record; + required?: string[]; + anyOf?: DoubaoJsonSchema[]; +}; + +type DoubaoParameterDefinition = DoubaoJsonSchema & { + description: string; +}; + +export type DoubaoFunctionDefinition = { + type?: 'function'; + name: string; + description?: string; + parameters: { + type: 'object'; + properties: Record; + required: string[]; + }; +}; + +const getZodObjectShape = ( + schema: unknown, +): Record | undefined => { + if (!schema) { + return undefined; + } + + const actualSchema = unwrapZodField(schema) as { + _def?: { + typeName?: string; + shape?: () => Record; + }; + shape?: Record; + }; + if (actualSchema._def?.typeName !== 'ZodObject') { + return undefined; + } + return typeof actualSchema._def.shape === 'function' + ? actualSchema._def.shape() + : actualSchema.shape; +}; + +const buildDoubaoJsonSchema = (field: unknown): DoubaoJsonSchema => { + const schema = unwrapZodField(field) as { + _def?: { + typeName?: string; + checks?: Array<{ kind?: string }>; + options?: unknown[]; + type?: unknown; + values?: unknown; + }; + }; + + switch (schema._def?.typeName) { + case 'ZodString': + return { type: 'string' }; + case 'ZodEnum': { + const values = schema._def.values; + return { + type: 'string', + ...(Array.isArray(values) && + values.every((value) => typeof value === 'string') + ? { enum: values } + : {}), + }; + } + case 'ZodNumber': + return { + type: schema._def.checks?.some((check) => check.kind === 'int') + ? 'integer' + : 'number', + }; + case 'ZodBoolean': + return { type: 'boolean' }; + case 'ZodArray': + return { + type: 'array', + items: buildDoubaoJsonSchema(schema._def.type), + }; + case 'ZodObject': { + const shapeEntries = Object.entries(getZodObjectShape(schema) ?? {}); + return { + type: 'object', + properties: Object.fromEntries( + shapeEntries.map(([name, nestedField]) => { + const description = getZodDescription(nestedField); + return [ + name, + { + ...buildDoubaoJsonSchema(nestedField), + ...(description ? { description } : {}), + }, + ]; + }), + ), + required: shapeEntries + .filter( + ([, nestedField]) => + !( + typeof nestedField.isOptional === 'function' && + nestedField.isOptional() + ), + ) + .map(([name]) => name), + }; + } + case 'ZodUnion': + return { + anyOf: (schema._def.options ?? []).map(buildDoubaoJsonSchema), + }; + default: + return { type: 'object' }; + } +}; + +export const buildDoubaoActionDescription = ({ + action, + locateFieldDescription, +}: PlanningActionDescriptionBuildInput): DoubaoFunctionDefinition => { + const shapeEntries = Object.entries( + getZodObjectShape(action.paramSchema) ?? {}, + ); + const properties = Object.fromEntries( + shapeEntries.map(([name, field]) => { + const isLocator = isMidsceneLocatorField(field); + const fieldDescription = getZodDescription(field); + return [ + name, + { + ...(isLocator ? { type: 'string' } : buildDoubaoJsonSchema(field)), + description: isLocator + ? [ + fieldDescription || `Target element for ${action.name}.`, + locateFieldDescription, + ].join(' ') + : fieldDescription || `Parameter ${name} for ${action.name}.`, + }, + ]; + }), + ) as Record; + const required = shapeEntries + .filter( + ([, field]) => + !(typeof field.isOptional === 'function' && field.isOptional()), + ) + .map(([name]) => name); + + return { + name: action.name, + description: action.description || 'No description provided.', + parameters: { + type: 'object', + properties, + required, + }, + }; +}; + +export const buildDoubaoLocateFieldDescription = ( + locatePromptSpec?: LocateResultPromptSpec, +) => { + if (locatePromptSpec && locatePromptSpec.resultKey !== 'point') { + throw new Error( + 'Doubao planning protocol requires a point locate result adapter', + ); + } + + return locatePromptSpec + ? `The format is: element descriptionx y. ${locatePromptSpec.resultValueDescription}` + : 'The format is: element description.'; +}; diff --git a/packages/core/src/ai-model/models/doubao/adapter.ts b/packages/core/src/ai-model/models/doubao/adapter.ts new file mode 100644 index 0000000000..51455ba209 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/adapter.ts @@ -0,0 +1,113 @@ +import type { TModelFamily } from '@midscene/shared/env'; +import type { + ChatCompletionCallContext, + ChatCompletionParamsResult, + ModelAdapterDefinition, +} from '../../model-adapter/types'; +import { + type LocateResultValue, + createLocateResultValue, +} from '../../shared/model-locate-result'; +import { doubaoSearchAreaProtocol } from './area-protocol'; +import { doubaoElementProtocol } from './element-protocol'; +import { createDoubaoInsightProtocol } from './insight-protocol'; +import { createDoubaoPlanningProtocol } from './planning-protocol'; + +const doubaoPointCoordinates = { + shape: 'point', + order: 'xy', + normalizedBy: 1000, + rounding: 'round', +} as const; + +const parseDoubaoRawLocateValue = (input: unknown): LocateResultValue => { + if (typeof input !== 'string') { + throw new Error(`invalid point data: ${JSON.stringify(input)} `); + } + + const pointMatch = + input.match(/\s*(\d+)\s+(\d+)\s*<\/point>/i) ?? + input.match(/(\d+)\s+(\d+)/); + if (!pointMatch) { + throw new Error(`invalid point data: ${JSON.stringify(input)} `); + } + + return createLocateResultValue(doubaoPointCoordinates, [ + Number(pointMatch[1]), + Number(pointMatch[2]), + ]); +}; + +const buildDoubaoChatCompletionParams = ( + input: ChatCompletionCallContext, +): ChatCompletionParamsResult => { + const { midsceneDefaults, userConfig } = input; + const { reasoningEnabled, reasoningEffort } = userConfig; + const commonOverrideConfig: Record = {}; + + if (userConfig.temperature !== undefined) { + commonOverrideConfig.temperature = userConfig.temperature; + } + + // Doubao Chat Completions JSON mode: + // https://docs.volcengine.com/docs/82379/1568221?lang=zh + if ( + userConfig.responseFormat !== 'none' && + input.expectedJsonObjectResponse + ) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } + + const modelSpecificConfig: Record = {}; + + if (reasoningEnabled !== 'default') { + modelSpecificConfig.thinking = { + type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled', + }; + if (reasoningEffort) { + modelSpecificConfig.reasoning_effort = reasoningEffort; + } + } + + return { + config: { + ...midsceneDefaults, + ...commonOverrideConfig, + ...modelSpecificConfig, + }, + }; +}; + +const doubaoAdapter: ModelAdapterDefinition = { + chatCompletion: { + unsupportedUserConfig: ['reasoningBudget'], + buildChatCompletionParams: buildDoubaoChatCompletionParams, + useReasoningAsContentFallback: true, + }, + insight: { + protocol: createDoubaoInsightProtocol, + }, + locate: { + element: { + protocol: doubaoElementProtocol, + resultFormat: { + coordinates: doubaoPointCoordinates, + parseRawLocateValue: parseDoubaoRawLocateValue, + }, + }, + searchArea: { + protocol: doubaoSearchAreaProtocol, + }, + }, + planning: { + protocol: createDoubaoPlanningProtocol, + }, +}; + +export const doubaoAdapters = { + 'doubao-vision': doubaoAdapter, + 'doubao-seed': doubaoAdapter, +} satisfies Pick< + Record, + 'doubao-vision' | 'doubao-seed' +>; diff --git a/packages/core/src/ai-model/models/doubao/area-protocol.ts b/packages/core/src/ai-model/models/doubao/area-protocol.ts new file mode 100644 index 0000000000..29d3829957 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/area-protocol.ts @@ -0,0 +1,93 @@ +import type { StandardLocateProtocol } from '../../model-adapter/locate-protocol'; +import type { LocateResultPromptSpec } from '../../shared/model-locate-result'; +import { SEED_RESPONSE_PREFIX, SEED_TOOL_CALL_TAG_NAME } from './constants'; +import { parseDoubaoSearchAreaOutput } from './locate-output-parser'; +import { + assertPointLocatePromptSpec, + buildClickFunctionDefinition, + buildClickToolCallExample, +} from './locate-shared'; + +const clickFunctionDefinition = buildClickFunctionDefinition({ + includeRole: true, +}); + +const rowTargetClickToolCallExample = buildClickToolCallExample({ + role: 'target', + point: '680 460', +}); +const rowReferenceClickToolCallExample = buildClickToolCallExample({ + role: 'reference', + point: '320 460', +}); +const betweenTargetClickToolCallExample = buildClickToolCallExample({ + role: 'target', + point: '500 460', +}); +const betweenLeftReferenceClickToolCallExample = buildClickToolCallExample({ + role: 'reference', + point: '320 460', +}); +const betweenRightReferenceClickToolCallExample = buildClickToolCallExample({ + role: 'reference', + point: '680 460', +}); + +const buildSearchAreaResponseInstructions = ( + locatePromptSpec: LocateResultPromptSpec, +) => { + assertPointLocatePromptSpec(locatePromptSpec); + + return `## Objective + +- Locate exactly one target element that the user ultimately wants to operate. +- Locate every visible reference element used by the description to identify that target. + +## Target and Reference Rules + +- A reference is any other visible UI element or text used to distinguish, select, or describe the target. +- Row values, column values, labels, nearby controls, relative-position anchors, ordinal-position anchors, containers, and endpoints are references when the description uses them to identify the target. +- If the description uses a row, column, relative position, ordinal position, ownership, containment, or endpoints to identify the target, every visible element that expresses those constraints MUST be returned as a reference. +- Do not omit a reference merely because the target appears unique. +- Return the target as the first click tool call. +- Return one additional click tool call for every visible reference element after the target. The total number of click tool calls must be one plus the number of visible references. +- A response containing only the target is invalid when the description uses any visible element to identify the target. +- Do not repeat the target point as a reference. +- Do not return elements unrelated to identifying the target. An element participating in a row, column, relative-position, ordinal-position, ownership, containment, or endpoint constraint is related and MUST be returned. + +## Function Definition + +- You have access to the following functions: +${JSON.stringify(clickFunctionDefinition)} + +- Return one or more click tool calls using the following structure without any suffix. + +For the description "the price in the row whose product name is Tomato", return the visible "$3.00" text as the target, followed by the visible "Tomato" text as its reference: + +${SEED_RESPONSE_PREFIX} +${rowTargetClickToolCallExample} +${rowReferenceClickToolCallExample} + +For the description "the plus button between the Start and End nodes", return the plus icon as the target, followed by both endpoint nodes as references: + +${SEED_RESPONSE_PREFIX} +${betweenTargetClickToolCallExample} +${betweenLeftReferenceClickToolCallExample} +${betweenRightReferenceClickToolCallExample} + +## Important Notes +- Return each point as a separate <${SEED_TOOL_CALL_TAG_NAME}> block, including both the and wrappers. +- Use integer coordinates following this definition: ${locatePromptSpec.resultValueDescription} The origin is the top-left of the full screenshot. +`; +}; + +export const doubaoSearchAreaProtocol: StandardLocateProtocol = { + systemPromptIntroduction: ['## Role:', 'You are a GUI grounding agent.'].join( + '\n', + ), + buildResponseInstructions: buildSearchAreaResponseInstructions, + buildUserPrompt: (sectionDescription) => + `Locate the target and all visible reference elements used to identify it: ${sectionDescription}`, + expectedJsonObjectResponse: false, + parseRawResponse: parseDoubaoSearchAreaOutput, +}; diff --git a/packages/core/src/ai-model/models/doubao/constants.ts b/packages/core/src/ai-model/models/doubao/constants.ts new file mode 100644 index 0000000000..28e28fbe99 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/constants.ts @@ -0,0 +1,6 @@ +export const SEED_TOOL_CALL_TAG_NAME = 'seed:tool_call'; + +export const SEED_THINK_TOKEN = + 'think_never_used_51bce0c785ca2f68081bfa7d91973934'; + +export const SEED_RESPONSE_PREFIX = `<${SEED_THINK_TOKEN}> reasoning process `; diff --git a/packages/core/src/ai-model/models/doubao/element-protocol.ts b/packages/core/src/ai-model/models/doubao/element-protocol.ts new file mode 100644 index 0000000000..d3c2947e79 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/element-protocol.ts @@ -0,0 +1,50 @@ +import type { StandardLocateProtocol } from '../../model-adapter/locate-protocol'; +import type { LocateResultPromptSpec } from '../../shared/model-locate-result'; +import { SEED_RESPONSE_PREFIX } from './constants'; +import { parseDoubaoLocateOutput } from './locate-output-parser'; +import { + assertPointLocatePromptSpec, + buildClickFunctionDefinition, + buildClickToolCallExample, +} from './locate-shared'; + +const clickFunctionDefinition = buildClickFunctionDefinition(); + +const clickToolCallExample = buildClickToolCallExample({ point: 'x y' }); + +const buildElementResponseInstructions = ( + locatePromptSpec: LocateResultPromptSpec, +) => { + assertPointLocatePromptSpec(locatePromptSpec); + + return `## Function Definition + +- You have access to the following functions: +${JSON.stringify(clickFunctionDefinition)} + +- To call a function, use the following structure without any suffix: + +${SEED_RESPONSE_PREFIX} +${clickToolCallExample} + +## Important Notes +- Return the exact XML structure shown above, including the wrapper. +- Use integer coordinates following this definition: ${locatePromptSpec.resultValueDescription} The origin is the top-left of the full screenshot. +`; +}; + +export const doubaoElementProtocol: StandardLocateProtocol = { + systemPromptIntroduction: [ + '## Role:', + 'You are a GUI click grounding agent.', + '', + '## Objective:', + "- Identify elements in screenshots that match the user's description.", + "- Provide the coordinates of the element that matches the user's description.", + ].join('\n'), + buildResponseInstructions: buildElementResponseInstructions, + buildUserPrompt: (targetElementDescription: string) => + `## User Instruction: What element matches the following task: ${targetElementDescription}`, + expectedJsonObjectResponse: false, + parseRawResponse: parseDoubaoLocateOutput, +}; diff --git a/packages/core/src/ai-model/models/doubao/insight-protocol.ts b/packages/core/src/ai-model/models/doubao/insight-protocol.ts new file mode 100644 index 0000000000..74710ff395 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/insight-protocol.ts @@ -0,0 +1,72 @@ +import type { InsightProtocolFactory } from '../../model-adapter/insight-protocol'; +import { SEED_RESPONSE_PREFIX } from './constants'; +import { parseDoubaoToolCall } from './tool-call-parser'; +import { serializeDoubaoToolCall } from './tool-call-serializer'; + +const extractDataFunctionDefinition = { + type: 'function', + name: 'extract_data', + parameters: { + type: 'object', + properties: { + data: { + type: 'string', + description: + 'The extracted data encoded as JSON. Its value and schema must match DATA_DEMAND.', + }, + }, + required: ['data'], + }, +}; + +const buildExtractDataToolCall = (serializedData: string) => + serializeDoubaoToolCall({ + functionName: 'extract_data', + parameters: [ + { + name: 'data', + stringAttribute: 'true', + content: serializedData, + }, + ], + }); + +const dataOutputRules = `## Function Definition + +- You have access to the following functions: +${JSON.stringify(extractDataFunctionDefinition)} + +The data parameter must contain valid JSON, including when the result is a string, number, boolean, array, or null.`; + +export const createDoubaoInsightProtocol: InsightProtocolFactory = ({ + jsonParser, +}) => ({ + responsePrefix: SEED_RESPONSE_PREFIX, + dataOutput: { + tagNames: ['seed:tool_call'], + rules: dataOutputRules, + placeholder: buildExtractDataToolCall('{"StatementIsTruthy":true}'), + buildExample: buildExtractDataToolCall, + parse: (content: string): T => { + const toolCall = parseDoubaoToolCall(content); + if (!toolCall || toolCall.functionName !== 'extract_data') { + throw new Error('Missing required Seed extract_data tool call'); + } + + const dataParameter = toolCall.parameters.find( + ({ name }) => name === 'data', + ); + if (!dataParameter) { + throw new Error('Missing required Seed parameter: data'); + } + + try { + return jsonParser(dataParameter.rawValue, { + source: 'generic-object', + }) as T; + } catch (error) { + throw new Error(`Failed to parse Seed data parameter: ${error}`); + } + }, + }, +}); diff --git a/packages/core/src/ai-model/models/doubao/locate-output-parser.ts b/packages/core/src/ai-model/models/doubao/locate-output-parser.ts new file mode 100644 index 0000000000..0781ee231c --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/locate-output-parser.ts @@ -0,0 +1,123 @@ +import type { ParsedLocateResponse } from '../../model-adapter/locate-protocol'; +import { SEED_TOOL_CALL_TAG_NAME } from './constants'; +import { + type RawDoubaoToolCall, + parseDoubaoToolCalls, +} from './tool-call-parser'; + +const rawSeedToolCallPattern = new RegExp( + `<${SEED_TOOL_CALL_TAG_NAME}\\b[^>]*>[\\s\\S]*?<\\/${SEED_TOOL_CALL_TAG_NAME}>`, + 'i', +); + +const hasXmlTagMarker = (content: string, tagName: string) => + new RegExp(`(?:<${tagName}\\b|\\b${tagName}\\s*\\/?>)`, 'i').test(content); + +const hasInnerXmlTagMarker = (content: string) => + ['function', 'parameter', 'point'].some((tagName) => + hasXmlTagMarker(content, tagName), + ); + +const parseWithRawSeedToolCallFallback = ( + content: string, + parse: (content: string) => ParsedLocateResponse, +): ParsedLocateResponse => { + try { + return parse(content); + } catch (error) { + const rawSeedToolCall = content.match(rawSeedToolCallPattern)?.[0]; + if (!rawSeedToolCall || !hasInnerXmlTagMarker(rawSeedToolCall)) { + throw error; + } + + return { + kind: 'located', + target: rawSeedToolCall, + }; + } +}; + +const rawPointFromToolCall = (toolCall: RawDoubaoToolCall): string => { + if (toolCall.functionName !== 'click') { + throw new Error( + `Doubao locate response requires a click function, but received "${toolCall.functionName}"`, + ); + } + + const pointParameters = toolCall.parameters.filter( + ({ name }) => name === 'point', + ); + if (pointParameters.length !== 1) { + throw new Error( + 'Doubao click function requires exactly one point parameter', + ); + } + + const rawPoint = pointParameters[0].rawValue; + if (!rawPoint.trim()) { + throw new Error( + 'Doubao click function requires a non-empty point parameter', + ); + } + + return rawPoint; +}; + +const roleFromToolCall = (toolCall: RawDoubaoToolCall): string => { + const roleParameters = toolCall.parameters.filter( + ({ name }) => name === 'role', + ); + if (roleParameters.length !== 1) { + throw new Error( + 'Doubao search-area click function requires exactly one role parameter', + ); + } + + return roleParameters[0].rawValue.trim(); +}; + +const parseLocateToolCalls = (content: string): ParsedLocateResponse => { + const toolCalls = parseDoubaoToolCalls(content); + if (toolCalls.length !== 1) { + throw new Error( + 'Doubao locate response requires exactly one click function', + ); + } + + return { kind: 'located', target: rawPointFromToolCall(toolCalls[0]) }; +}; + +const parseSearchAreaToolCalls = (content: string): ParsedLocateResponse => { + const locatedPoints = parseDoubaoToolCalls(content).map( + (toolCall: RawDoubaoToolCall) => ({ + point: rawPointFromToolCall(toolCall), + role: roleFromToolCall(toolCall), + }), + ); + const targets = locatedPoints.filter(({ role }) => role === 'target'); + if (targets.length !== 1) { + throw new Error( + 'Doubao search-area response requires exactly one target click function', + ); + } + + const references = locatedPoints + .filter(({ role }) => role === 'reference') + .map(({ point }) => point); + + return { + kind: 'located', + target: targets[0].point, + ...(references.length > 0 ? { references } : {}), + }; +}; + +export const parseDoubaoLocateOutput = ( + content: string, +): ParsedLocateResponse => + parseWithRawSeedToolCallFallback(content, parseLocateToolCalls); + +export const parseDoubaoSearchAreaOutput = ( + content: string, +): ParsedLocateResponse => + parseWithRawSeedToolCallFallback(content, parseSearchAreaToolCalls); diff --git a/packages/core/src/ai-model/models/doubao/locate-shared.ts b/packages/core/src/ai-model/models/doubao/locate-shared.ts new file mode 100644 index 0000000000..8ed88156bb --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/locate-shared.ts @@ -0,0 +1,67 @@ +import type { LocateResultPromptSpec } from '../../shared/model-locate-result'; +import type { DoubaoFunctionDefinition } from './action-space'; +import { serializeDoubaoToolCall } from './tool-call-serializer'; + +export const buildClickFunctionDefinition = ({ + includeRole = false, +}: { + includeRole?: boolean; +} = {}): DoubaoFunctionDefinition => ({ + type: 'function', + name: 'click', + parameters: { + type: 'object', + properties: { + ...(includeRole + ? { + role: { + type: 'string', + enum: ['target', 'reference'], + description: + 'Whether the point identifies the target or a reference element.', + }, + } + : {}), + point: { + type: 'string', + description: 'Click coordinates. The format is: x y', + }, + }, + required: includeRole ? ['role', 'point'] : ['point'], + }, +}); + +export const buildClickToolCallExample = ({ + point, + role, +}: { + point: string; + role?: 'target' | 'reference'; +}) => + serializeDoubaoToolCall({ + functionName: 'click', + parameters: [ + ...(role + ? [ + { + name: 'role', + stringAttribute: 'true' as const, + content: role, + }, + ] + : []), + { + name: 'point', + stringAttribute: 'true' as const, + content: `${point}`, + }, + ], + }); + +export const assertPointLocatePromptSpec = ( + locatePromptSpec: LocateResultPromptSpec, +) => { + if (locatePromptSpec.resultKey !== 'point') { + throw new Error('Doubao locate protocol requires a point result adapter'); + } +}; diff --git a/packages/core/src/ai-model/models/doubao/planning-protocol.ts b/packages/core/src/ai-model/models/doubao/planning-protocol.ts new file mode 100644 index 0000000000..3934963982 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/planning-protocol.ts @@ -0,0 +1,57 @@ +import type { StandardPlanningProtocolFactory } from '../../model-adapter/planning-protocol'; +import { + createDoubaoPlanningActionOutputParser, + parseDoubaoRawLocateParameter, +} from './action-output-parser'; +import { + buildDoubaoActionDescription, + buildDoubaoLocateFieldDescription, +} from './action-space'; +import { SEED_RESPONSE_PREFIX, SEED_TOOL_CALL_TAG_NAME } from './constants'; +import { + buildDoubaoPlanningActionOutput, + serializeDoubaoToolCall, +} from './tool-call-serializer'; + +const actionOutputPlaceholder = serializeDoubaoToolCall({ + functionName: '...', + parameters: [ + { + name: '...', + stringAttribute: 'false|true', + content: '...', + }, + ], +}); + +export const createDoubaoPlanningProtocol: StandardPlanningProtocolFactory = ({ + jsonParser, +}) => { + const parseActionOutput = createDoubaoPlanningActionOutputParser(jsonParser); + + return { + responsePrefix: SEED_RESPONSE_PREFIX, + actionSpaceProtocol: { + title: 'Function Definition', + format: 'jsonl', + includeActionOutputExample: false, + buildLocateFieldDescription: buildDoubaoLocateFieldDescription, + buildActionDescription: buildDoubaoActionDescription, + }, + actionOutputProtocol: { + actionOutputTagNames: [SEED_TOOL_CALL_TAG_NAME], + actionOutputRules: [ + `- Output exactly one <${SEED_TOOL_CALL_TAG_NAME}> using a function from Function Definition.`, + '- The function name inside MUST exactly match the name of one function in Function Definition.', + '- All required parameters must be explicitly provided.', + '- Set string="true" when a parameter value is a string. Set string="false" when it is an integer, number, or boolean.', + '- For complex parameter values such as arrays or objects, set string="false" and encode the value as JSON.', + '- For locator parameters, set string="true" and always preserve the target description as element description. If the Function Definition also requires coordinates, append x y.', + ].join('\n'), + actionOutputPlaceholder, + buildActionOutput: buildDoubaoPlanningActionOutput, + parseActionOutput, + parseRawLocateParameter: parseDoubaoRawLocateParameter, + }, + }; +}; diff --git a/packages/core/src/ai-model/models/doubao/tool-call-parser.ts b/packages/core/src/ai-model/models/doubao/tool-call-parser.ts new file mode 100644 index 0000000000..e7332107e1 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/tool-call-parser.ts @@ -0,0 +1,67 @@ +import { SEED_TOOL_CALL_TAG_NAME } from './constants'; +import { extractXmlAttribute } from './xml'; + +export type RawDoubaoParameter = { + name: string; + isString: boolean | undefined; + rawValue: string; +}; + +export type RawDoubaoToolCall = { + functionName: string; + parameters: RawDoubaoParameter[]; +}; + +const parseFunction = ( + attributes: string, + content: string, +): RawDoubaoToolCall => { + const functionName = extractXmlAttribute(attributes, 'name'); + if (!functionName) { + throw new Error( + `Failed to parse ${SEED_TOOL_CALL_TAG_NAME}: missing function name`, + ); + } + + const parameterMatches = content.matchAll( + /]*)>([\s\S]*?)<\/parameter>/gi, + ); + const parameters = Array.from(parameterMatches, (parameterMatch) => { + const name = extractXmlAttribute(parameterMatch[1], 'name'); + const stringAttribute = extractXmlAttribute(parameterMatch[1], 'string'); + if (!name) { + throw new Error( + `Failed to parse ${SEED_TOOL_CALL_TAG_NAME}: parameter requires a name attribute`, + ); + } + + if ( + stringAttribute !== undefined && + stringAttribute !== 'true' && + stringAttribute !== 'false' + ) { + throw new Error( + `Failed to parse ${SEED_TOOL_CALL_TAG_NAME}: parameter requires a valid string attribute`, + ); + } + + return { + name, + isString: + stringAttribute === undefined ? undefined : stringAttribute === 'true', + rawValue: parameterMatch[2], + }; + }); + + return { functionName, parameters }; +}; + +export const parseDoubaoToolCalls = (content: string): RawDoubaoToolCall[] => + Array.from( + content.matchAll(/]*)>([\s\S]*?)<\/function>/gi), + (functionMatch) => parseFunction(functionMatch[1], functionMatch[2]), + ); + +export const parseDoubaoToolCall = ( + content: string, +): RawDoubaoToolCall | null => parseDoubaoToolCalls(content)[0] ?? null; diff --git a/packages/core/src/ai-model/models/doubao/tool-call-serializer.ts b/packages/core/src/ai-model/models/doubao/tool-call-serializer.ts new file mode 100644 index 0000000000..e12aef1003 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/tool-call-serializer.ts @@ -0,0 +1,123 @@ +import type { PlanningActionOutputBuildInput } from '../../model-adapter/planning-protocol'; +import { SEED_TOOL_CALL_TAG_NAME } from './constants'; +import { escapeXmlAttribute, escapeXmlText } from './xml'; + +type SerializedDoubaoParameter = { + name: string; + // `false|true` is a prompt placeholder only and is never emitted by a real tool call. + stringAttribute: 'true' | 'false' | 'false|true'; + content: string; +}; + +type SerializedDoubaoParameterValue = Omit; + +export const serializeDoubaoToolCall = ({ + functionName, + parameters, +}: { + functionName: string; + parameters: SerializedDoubaoParameter[]; +}) => { + const serializedParameters = parameters + .map( + ({ name, stringAttribute, content }) => + `${content}`, + ) + .join(''); + + return `<${SEED_TOOL_CALL_TAG_NAME}>${serializedParameters}`; +}; + +const serializeParameterValue = ( + value: unknown, +): SerializedDoubaoParameterValue => { + if (typeof value === 'string') { + return { + stringAttribute: 'true', + content: escapeXmlText(value), + }; + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return { + stringAttribute: 'false', + content: String(value), + }; + } + + const json = JSON.stringify(value); + if (json === undefined) { + throw new Error('Failed to serialize Seed parameter value as JSON'); + } + + return { + stringAttribute: 'false', + content: escapeXmlText(json), + }; +}; + +const serializeLocateParameter = ( + value: unknown, + locateResultKey: string | undefined, +) => { + // This serializes samples defined by the actionSpace, whose locator prompt + // is a plain string. Unlike runtime aiTap or YAML input, it does not handle + // the multimodal TUserPrompt shape nested under `prompt`. + if ( + !value || + typeof value !== 'object' || + !('prompt' in value) || + typeof value.prompt !== 'string' + ) { + throw new Error( + 'Failed to serialize Seed locator parameter: missing prompt', + ); + } + + const promptTag = `${escapeXmlText(value.prompt)}`; + if (!locateResultKey) { + return promptTag; + } + + const locateResult = (value as Record)[locateResultKey]; + if ( + locateResultKey !== 'point' || + !Array.isArray(locateResult) || + locateResult.length !== 2 || + !locateResult.every((coordinate) => typeof coordinate === 'number') + ) { + throw new Error( + 'Seed planning locator output requires point: [number, number]', + ); + } + + return `${promptTag}${locateResult[0]} ${locateResult[1]}`; +}; + +export const buildDoubaoPlanningActionOutput = ({ + actionName, + param, + locateFields = [], + locateResultKey, +}: PlanningActionOutputBuildInput) => { + const locateFieldSet = new Set(locateFields); + const parameters = Object.entries(param) + .filter(([, value]) => value !== undefined) + .map(([name, value]) => { + const serializedValue: SerializedDoubaoParameterValue = + locateFieldSet.has(name) + ? { + stringAttribute: 'true', + content: serializeLocateParameter(value, locateResultKey), + } + : serializeParameterValue(value); + + return { + name, + stringAttribute: serializedValue.stringAttribute, + content: serializedValue.content, + }; + }); + + return serializeDoubaoToolCall({ functionName: actionName, parameters }); +}; diff --git a/packages/core/src/ai-model/models/doubao/xml.ts b/packages/core/src/ai-model/models/doubao/xml.ts new file mode 100644 index 0000000000..cfc4412c90 --- /dev/null +++ b/packages/core/src/ai-model/models/doubao/xml.ts @@ -0,0 +1,14 @@ +export const escapeXmlAttribute = (value: string) => + value + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>'); + +export const escapeXmlText = (value: string) => + value.replace(/&/g, '&').replace(//g, '>'); + +export const extractXmlAttribute = (attributes: string, name: string) => { + const match = attributes.match(new RegExp(`\\b${name}="([^"]*)"`, 'i')); + return match?.[1]; +}; diff --git a/packages/core/src/ai-model/models/registry.ts b/packages/core/src/ai-model/models/registry.ts index c6b3c63769..c81d0d6170 100644 --- a/packages/core/src/ai-model/models/registry.ts +++ b/packages/core/src/ai-model/models/registry.ts @@ -9,7 +9,7 @@ import type { import { autoGlmAdapters } from './auto-glm/adapter'; import { deepSeekAdapters } from './deepseek/adapter'; import { defaultOpenAICompatibleAdapterConfig } from './default'; -import { doubaoAdapters } from './doubao'; +import { doubaoAdapters } from './doubao/adapter'; import { geminiAdapters } from './gemini'; import { glmAdapters } from './glm'; import { gptAdapters } from './gpt'; diff --git a/packages/core/tests/unit-test/model-adapter/doubao/action-output-parser.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/action-output-parser.test.ts new file mode 100644 index 0000000000..22d201fcfd --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/action-output-parser.test.ts @@ -0,0 +1,209 @@ +import { + createDoubaoPlanningActionOutputParser, + parseDoubaoRawLocateParameter, +} from '@/ai-model/models/doubao/action-output-parser'; +import { createDoubaoPlanningProtocol } from '@/ai-model/models/doubao/planning-protocol'; +import { parseModelResponseJson } from '@/ai-model/shared/json'; +import { parseStandardPlanningResponse } from '@/ai-model/workflows/planning'; +import { getMidsceneLocationSchema } from '@/common'; +import { describe, expect, it, rs } from '@rstest/core'; +import { z } from 'zod'; + +const actionOutputProtocol = createDoubaoPlanningProtocol({ + jsonParser: parseModelResponseJson, +}).actionOutputProtocol; + +describe('Doubao planning action output parser', () => { + const parseActionOutputWithActionSpace = + createDoubaoPlanningActionOutputParser(parseModelResponseJson); + const parseActionOutput = (content: string) => + parseActionOutputWithActionSpace(content, []); + + it('parses primitive, complex and locator parameters', () => { + const content = + 'John & Jane2["first","second"]Submit & Continue500 600'; + + expect(parseActionOutput(content)).toEqual({ + type: 'Example', + param: { + text: 'John & Jane', + count: 2, + options: ['first', 'second'], + locate: 'Submit & Continue500 600', + }, + }); + }); + + it('keeps prompt-only and official point-only parameters as strings', () => { + expect( + parseActionOutput( + 'Submit', + ), + ).toEqual({ + type: 'Tap', + param: { locate: 'Submit' }, + }); + expect( + parseActionOutput( + '500 600', + ), + ).toEqual({ + type: 'Tap', + param: { locate: '500 600' }, + }); + }); + + it('uses the adapter JSON parser for non-string parameters', () => { + const jsonParser = rs.fn(() => ['parsed']); + const parser = createDoubaoPlanningActionOutputParser(jsonParser); + + expect( + parser( + '[1,2]', + [], + ), + ).toEqual({ type: 'Example', param: { options: ['parsed'] } }); + expect(jsonParser).toHaveBeenCalledWith('[1,2]', { + source: 'planning-action-param', + }); + }); + + it('does not parse locator tags when the parameter is not a string', () => { + const jsonParser = rs.fn(() => ({ parsedBy: 'jsonParser' })); + const parser = createDoubaoPlanningActionOutputParser(jsonParser); + + expect( + parser( + '500 600', + [], + ), + ).toEqual({ + type: 'Example', + param: { locate: { parsedBy: 'jsonParser' } }, + }); + expect(jsonParser).toHaveBeenCalledWith('500 600', { + source: 'planning-action-param', + }); + }); + + it('uses the action schema when the string attribute is missing', () => { + const content = + 'John2truedown{"duration":300}Submit500 600'; + const actionSpace = [ + { + name: 'Example', + paramSchema: z.object({ + text: z.string(), + count: z.number(), + enabled: z.boolean(), + direction: z.enum(['up', 'down']), + options: z.object({ duration: z.number() }), + locate: getMidsceneLocationSchema(), + }), + call: rs.fn(), + }, + ]; + + expect(parseActionOutputWithActionSpace(content, actionSpace)).toEqual({ + type: 'Example', + param: { + text: 'John', + count: 2, + enabled: true, + direction: 'down', + options: { duration: 300 }, + locate: 'Submit500 600', + }, + }); + }); + + it('returns null when no Seed tool call is present', () => { + expect( + parseActionOutput('Done'), + ).toBeNull(); + }); + + it('keeps completion dependent on the Midscene complete tag', () => { + expect( + parseStandardPlanningResponse( + 'Done', + { + includeThought: true, + actionOutputProtocol, + actionSpace: [], + }, + ), + ).toEqual({ + log: '', + finalizeMessage: 'Done', + finalizeSuccess: true, + action: null, + }); + expect( + parseStandardPlanningResponse('Finished without a tool call', { + includeThought: true, + actionOutputProtocol, + actionSpace: [], + }), + ).toEqual({ + log: '', + action: null, + }); + }); + + it('composes with the standard planning parser', () => { + const content = `Tap the target +Tap the Submit button +the Submit button500 600`; + + expect( + parseStandardPlanningResponse(content, { + includeThought: true, + actionOutputProtocol, + actionSpace: [], + }), + ).toEqual({ + thought: 'Tap the target', + log: 'Tap the Submit button', + action: { + type: 'Tap', + param: { + locate: 'the Submit button500 600', + }, + }, + }); + }); +}); + +describe('Doubao planning raw locator parameter parser', () => { + it('parses prompt and point after the action space identifies a locator field', () => { + expect( + parseDoubaoRawLocateParameter( + 'Submit & Continue500 600', + ), + ).toEqual({ + prompt: 'Submit & Continue', + point: '500 600', + }); + }); + + it('parses prompt-only and official point-only forms', () => { + expect(parseDoubaoRawLocateParameter('Submit')).toEqual({ + prompt: 'Submit', + }); + expect(parseDoubaoRawLocateParameter('500 600')).toEqual({ + point: '500 600', + }); + }); + + it('preserves multiple raw point values for the result codec', () => { + expect( + parseDoubaoRawLocateParameter( + 'Submit500 600700 800', + ), + ).toEqual({ + prompt: 'Submit', + point: '500 600700 800', + }); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/action-space.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/action-space.test.ts new file mode 100644 index 0000000000..eca97738fd --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/action-space.test.ts @@ -0,0 +1,179 @@ +import { + buildDoubaoActionDescription, + buildDoubaoLocateFieldDescription, +} from '@/ai-model/models/doubao/action-space'; +import { createDoubaoPlanningProtocol } from '@/ai-model/models/doubao/planning-protocol'; +import { serializeActionDescriptions } from '@/ai-model/prompt/planning'; +import { parseModelResponseJson } from '@/ai-model/shared/json'; +import type { LocateResultPromptSpec } from '@/ai-model/shared/model-locate-result'; +import { + actionInputParamSchema, + registerFileChooserAcceptParamSchema, +} from '@/device'; +import { getMidsceneLocationSchema } from '@/index'; +import { describe, expect, it } from '@rstest/core'; +import { z } from 'zod'; + +const pointPromptSpec: LocateResultPromptSpec = { + resultKey: 'point', + resultValueSchema: '[number, number]', + resultValueDescription: 'point coordinates in the 0-1000 range', + resultNoun: 'point', + resultNounPlural: 'points', + exampleValues: [ + [150, 150], + [402, 463], + ], +}; + +const planningProtocol = createDoubaoPlanningProtocol({ + jsonParser: parseModelResponseJson, +}); + +describe('Doubao Function Definition', () => { + it('builds an empty parameters definition for an action without params', () => { + expect( + buildDoubaoActionDescription({ + action: { + name: 'Reload', + description: 'Reload the current page', + call: async () => {}, + }, + locateFieldDescription: buildDoubaoLocateFieldDescription(), + }), + ).toEqual({ + name: 'Reload', + description: 'Reload the current page', + parameters: { + type: 'object', + properties: {}, + required: [], + }, + }); + }); + + it('builds JSONL with locator, enum and integer fields', () => { + const locateFieldDescription = + buildDoubaoLocateFieldDescription(pointPromptSpec); + const action = { + name: 'Swipe', + description: 'Swipe from an element in a direction', + paramSchema: z.object({ + locate: getMidsceneLocationSchema().describe( + 'The swipe starting point', + ), + direction: z.enum(['up', 'down']), + duration: z.number().int().optional(), + }), + call: async () => {}, + }; + const description = buildDoubaoActionDescription({ + action, + locateFieldDescription, + }); + + expect(description).toEqual({ + name: 'Swipe', + description: 'Swipe from an element in a direction', + parameters: { + type: 'object', + properties: { + locate: { + type: 'string', + description: + 'The swipe starting point The format is: element descriptionx y. point coordinates in the 0-1000 range', + }, + direction: { + type: 'string', + description: 'Parameter direction for Swipe.', + enum: ['up', 'down'], + }, + duration: { + type: 'integer', + description: 'Parameter duration for Swipe.', + }, + }, + required: ['locate', 'direction'], + }, + }); + expect( + serializeActionDescriptions( + [description], + planningProtocol.actionSpaceProtocol.format, + ), + ).toBe(JSON.stringify(description)); + }); + + it('preserves union branches from built-in action schemas', () => { + const locateFieldDescription = buildDoubaoLocateFieldDescription(); + const inputDescription = buildDoubaoActionDescription({ + action: { + name: 'Input', + description: 'Input the value into the element', + paramSchema: actionInputParamSchema, + call: async () => {}, + }, + locateFieldDescription, + }); + const fileChooserDescription = buildDoubaoActionDescription({ + action: { + name: 'RegisterFileChooserAccept', + description: 'Configure files for file chooser dialogs', + paramSchema: registerFileChooserAcceptParamSchema, + call: async () => {}, + }, + locateFieldDescription, + }); + + expect(inputDescription.parameters.properties.value.anyOf).toEqual([ + { type: 'string' }, + { type: 'number' }, + ]); + expect(fileChooserDescription.parameters.properties.files.anyOf).toEqual([ + { type: 'string' }, + { + type: 'array', + items: { type: 'string' }, + }, + ]); + }); + + it('recursively describes nested objects and arrays', () => { + const description = buildDoubaoActionDescription({ + action: { + name: 'Configure', + description: 'Configure an operation', + paramSchema: z.object({ + options: z.object({ + duration: z.number().int().describe('Duration in milliseconds'), + labels: z.array(z.string()).optional(), + }), + }), + call: async () => {}, + }, + locateFieldDescription: buildDoubaoLocateFieldDescription(), + }); + + expect(description.parameters).toEqual({ + type: 'object', + properties: { + options: { + type: 'object', + properties: { + duration: { + type: 'integer', + description: 'Duration in milliseconds', + }, + labels: { + type: 'array', + items: { type: 'string' }, + }, + }, + required: ['duration'], + description: 'Parameter options for Configure.', + }, + }, + required: ['options'], + }); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/adapter.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/adapter.test.ts new file mode 100644 index 0000000000..1396d60eb8 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/adapter.test.ts @@ -0,0 +1,223 @@ +import { ResolvedModelAdapter } from '@/ai-model/model-adapter/resolve'; +import { doubaoAdapters } from '@/ai-model/models/doubao/adapter'; +import { describe, expect, it } from '@rstest/core'; + +const doubaoVisionAdapter = new ResolvedModelAdapter( + doubaoAdapters['doubao-vision'], + 'doubao-vision', +); +const doubaoSeedAdapter = new ResolvedModelAdapter( + doubaoAdapters['doubao-seed'], + 'doubao-seed', +); + +describe('doubao model adapter', () => { + it('keeps doubao-seed and doubao-vision on the same adapter definition', () => { + expect(doubaoAdapters['doubao-seed']).toBe(doubaoAdapters['doubao-vision']); + expect(doubaoSeedAdapter.jsonParser).toBe(doubaoVisionAdapter.jsonParser); + expect(doubaoSeedAdapter.chatCompletion.unsupportedUserConfig).toEqual([ + 'reasoningBudget', + ]); + }); + + it('uses the Seed standard planning protocol for both family aliases', () => { + expect(doubaoSeedAdapter.planning.kind).toBe('standard'); + expect(doubaoVisionAdapter.planning.kind).toBe('standard'); + if ( + doubaoSeedAdapter.planning.kind !== 'standard' || + doubaoVisionAdapter.planning.kind !== 'standard' + ) { + throw new Error('doubao should use standard planning adapters'); + } + + expect(doubaoSeedAdapter.planning.supportsActionDeepLocate).toBe(true); + expect( + doubaoSeedAdapter.planning.protocol.actionOutputProtocol + .actionOutputTagNames, + ).toEqual(['seed:tool_call']); + expect( + doubaoVisionAdapter.planning.protocol.actionOutputProtocol + .actionOutputTagNames, + ).toEqual(['seed:tool_call']); + expect(doubaoSeedAdapter.planning.protocol.responsePrefix).toBe( + ' reasoning process ', + ); + }); + + it('uses the Seed insight protocol for both family aliases', () => { + expect(doubaoSeedAdapter.insight.protocol.dataOutput.rules).toContain( + '"name":"extract_data"', + ); + expect(doubaoVisionAdapter.insight.protocol.dataOutput.rules).toContain( + '"name":"extract_data"', + ); + }); + + it('defaults doubao thinking to disabled when reasoning config is unset', () => { + const result = doubaoSeedAdapter.chatCompletion.buildChatCompletionParams({ + userConfig: {}, + }); + expect(result.config).toEqual({ + temperature: 0, + thinking: { type: 'disabled' }, + }); + }); + + it('uses json_object response format when expected unless disabled', () => { + const autoResult = + doubaoVisionAdapter.chatCompletion.buildChatCompletionParams({ + expectedJsonObjectResponse: true, + userConfig: {}, + }); + const disabledResult = + doubaoVisionAdapter.chatCompletion.buildChatCompletionParams({ + expectedJsonObjectResponse: true, + userConfig: { responseFormat: 'none' }, + }); + + expect(autoResult.config.response_format).toEqual({ type: 'json_object' }); + expect(disabledResult.config.response_format).toBeUndefined(); + }); + + it('applies an explicit temperature override', () => { + const result = doubaoSeedAdapter.chatCompletion.buildChatCompletionParams({ + userConfig: { + temperature: 0.7, + reasoningEnabled: true, + }, + }); + + expect(result.config).toEqual({ + temperature: 0.7, + thinking: { type: 'enabled' }, + }); + }); + + it('maps reasoningEnabled and reasoningEffort to Doubao parameters', () => { + const enabled = doubaoSeedAdapter.chatCompletion.buildChatCompletionParams({ + userConfig: { + reasoningEnabled: true, + reasoningEffort: 'high', + }, + }); + const disabled = + doubaoVisionAdapter.chatCompletion.buildChatCompletionParams({ + userConfig: { reasoningEnabled: false }, + }); + const providerDefault = + doubaoSeedAdapter.chatCompletion.buildChatCompletionParams({ + userConfig: { + reasoningEnabled: 'default', + reasoningEffort: 'high', + }, + }); + + expect(enabled.config).toEqual({ + temperature: 0, + thinking: { type: 'enabled' }, + reasoning_effort: 'high', + }); + expect(disabled.config).toEqual({ + temperature: 0, + thinking: { type: 'disabled' }, + }); + expect(providerDefault.config).toEqual({ temperature: 0 }); + }); + + it('keeps using the shared lenient JSON parser', () => { + expect( + doubaoVisionAdapter.jsonParser('{"point": [123 456]}', { + source: 'locate', + }), + ).toEqual({ point: [123, 456] }); + expect(() => + doubaoVisionAdapter.jsonParser('```', { source: 'generic-object' }), + ).toThrow(); + }); + + it.each([ + ['doubao-seed', doubaoSeedAdapter], + ['doubao-vision', doubaoVisionAdapter], + ])('uses point/1000 coordinates for %s', (_, adapter) => { + expect(adapter.locate.kind).toBe('standard'); + if (adapter.locate.kind !== 'standard') { + throw new Error('doubao should use a standard locate adapter'); + } + + expect(adapter.locate.searchArea).toBeDefined(); + expect(adapter.locate.element.protocol.expectedJsonObjectResponse).toBe( + false, + ); + expect(adapter.locate.element.resultCodec.promptSpec.resultKey).toBe( + 'point', + ); + expect(adapter.locate.searchArea?.resultCodec.promptSpec.resultKey).toBe( + 'point', + ); + expect( + adapter.locate.element.resultCodec.toPixelBbox('100 200', { + preparedSize: { width: 1000, height: 2000 }, + }), + ).toEqual([90, 380, 110, 420]); + expect( + adapter.locate.element.resultCodec.toPixelBbox( + '100 200900 900', + { + preparedSize: { width: 1000, height: 2000 }, + }, + ), + ).toEqual([90, 380, 110, 420]); + expect( + adapter.locate.element.resultCodec.toPixelBbox('100 200', { + preparedSize: { width: 1000, height: 2000 }, + }), + ).toEqual([90, 380, 110, 420]); + expect( + adapter.locate.element.resultCodec.toPixelBbox('100 200', { + preparedSize: { width: 1000, height: 2000 }, + }), + ).toEqual([90, 380, 110, 420]); + expect( + adapter.locate.element.resultCodec.toPixelBbox( + '100 200', + { + preparedSize: { width: 1000, height: 2000 }, + }, + ), + ).toEqual([90, 380, 110, 420]); + }); + + it('adapts a planning point to a pixel bbox', () => { + const locateAdapter = doubaoVisionAdapter.locate; + expect(locateAdapter.kind).toBe('standard'); + if (locateAdapter.kind !== 'standard') { + throw new Error('doubao should use a standard locate adapter'); + } + + expect( + locateAdapter.element.resultCodec.toPixelBbox('500 500', { + preparedSize: { width: 1000, height: 1000 }, + }), + ).toEqual([490, 490, 509, 509]); + }); + + it('rejects the retired bbox locate format', () => { + const locateAdapter = doubaoSeedAdapter.locate; + expect(locateAdapter.kind).toBe('standard'); + if (locateAdapter.kind !== 'standard') { + throw new Error('doubao should use a standard locate adapter'); + } + + expect(() => + locateAdapter.element.resultCodec.toPixelBbox( + { bbox: [100, 200, 300, 400] }, + { preparedSize: { width: 1000, height: 2000 } }, + ), + ).toThrow(/invalid point data/); + expect(() => + locateAdapter.element.resultCodec.toPixelBbox([100, 200, 300, 400], { + preparedSize: { width: 1000, height: 2000 }, + }), + ).toThrow(/invalid point data/); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/area-protocol.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/area-protocol.test.ts new file mode 100644 index 0000000000..801f945166 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/area-protocol.test.ts @@ -0,0 +1,151 @@ +import { doubaoSearchAreaProtocol } from '@/ai-model/models/doubao/area-protocol'; +import { createLocateResultPromptSpec } from '@/ai-model/shared/model-locate-result/prompt-spec'; +import { describe, expect, it } from '@rstest/core'; + +const locatePromptSpec = createLocateResultPromptSpec({ + shape: 'point', + order: 'xy', + normalizedBy: 1000, +}); +const parseSearchAreaResponse = (content: string) => + doubaoSearchAreaProtocol.parseRawResponse(content, locatePromptSpec); + +describe('doubao search-area locate protocol', () => { + it('builds instructions that distinguish target and references', () => { + const instructions = + doubaoSearchAreaProtocol.buildResponseInstructions(locatePromptSpec); + + expect(doubaoSearchAreaProtocol.systemPromptIntroduction).toContain( + 'You are a GUI grounding agent.', + ); + expect(instructions).toContain( + 'Locate exactly one target element that the user ultimately wants to operate.', + ); + expect(instructions).toContain( + 'A response containing only the target is invalid when the description uses any visible element to identify the target.', + ); + expect(instructions).toContain( + 'For the description "the price in the row whose product name is Tomato"', + ); + expect(instructions).toContain( + 'For the description "the plus button between the Start and End nodes"', + ); + const functionDefinition = JSON.parse( + instructions.split('\n').find((line) => line.startsWith('{')) ?? '', + ); + expect(functionDefinition).toMatchObject({ + name: 'click', + parameters: { + properties: { + role: { + type: 'string', + enum: ['target', 'reference'], + }, + }, + required: ['role', 'point'], + }, + }); + expect(instructions).toContain( + 'target', + ); + expect(instructions).toContain( + 'reference', + ); + expect(instructions.match(/ { + expect( + parseSearchAreaResponse( + 'reference510 460' + + 'target320 460' + + 'reference680 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460', + references: ['510 460', '680 460'], + }); + }); + + it('parses a target without references', () => { + expect( + parseSearchAreaResponse( + 'target320 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460', + }); + }); + + it('preserves multiple points inside one point parameter', () => { + expect( + parseSearchAreaResponse( + 'target320 460510 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460510 460', + }); + }); + + it('does not require the role parameter to be marked as a string', () => { + expect( + parseSearchAreaResponse( + 'target320 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460', + }); + }); + + it('ignores unrecognized roles', () => { + expect( + parseSearchAreaResponse( + 'target320 460' + + 'candidate510 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460', + }); + }); + + it('rejects missing or duplicate roles and duplicate targets', () => { + expect(() => + parseSearchAreaResponse( + '320 460', + ), + ).toThrow('requires exactly one role parameter'); + expect(() => + parseSearchAreaResponse( + 'targetreference320 460', + ), + ).toThrow('requires exactly one role parameter'); + expect(() => + parseSearchAreaResponse( + 'target320 460' + + 'target510 460', + ), + ).toThrow('requires exactly one target click function'); + }); + + it('preserves the raw Seed tool call when its inner XML is malformed', () => { + const response = + 'target320 460'; + + expect(parseSearchAreaResponse(response)).toEqual({ + kind: 'located', + target: response, + }); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/element-protocol.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/element-protocol.test.ts new file mode 100644 index 0000000000..52f4de5494 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/element-protocol.test.ts @@ -0,0 +1,124 @@ +import { doubaoElementProtocol } from '@/ai-model/models/doubao/element-protocol'; +import { buildElementLocateSystemPrompt } from '@/ai-model/prompt/locate'; +import { createLocateResultPromptSpec } from '@/ai-model/shared/model-locate-result/prompt-spec'; +import { describe, expect, it } from '@rstest/core'; + +const locatePromptSpec = createLocateResultPromptSpec({ + shape: 'point', + order: 'xy', + normalizedBy: 1000, +}); +const parseElementResponse = (content: string) => + doubaoElementProtocol.parseRawResponse(content, locatePromptSpec); + +describe('doubao element locate protocol', () => { + it('builds the click Function Definition system prompt', () => { + const prompt = buildElementLocateSystemPrompt({ + systemPromptIntroduction: doubaoElementProtocol.systemPromptIntroduction, + responseInstructions: + doubaoElementProtocol.buildResponseInstructions(locatePromptSpec), + }); + + expect(prompt).toContain('## Function Definition'); + const functionDefinition = JSON.parse( + prompt.split('\n').find((line) => line.startsWith('{')) ?? '', + ); + expect(functionDefinition).toMatchObject({ + type: 'function', + name: 'click', + parameters: { + required: ['point'], + }, + }); + expect(prompt).toContain( + 'x y', + ); + expect(prompt).toContain( + ' reasoning process ', + ); + expect(prompt).toContain('normalized to 0-1000'); + expect(doubaoElementProtocol.expectedJsonObjectResponse).toBe(false); + }); + + it('uses the result adapter coordinate description', () => { + const locatePromptSpec = createLocateResultPromptSpec({ + shape: 'point', + order: 'xy', + normalizedBy: 100, + }); + + const instructions = + doubaoElementProtocol.buildResponseInstructions(locatePromptSpec); + + expect(instructions).toContain('normalized to 0-100'); + expect(instructions).not.toContain('normalized to 0-1000'); + }); + + it('builds the click grounding user prompt', () => { + const prompt = doubaoElementProtocol.buildUserPrompt('the Submit button'); + + expect(prompt).not.toContain('Coordinates'); + expect(prompt).toContain( + '## User Instruction: What element matches the following task: the Submit button', + ); + }); + + it('parses the click tool call point', () => { + expect( + parseElementResponse( + '320 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460', + }); + }); + + it('rejects responses that do not follow the click function protocol', () => { + expect(() => parseElementResponse('No tool call')).toThrow( + 'requires exactly one click function', + ); + expect(() => + parseElementResponse(''), + ).toThrow('requires a click function'); + expect(() => + parseElementResponse( + '806 292', + ), + ).toThrow('requires exactly one click function'); + expect(() => parseElementResponse('123.4 567.8')).toThrow( + 'requires exactly one click function', + ); + }); + + it('preserves the raw point parameter value', () => { + expect( + parseElementResponse( + '320 460510 460', + ), + ).toEqual({ + kind: 'located', + target: '320 460510 460', + }); + }); + + it('preserves the raw Seed tool call when its inner XML is malformed', () => { + const response = + '320 460'; + + expect(parseElementResponse(response)).toEqual({ + kind: 'located', + target: response, + }); + }); + + it('falls back to the raw Seed tool call when a malformed point attribute leaves the parameter empty', () => { + const response = + '', + ); + expect(protocol.dataOutput.rules).not.toContain(''); + }); + + it.each([ + ['object', '{"enabled":true}', { enabled: true }], + ['array', '["one","two"]', ['one', 'two']], + ['string', '"Midscene"', 'Midscene'], + ['number', '42', 42], + ['boolean', 'false', false], + ])('parses %s insight data', (_, rawData, expectedData) => { + const response = `Visible evidence${rawData}`; + + expect(parseDoubaoInsightResponse(response)).toEqual({ + thought: 'Visible evidence', + data: expectedData, + }); + }); + + it('parses optional errors without discarding valid data', () => { + const response = `["target is not visible"]null`; + + expect(parseDoubaoInsightResponse(response)).toEqual({ + data: null, + errors: ['target is not visible'], + }); + }); + + it('throws when the required data parameter is missing', () => { + expect(() => + parseDoubaoInsightResponse( + '', + ), + ).toThrow('Missing required Seed parameter: data'); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/insight.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/insight.test.ts new file mode 100644 index 0000000000..77d3f9d600 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/insight.test.ts @@ -0,0 +1,68 @@ +import { getModelRuntime } from '@/ai-model/models'; +import { callAI } from '@/ai-model/service-caller/index'; +import { AiExtractElementInfo } from '@/ai-model/workflows/insight'; +import type { IModelConfig } from '@midscene/shared/env'; +import { beforeEach, describe, expect, it, rs } from '@rstest/core'; +import { createFakeContext } from '../../../utils'; + +import * as serviceCallerActual from '@/ai-model/service-caller/index' with { + rstest: 'importActual', +}; + +rs.mock('@/ai-model/service-caller/index', () => ({ + ...serviceCallerActual, + AIResponseParseError: class AIResponseParseError extends Error {}, + callAI: rs.fn(), +})); + +describe('doubao insight', () => { + const modelConfig: IModelConfig = { + modelFamily: 'doubao-seed', + modelName: 'test-model', + modelDescription: 'test-model-desc', + intent: 'insight', + slot: 'insight', + retryCount: 1, + retryInterval: 0, + }; + + beforeEach(() => { + rs.clearAllMocks(); + }); + + it('uses the Seed protocol for the shared Insight workflow', async () => { + rs.mocked(callAI).mockResolvedValue({ + content: + 'The success toast is visible.{"StatementIsTruthy":true}', + usage: undefined, + reasoning_content: undefined, + } as any); + + const result = await AiExtractElementInfo<{ + StatementIsTruthy: boolean; + }>({ + context: createFakeContext(), + dataQuery: { + StatementIsTruthy: 'Boolean, whether the success toast is visible', + }, + modelRuntime: getModelRuntime(modelConfig), + }); + + expect(result.parseResult).toEqual({ + thought: 'The success toast is visible.', + data: { StatementIsTruthy: true }, + }); + + const [messages] = rs.mocked(callAI).mock.calls[0]; + expect(messages[0].content).toContain('"name":"extract_data"'); + expect(messages[1].content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'image_url' }), + expect.objectContaining({ + type: 'text', + text: expect.stringContaining(''), + }), + ]), + ); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/locate.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/locate.test.ts new file mode 100644 index 0000000000..92fe410ea9 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/locate.test.ts @@ -0,0 +1,121 @@ +import { getModelRuntime } from '@/ai-model/models'; +import { callAI } from '@/ai-model/service-caller/index'; +import { + AiLocateElement, + AiLocateSection, +} from '@/ai-model/workflows/grounding'; +import type { IModelConfig } from '@midscene/shared/env'; +import { beforeEach, describe, expect, it, rs } from '@rstest/core'; +import { createFakeContext } from '../../../utils'; + +import * as serviceCallerActual from '@/ai-model/service-caller/index' with { + rstest: 'importActual', +}; + +rs.mock('@/ai-model/service-caller/index', () => ({ + ...serviceCallerActual, + callAI: rs.fn(), +})); + +describe('doubao standard locate', () => { + const modelConfig: IModelConfig = { + modelFamily: 'doubao-seed', + modelName: 'doubao-test-model', + modelDescription: 'doubao-test-model', + intent: 'default', + slot: 'default', + retryCount: 0, + retryInterval: 0, + }; + + beforeEach(() => { + rs.clearAllMocks(); + rs.mocked(callAI).mockResolvedValue({ + content: + '500 500', + reasoning_content: 'Found the target', + isStreamed: false, + }); + }); + + it('sends the locate message and parses the click point', async () => { + const result = await AiLocateElement({ + context: createFakeContext(), + targetElementDescription: 'the Submit button', + modelRuntime: getModelRuntime(modelConfig), + }); + + const [messages, , callOptions] = rs.mocked(callAI).mock.calls[0]; + expect(messages[0]).toMatchObject({ + role: 'system', + content: expect.stringMatching(/"name"\s*:\s*"click"/), + }); + expect(messages[1]).toMatchObject({ + role: 'user', + content: [ + { + type: 'image_url', + image_url: expect.any(Object), + }, + { + type: 'text', + text: expect.stringContaining( + '## User Instruction: What element matches the following task: the Submit button', + ), + }, + ], + }); + expect(callOptions).toMatchObject({ + expectedJsonObjectResponse: false, + }); + expect(result.rect).toBeDefined(); + expect(result.parseResult.errors).toEqual([]); + expect(result.rawResponse).toContain('500 500'); + expect(result.reasoning_content).toBe('Found the target'); + }); + + it('uses ordered click tool calls to build a deepLocate search area', async () => { + rs.mocked(callAI).mockResolvedValue({ + content: + 'target320 460' + + 'reference510 460', + reasoning_content: 'Found the target and its reference', + isStreamed: false, + }); + + const result = await AiLocateSection({ + context: createFakeContext(), + sectionDescription: 'the edit icon in the row containing Apollo', + modelRuntime: getModelRuntime(modelConfig), + }); + + const [messages, , callOptions] = rs.mocked(callAI).mock.calls[0]; + expect(messages[0]).toMatchObject({ + role: 'system', + content: expect.stringContaining( + 'A response containing only the target is invalid when the description uses any visible element to identify the target.', + ), + }); + expect(messages[1]).toMatchObject({ + role: 'user', + content: [ + { + type: 'image_url', + image_url: expect.any(Object), + }, + { + type: 'text', + text: expect.stringContaining( + 'Locate the target and all visible reference elements used to identify it: the edit icon in the row containing Apollo', + ), + }, + ], + }); + expect(callOptions).toMatchObject({ + expectedJsonObjectResponse: false, + }); + expect(result.searchAreaConfig).toBeDefined(); + expect(result.rawResponse).toContain('320 460'); + expect(result.rawResponse).toContain('510 460'); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/planning-protocol.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/planning-protocol.test.ts new file mode 100644 index 0000000000..ad3c48305f --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/planning-protocol.test.ts @@ -0,0 +1,68 @@ +import { SEED_RESPONSE_PREFIX } from '@/ai-model/models/doubao/constants'; +import { createDoubaoPlanningProtocol } from '@/ai-model/models/doubao/planning-protocol'; +import { + buildStandardPlanningSystemPrompt, + createSampleTapAction, +} from '@/ai-model/prompt/planning'; +import { parseModelResponseJson } from '@/ai-model/shared/json'; +import type { LocateResultPromptSpec } from '@/ai-model/shared/model-locate-result'; +import { describe, expect, it } from '@rstest/core'; + +const pointPromptSpec: LocateResultPromptSpec = { + resultKey: 'point', + resultValueSchema: '[number, number]', + resultValueDescription: 'point coordinates in the 0-1000 range', + resultNoun: 'point', + resultNounPlural: 'points', + exampleValues: [ + [150, 150], + [402, 463], + ], +}; + +const planningProtocol = createDoubaoPlanningProtocol({ + jsonParser: parseModelResponseJson, +}); + +describe('Doubao planning prompt', () => { + const tapAction = { + ...createSampleTapAction('the Submit button'), + description: 'Tap an element', + call: async () => {}, + }; + + it('renders Function Definition, response start and Seed examples', async () => { + const prompt = await buildStandardPlanningSystemPrompt({ + actionSpace: [tapAction], + includeLocateInPlanning: true, + locatePromptSpec: pointPromptSpec, + planningProtocol, + }); + + expect(prompt).toContain('### Function Definition'); + expect(prompt).toContain('"name":"Tap"'); + expect(prompt).toContain(SEED_RESPONSE_PREFIX); + expect(prompt).not.toContain('Every response MUST begin'); + expect(prompt).toContain( + 'Add to cart button for Sauce Labs Backpack402 463', + ); + expect(prompt).toContain(''); + expect(prompt).not.toContain(''); + expect(prompt).not.toContain(''); + }); + + it('keeps locator prompt but omits point when planning does not locate', async () => { + const prompt = await buildStandardPlanningSystemPrompt({ + actionSpace: [tapAction], + includeLocateInPlanning: false, + planningProtocol, + }); + + expect(prompt).toContain( + 'Add to cart button for Sauce Labs Backpack', + ); + expect(prompt).not.toContain( + 'Add to cart button for Sauce Labs Backpack', + ); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/planning.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/planning.test.ts new file mode 100644 index 0000000000..51d158250b --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/planning.test.ts @@ -0,0 +1,189 @@ +import { getModelRuntime } from '@/ai-model/models'; +import { callAI } from '@/ai-model/service-caller/index'; +import { prepareUserPrompt } from '@/ai-model/shared/multimodal-prompt'; +import { AiLocateElement } from '@/ai-model/workflows/grounding'; +import { standardPlan } from '@/ai-model/workflows/planning'; +import { ConversationHistory } from '@/ai-model/workflows/planning/conversation-history'; +import type { PlanOptions } from '@/ai-model/workflows/planning/types'; +import type { IModelConfig } from '@midscene/shared/env'; +import { beforeEach, describe, expect, it, rs } from '@rstest/core'; +import { mockActionSpace } from '../../../common'; +import { createFakeContext } from '../../../utils'; + +import * as serviceCallerActual from '@/ai-model/service-caller/index' with { + rstest: 'importActual', +}; + +rs.mock('@/ai-model/service-caller/index', () => ({ + ...serviceCallerActual, + callAI: rs.fn(), +})); + +const runDoubaoPlan = async (userInstruction: string, options: PlanOptions) => + standardPlan(await prepareUserPrompt(userInstruction), options); + +describe('doubao standard planning', () => { + const modelConfig: IModelConfig = { + modelFamily: 'doubao-seed', + modelName: 'doubao-test-model', + modelDescription: 'doubao-test-model', + intent: 'planning', + slot: 'planning', + retryCount: 0, + retryInterval: 0, + }; + + beforeEach(() => { + rs.clearAllMocks(); + }); + + it('parses a Seed action locator and normalizes its point to a pixel bbox', async () => { + rs.mocked(callAI).mockResolvedValueOnce({ + content: `Tap the Submit button +Tap the Submit button +the Submit button500 600`, + isStreamed: false, + }); + + const result = await runDoubaoPlan('tap the Submit button', { + context: createFakeContext(), + actionSpace: mockActionSpace, + modelRuntime: getModelRuntime(modelConfig), + conversationHistory: new ConversationHistory(), + includeLocateInPlanning: true, + effort: 'balance', + }); + + expect(result.actions).toEqual([ + { + type: 'Tap', + param: { + locate: { + prompt: 'the Submit button', + point: '500 600', + locatedPixelBbox: [940, 637, 979, 658], + }, + }, + }, + ]); + }); + + it('uses independent locate after planning returns only a locator prompt', async () => { + rs.mocked(callAI) + .mockResolvedValueOnce({ + content: `Tap the Submit button +Tap the Submit button +the Submit button`, + isStreamed: false, + }) + .mockResolvedValueOnce({ + content: + '500 600', + isStreamed: false, + }); + + const planningResult = await runDoubaoPlan('tap the Submit button', { + context: createFakeContext(), + actionSpace: mockActionSpace, + modelRuntime: getModelRuntime(modelConfig), + conversationHistory: new ConversationHistory(), + includeLocateInPlanning: false, + effort: 'balance', + }); + const plannedAction = planningResult.actions?.[0]; + if (!plannedAction) { + throw new Error('Expected planning to return a Tap action'); + } + + expect(plannedAction).toEqual({ + type: 'Tap', + param: { + locate: { + prompt: 'the Submit button', + }, + }, + }); + + const locateResult = await AiLocateElement({ + context: createFakeContext(), + targetElementDescription: plannedAction.param.locate.prompt, + modelRuntime: getModelRuntime({ + ...modelConfig, + intent: 'default', + slot: 'default', + }), + }); + + expect(callAI).toHaveBeenCalledTimes(2); + expect(locateResult.rect).toBeDefined(); + expect(locateResult.parseResult.errors).toEqual([]); + }); + + it('stops planning when the response completes without an action', async () => { + rs.mocked(callAI).mockResolvedValueOnce({ + content: `The task is complete +Done`, + isStreamed: false, + }); + + const result = await runDoubaoPlan('finish the task', { + context: createFakeContext(), + actionSpace: mockActionSpace, + modelRuntime: getModelRuntime(modelConfig), + conversationHistory: new ConversationHistory(), + includeLocateInPlanning: false, + effort: 'balance', + }); + + expect(result.actions).toEqual([]); + expect(result.finalizeSuccess).toBe(true); + expect(result.finalizeMessage).toBe('Done'); + expect(result.shouldContinuePlanning).toBe(false); + }); + + it('updates sub-goal history in deepThink planning', async () => { + rs.mocked(callAI).mockResolvedValueOnce({ + content: `I should tap the Submit button first + + Tap the Submit button + Confirm submission + +Tap the Submit button +the Submit button`, + isStreamed: false, + }); + const conversationHistory = new ConversationHistory(); + + const result = await runDoubaoPlan('submit the form', { + context: createFakeContext(), + actionSpace: mockActionSpace, + modelRuntime: getModelRuntime(modelConfig), + conversationHistory, + includeLocateInPlanning: false, + effort: 'deepThink', + }); + + expect(result.actions).toEqual([ + { + type: 'Tap', + param: { + locate: { + prompt: 'the Submit button', + }, + }, + }, + ]); + expect(conversationHistory.subGoalsToText()).toContain( + 'Tap the Submit button (running)', + ); + expect(conversationHistory.subGoalsToText()).toContain( + 'Confirm submission (pending)', + ); + expect(conversationHistory.subGoalsToText()).toContain( + '- Tap the Submit button', + ); + + const [messages] = rs.mocked(callAI).mock.calls[0]; + expect(messages[0].content).toContain(''); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/tool-call-parser.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/tool-call-parser.test.ts new file mode 100644 index 0000000000..0fa751817c --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/tool-call-parser.test.ts @@ -0,0 +1,132 @@ +import { + parseDoubaoToolCall, + parseDoubaoToolCalls, +} from '@/ai-model/models/doubao/tool-call-parser'; +import { describe, expect, it } from '@rstest/core'; + +describe('Doubao tool call parser', () => { + it('parses XML structure while preserving raw parameter values', () => { + expect( + parseDoubaoToolCall( + 'John & Jane[1,2]', + ), + ).toEqual({ + functionName: 'Example', + parameters: [ + { + name: 'text', + isString: true, + rawValue: 'John & Jane', + }, + { + name: 'options', + isString: false, + rawValue: '[1,2]', + }, + ], + }); + }); + + it('returns null when no function call is present', () => { + expect( + parseDoubaoToolCall('Done'), + ).toBeNull(); + }); + + it('preserves a missing string attribute on point parameters', () => { + expect( + parseDoubaoToolCall( + '500 600', + ), + ).toEqual({ + functionName: 'click', + parameters: [ + { + name: 'point', + isString: undefined, + rawValue: '500 600', + }, + ], + }); + }); + + it('preserves a missing string attribute on other parameters', () => { + expect( + parseDoubaoToolCall( + 'John', + ), + ).toEqual({ + functionName: 'Input', + parameters: [ + { + name: 'value', + isString: undefined, + rawValue: 'John', + }, + ], + }); + }); + + it('rejects an invalid string attribute', () => { + expect(() => + parseDoubaoToolCall( + 'John', + ), + ).toThrow('parameter requires a valid string attribute'); + }); + + it('parses multiple Seed tool call blocks in their output order', () => { + expect( + parseDoubaoToolCalls( + '320 460' + + '510 460', + ), + ).toEqual([ + { + functionName: 'click', + parameters: [ + { + name: 'point', + isString: true, + rawValue: '320 460', + }, + ], + }, + { + functionName: 'click', + parameters: [ + { + name: 'point', + isString: true, + rawValue: '510 460', + }, + ], + }, + ]); + }); + + it('parses multiple functions inside one Seed tool call block', () => { + expect( + parseDoubaoToolCalls( + '320 460510 460', + ), + ).toHaveLength(2); + }); + + it('parses a function when the surrounding Seed tool call is malformed', () => { + expect( + parseDoubaoToolCall( + '320 460', + ), + ).toEqual({ + functionName: 'click', + parameters: [ + { + name: 'point', + isString: true, + rawValue: '320 460', + }, + ], + }); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao/tool-call-serializer.test.ts b/packages/core/tests/unit-test/model-adapter/doubao/tool-call-serializer.test.ts new file mode 100644 index 0000000000..5858bb51e5 --- /dev/null +++ b/packages/core/tests/unit-test/model-adapter/doubao/tool-call-serializer.test.ts @@ -0,0 +1,76 @@ +import { buildDoubaoPlanningActionOutput } from '@/ai-model/models/doubao/tool-call-serializer'; +import { + buildActionOutputExample, + createSampleTapAction, +} from '@/ai-model/prompt/planning'; +import type { LocateResultPromptSpec } from '@/ai-model/shared/model-locate-result'; +import { describe, expect, it } from '@rstest/core'; + +const pointPromptSpec: LocateResultPromptSpec = { + resultKey: 'point', + resultValueSchema: '[number, number]', + resultValueDescription: 'point coordinates in the 0-1000 range', + resultNoun: 'point', + resultNounPlural: 'points', + exampleValues: [ + [150, 150], + [402, 463], + ], +}; + +describe('Doubao planning action output serializer', () => { + it('serializes primitive and complex parameters using the Seed protocol', () => { + expect( + buildDoubaoPlanningActionOutput({ + actionName: 'Example', + param: { + text: 'John & Jane', + count: 2, + enabled: true, + options: ['first', 'second'], + config: { mode: 'fast' }, + }, + }), + ).toBe( + 'John & Jane2true["first","second"]{"mode":"fast"}', + ); + }); + + it('always serializes a locator prompt and conditionally includes point', () => { + const sampleTapAction = createSampleTapAction('the Submit button'); + + expect( + buildActionOutputExample(sampleTapAction, { + buildActionOutput: buildDoubaoPlanningActionOutput, + }), + ).toBe( + 'the Submit button', + ); + expect( + buildActionOutputExample(sampleTapAction, { + buildActionOutput: buildDoubaoPlanningActionOutput, + locatePromptSpec: pointPromptSpec, + locateResultExampleIndex: 1, + }), + ).toBe( + 'the Submit button402 463', + ); + }); + + it('serializes multiple locator fields independently', () => { + expect( + buildDoubaoPlanningActionOutput({ + actionName: 'Swipe', + param: { + start: { prompt: 'the slider thumb', point: [200, 500] }, + end: { prompt: 'the right end of the slider', point: [800, 500] }, + duration: 300, + }, + locateFields: ['start', 'end'], + locateResultKey: 'point', + }), + ).toBe( + 'the slider thumb200 500the right end of the slider800 500300', + ); + }); +});