diff --git a/packages/core/src/ai-model/prompt/describe.ts b/packages/core/src/ai-model/prompt/describe.ts index deefcf2fad..b95704a77e 100644 --- a/packages/core/src/ai-model/prompt/describe.ts +++ b/packages/core/src/ai-model/prompt/describe.ts @@ -33,14 +33,14 @@ export const elementDescriberInstruction = () => { const preferredLanguage = getPreferredLanguage(); return ` -Describe the real page element indicated by the temporary callout. -The callout is an annotation overlay. It is not part of the page or target. +Describe the real page element indicated by the temporary target marker (a ring with a crosshair). +The marker is an annotation overlay. It is not part of the page or target. The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption. IMPORTANT: You MUST write the description in ${preferredLanguage}. OBSERVE IN THIS ORDER: -1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region. +1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region. 2. Primitive: name what that smallest part is before adding surrounding context. 3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text. 4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state. @@ -48,20 +48,20 @@ OBSERVE IN THIS ORDER: RULES: - Keep description under 35 words. - Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it. -- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target. -- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center. +- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target. +- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center. - For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group. -- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself. -- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field. -- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers. -- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries. -- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive. -- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible. -- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line. +- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself. +- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field. +- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers. +- If the marker center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries. +- Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive. +- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible. +- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line. - For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label. - For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible. -- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state. -- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control. +- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state. +- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control. - Use actual visible text from the current screenshot when available; do not copy labels from the examples. - **Write the description in ${preferredLanguage}** diff --git a/packages/core/src/service/index.ts b/packages/core/src/service/index.ts index 2e245cf38c..6d2cdb155a 100644 --- a/packages/core/src/service/index.ts +++ b/packages/core/src/service/index.ts @@ -549,7 +549,7 @@ export default class Service { imageContent = [ { type: 'text' as const, - text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.', + text: 'Use these images together to describe the real UI target marked by the temporary target marker. Do not describe the marker itself.', }, { type: 'text' as const, @@ -568,7 +568,7 @@ export default class Service { imageContent = [ { type: 'text' as const, - text: 'Full screenshot with a temporary callout marking the target:', + text: 'Full screenshot with a temporary target marker (a ring with a crosshair) marking the target:', }, { type: 'image_url' as const, diff --git a/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap b/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap index be9a52e805..de8a6eb535 100644 --- a/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap +++ b/packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap @@ -2,14 +2,14 @@ exports[`elementDescriberInstruction > should return the correct instruction 1`] = ` " -Describe the real page element indicated by the temporary callout. -The callout is an annotation overlay. It is not part of the page or target. +Describe the real page element indicated by the temporary target marker (a ring with a crosshair). +The marker is an annotation overlay. It is not part of the page or target. The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption. IMPORTANT: You MUST write the description in English. OBSERVE IN THIS ORDER: -1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region. +1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region. 2. Primitive: name what that smallest part is before adding surrounding context. 3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text. 4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state. @@ -17,20 +17,20 @@ OBSERVE IN THIS ORDER: RULES: - Keep description under 35 words. - Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it. -- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target. -- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center. +- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target. +- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center. - For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group. -- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself. -- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field. -- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers. -- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries. -- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive. -- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible. -- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line. +- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself. +- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field. +- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers. +- If the marker center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries. +- Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive. +- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible. +- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line. - For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label. - For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible. -- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state. -- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control. +- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state. +- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control. - Use actual visible text from the current screenshot when available; do not copy labels from the examples. - **Write the description in English** diff --git a/packages/core/tests/unit-test/prompt/describe.test.ts b/packages/core/tests/unit-test/prompt/describe.test.ts index 2f08387bf4..b010db0042 100644 --- a/packages/core/tests/unit-test/prompt/describe.test.ts +++ b/packages/core/tests/unit-test/prompt/describe.test.ts @@ -31,7 +31,7 @@ describe('elementDescriberInstruction', () => { 'use primitive "option" for selectable list options', ); expect(instruction).toContain( - 'Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes', + 'Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes', ); }); }); diff --git a/packages/core/tests/unit-test/service-describe.test.ts b/packages/core/tests/unit-test/service-describe.test.ts index 3b03da5b95..8eab97062d 100644 --- a/packages/core/tests/unit-test/service-describe.test.ts +++ b/packages/core/tests/unit-test/service-describe.test.ts @@ -86,7 +86,7 @@ describe('service.describe', () => { expect(prompt).toContain('For inline text, links, or substrings'); expect(prompt).toContain('For repeated rows, cards, or options'); expect(prompt).toContain('Use selected, highlighted, hovered'); - expect(prompt).toContain('if the callout endpoint/center is inside'); + expect(prompt).toContain('if the marker center is inside'); expect(prompt).toContain('empty region/gap'); expect(prompt).toContain('Do not borrow the text, glyph, direction'); }); @@ -247,7 +247,7 @@ describe('service.describe', () => { content.filter((item) => item.type === 'image_url')[0]?.image_url?.url, ).toBe('data:image/png;base64,boxed'); expect(content.map((item) => item.text).filter(Boolean)).toEqual([ - 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.', + 'Use these images together to describe the real UI target marked by the temporary target marker. Do not describe the marker itself.', 'Image 1: full screenshot overview with the target marker, for page position and ownership context.', 'Image 2: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.', ]); diff --git a/packages/shared/src/img/box-select.ts b/packages/shared/src/img/box-select.ts index 1b76deb366..ea932951d4 100644 --- a/packages/shared/src/img/box-select.ts +++ b/packages/shared/src/img/box-select.ts @@ -267,48 +267,6 @@ function drawCircle( } } -function drawEllipse( - pixels: Uint8Array, - width: number, - height: number, - center: { x: number; y: number }, - radiusX: number, - radiusY: number, - color: { r: number; g: number; b: number; a: number }, - thickness = 2, -) { - const cx = Math.round(center.x); - const cy = Math.round(center.y); - const outerRadiusX = Math.max(Math.round(radiusX), 1); - const outerRadiusY = Math.max(Math.round(radiusY), 1); - const innerRadiusX = Math.max(outerRadiusX - Math.round(thickness), 0); - const innerRadiusY = Math.max(outerRadiusY - Math.round(thickness), 0); - - for (let py = cy - outerRadiusY; py <= cy + outerRadiusY; py++) { - for (let px = cx - outerRadiusX; px <= cx + outerRadiusX; px++) { - if (px < 0 || py < 0 || px >= width || py >= height) continue; - - const dx = px - cx; - const dy = py - cy; - const outer = - (dx * dx) / (outerRadiusX * outerRadiusX) + - (dy * dy) / (outerRadiusY * outerRadiusY); - const inner = - innerRadiusX === 0 || innerRadiusY === 0 - ? 0 - : (dx * dx) / (innerRadiusX * innerRadiusX) + - (dy * dy) / (innerRadiusY * innerRadiusY); - if (outer > 1 || inner < 1) continue; - - const idx = (py * width + px) * 4; - pixels[idx + 0] = color.r; - pixels[idx + 1] = color.g; - pixels[idx + 2] = color.b; - pixels[idx + 3] = color.a; - } - } -} - function drawLine( pixels: Uint8Array, width: number, @@ -343,108 +301,6 @@ function drawLine( } } -function drawCallout( - pixels: Uint8Array, - width: number, - height: number, - targetPoint: { x: number; y: number }, - markerRadius: number, - style: { - callout: { r: number; g: number; b: number; a: number }; - calloutBorder: { r: number; g: number; b: number; a: number }; - calloutText: { r: number; g: number; b: number; a: number }; - indexId: number; - }, -) { - const cx = Math.round(targetPoint.x); - const cy = Math.round(targetPoint.y); - const gap = Math.max(markerRadius + 18, 30); - const candidates = [ - { x: cx + gap, y: cy - gap }, - { x: cx + gap, y: cy + gap }, - { x: cx - gap, y: cy - gap }, - { x: cx - gap, y: cy + gap }, - { x: cx, y: cy - gap }, - { x: cx, y: cy + gap }, - { x: cx + gap, y: cy }, - { x: cx - gap, y: cy }, - ]; - const calloutCenter = candidates.find( - (candidate) => - candidate.x - markerRadius >= 0 && - candidate.x + markerRadius < width && - candidate.y - markerRadius >= 0 && - candidate.y + markerRadius < height, - ) || { - x: Math.min(Math.max(cx + gap, markerRadius), width - markerRadius - 1), - y: Math.min(Math.max(cy - gap, markerRadius), height - markerRadius - 1), - }; - - const lineAngle = Math.atan2(cy - calloutCenter.y, cx - calloutCenter.x); - const lineStart = { - x: calloutCenter.x + Math.cos(lineAngle) * (markerRadius + 1), - y: calloutCenter.y + Math.sin(lineAngle) * (markerRadius + 1), - }; - - const targetRadiusX = 30; - const targetRadiusY = 15; - const ellipseBoundaryDistance = - 1 / - Math.sqrt( - Math.cos(lineAngle) ** 2 / targetRadiusX ** 2 + - Math.sin(lineAngle) ** 2 / targetRadiusY ** 2, - ); - const lineEnd = { - x: cx - Math.cos(lineAngle) * ellipseBoundaryDistance, - y: cy - Math.sin(lineAngle) * ellipseBoundaryDistance, - }; - - drawLine(pixels, width, height, lineStart, lineEnd, style.callout, 2); - drawCircle( - pixels, - width, - height, - calloutCenter, - markerRadius + 2, - style.calloutBorder, - { - thickness: 3, - }, - ); - drawCircle( - pixels, - width, - height, - calloutCenter, - markerRadius, - style.callout, - { - fill: true, - }, - ); - drawEllipse( - pixels, - width, - height, - targetPoint, - targetRadiusX, - targetRadiusY, - style.callout, - 2, - ); - const textWidth = getNumberWidth(style.indexId); - const textHeight = FONT_HEIGHT * FONT_SCALE; - drawNumber( - pixels, - width, - height, - style.indexId, - Math.round(calloutCenter.x - textWidth / 2), - Math.round(calloutCenter.y - textHeight / 2), - style.calloutText, - ); -} - function drawPointMarker( pixels: Uint8Array, width: number, @@ -457,16 +313,45 @@ function drawPointMarker( { r: 0xc6, g: 0x23, b: 0x00, a: 0xee }, { r: 0x00, g: 0x00, b: 0xff, a: 0xee }, ]; - const callout = markerColors[(indexId - 1) % markerColors.length]; - const calloutBorder = { r: 0xff, g: 0xff, b: 0xff, a: 0xff }; - const calloutText = { r: 0xff, g: 0xff, b: 0xff, a: 0xff }; - const markerRadius = Math.max(Math.round(radius), 10); - drawCallout(pixels, width, height, point, markerRadius, { - callout, - calloutBorder, - calloutText, - indexId, + const marker = markerColors[(indexId - 1) % markerColors.length]; + const markerBorder = { r: 0xff, g: 0xff, b: 0xff, a: 0xff }; + const ringRadius = Math.max(Math.round(radius), 10); + + // Draw a hollow target reticle centered on the point instead of an offset + // numbered callout bubble. The red filled circle with a white digit looked + // like a notification badge and was often reported as part of the page UI. + drawCircle(pixels, width, height, point, ringRadius + 2, markerBorder, { + thickness: 3, + }); + drawCircle(pixels, width, height, point, ringRadius, marker, { + thickness: 3, }); + + const tickInner = ringRadius + 5; + const tickOuter = ringRadius + 12; + const tickDirections = [ + { x: 0, y: -1 }, + { x: 0, y: 1 }, + { x: -1, y: 0 }, + { x: 1, y: 0 }, + ]; + for (const direction of tickDirections) { + drawLine( + pixels, + width, + height, + { + x: point.x + direction.x * tickInner, + y: point.y + direction.y * tickInner, + }, + { + x: point.x + direction.x * tickOuter, + y: point.y + direction.y * tickOuter, + }, + markerBorder, + 3, + ); + } } function blendPixels( diff --git a/packages/shared/tests/unit-test/image/index.test.ts b/packages/shared/tests/unit-test/image/index.test.ts index eabff87a74..f43a076960 100644 --- a/packages/shared/tests/unit-test/image/index.test.ts +++ b/packages/shared/tests/unit-test/image/index.test.ts @@ -330,7 +330,7 @@ describe('image utils', () => { expect(markedInfo).toEqual(originalInfo); }); - it('compositePointMarkerImg uses red target and blue locator callout markers without covering the target point', async () => { + it('compositePointMarkerImg uses red and blue centered reticle markers without covering the target point', async () => { const inputBuffer = await sharp({ create: { width: 120, @@ -400,29 +400,29 @@ describe('image utils', () => { targetPointOffset, targetPointOffset + 3, ); - const rightEdgeOffset = (60 * info.width + 90) * info.channels; - const lowerEdgeOffset = (75 * info.width + 60) * info.channels; - const circleBottomOffset = (90 * info.width + 60) * info.channels; - const [rightR, rightG, rightB] = data.slice( - rightEdgeOffset, - rightEdgeOffset + 3, + // The reticle ring (radius 14) is centered exactly on the point. + const ringRightOffset = (60 * info.width + 74) * info.channels; + const ringBottomOffset = (74 * info.width + 60) * info.channels; + // Pixels beyond the ring/crosshair area stay untouched. + const outsideOffset = (100 * info.width + 60) * info.channels; + const [ringRightR, ringRightG, ringRightB] = data.slice( + ringRightOffset, + ringRightOffset + 3, ); - const [lowerR, lowerG, lowerB] = data.slice( - lowerEdgeOffset, - lowerEdgeOffset + 3, + const [ringBottomR, ringBottomG, ringBottomB] = data.slice( + ringBottomOffset, + ringBottomOffset + 3, ); - const [circleBottomR, circleBottomG, circleBottomB] = data.slice( - circleBottomOffset, - circleBottomOffset + 3, + const [outsideR, outsideG, outsideB] = data.slice( + outsideOffset, + outsideOffset + 3, ); expect(targetR).toBeGreaterThan(245); expect(targetG).toBeGreaterThan(245); expect(targetB).toBeGreaterThan(245); - expect(isRedMarkerPixel(rightR, rightG, rightB)).toBe(true); - expect(isRedMarkerPixel(lowerR, lowerG, lowerB)).toBe(true); - expect(isRedMarkerPixel(circleBottomR, circleBottomG, circleBottomB)).toBe( - false, - ); + expect(isRedMarkerPixel(ringRightR, ringRightG, ringRightB)).toBe(true); + expect(isRedMarkerPixel(ringBottomR, ringBottomG, ringBottomB)).toBe(true); + expect(isRedMarkerPixel(outsideR, outsideG, outsideB)).toBe(false); expect(redDominantPixels).toBeGreaterThan(20); expect(blueCalloutPixels).toBe(0); expect(locatorBlueCalloutPixels).toBeGreaterThan(20);