fix(core): replace numbered point callout with badge-neutral centered marker

packages/core/src/ai-model/prompt/describe.ts

@@ -33,35 +33,35 @@ export const elementDescriberInstruction = () => {
   const preferredLanguage = getPreferredLanguage();
 
   return `
-Describe the real page element indicated by the temporary callout.
-The callout is an annotation overlay. It is not part of the page or target.
+Describe the real page element indicated by the temporary target marker (a ring with a crosshair).
+The marker is an annotation overlay. It is not part of the page or target.
 The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.
 
 IMPORTANT: You MUST write the description in ${preferredLanguage}.
 
 OBSERVE IN THIS ORDER:
-1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
+1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
 2. Primitive: name what that smallest part is before adding surrounding context.
 3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.
 4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.
 
 RULES:
 - Keep description under 35 words.
 - Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.
-- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.
-- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.
+- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target.
+- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center.
 - For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.
-- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.
-- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
-- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
-- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
-- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
-- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.
-- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.
+- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself.
+- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
+- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
+- If the marker center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
+- Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
+- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible.
+- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line.
 - For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.
 - For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.
-- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.
-- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
+- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state.
+- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
 - Use actual visible text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in ${preferredLanguage}**
 

packages/core/src/service/index.ts

@@ -549,7 +549,7 @@ export default class Service {
       imageContent = [
         {
           type: 'text' as const,
-          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',
+          text: 'Use these images together to describe the real UI target marked by the temporary target marker. Do not describe the marker itself.',
         },
         {
           type: 'text' as const,
@@ -568,7 +568,7 @@ export default class Service {
       imageContent = [
         {
           type: 'text' as const,
-          text: 'Full screenshot with a temporary callout marking the target:',
+          text: 'Full screenshot with a temporary target marker (a ring with a crosshair) marking the target:',
         },
         {
           type: 'image_url' as const,

packages/core/tests/unit-test/prompt/__snapshots__/describe.test.ts.snap

@@ -2,35 +2,35 @@
 
 exports[`elementDescriberInstruction > should return the correct instruction 1`] = `
 "
-Describe the real page element indicated by the temporary callout.
-The callout is an annotation overlay. It is not part of the page or target.
+Describe the real page element indicated by the temporary target marker (a ring with a crosshair).
+The marker is an annotation overlay. It is not part of the page or target.
 The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.
 
 IMPORTANT: You MUST write the description in English.
 
 OBSERVE IN THIS ORDER:
-1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
+1. Target first: identify the smallest real UI part at the marker center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
 2. Primitive: name what that smallest part is before adding surrounding context.
 3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.
 4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.
 
 RULES:
 - Keep description under 35 words.
 - Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.
-- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.
-- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.
+- Ignore every annotation overlay, including ring, crosshair, line, color, marker, border, dot, or selection box. Never describe the annotation as the target.
+- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the marker center.
 - For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.
-- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.
-- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
-- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
-- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
-- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
-- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.
-- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.
+- If the marker center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the marker center is on that icon itself.
+- If the marker center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
+- If the marker center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly at the marker center. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
+- If the marker center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
+- Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
+- For compound controls or stacked glyphs, describe only the sub-part containing the marker center, using upper/lower or left/right only when visible.
+- For inline text, links, or substrings, describe only the exact substring/link at the marker center, not the whole line.
 - For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.
 - For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.
-- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.
-- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
+- Use selected, highlighted, hovered, focused, or active state only if the marker center is inside that state.
+- If the marker center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
 - Use actual visible text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in English**
 

packages/core/tests/unit-test/prompt/describe.test.ts

@@ -31,7 +31,7 @@ describe('elementDescriberInstruction', () => {
       'use primitive "option" for selectable list options',
     );
     expect(instruction).toContain(
-      'Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes',
+      'Only use primitive "icon" or "arrow" when the marker center directly overlaps the real glyph strokes',
     );
   });
 });

packages/core/tests/unit-test/service-describe.test.ts

@@ -86,7 +86,7 @@ describe('service.describe', () => {
     expect(prompt).toContain('For inline text, links, or substrings');
     expect(prompt).toContain('For repeated rows, cards, or options');
     expect(prompt).toContain('Use selected, highlighted, hovered');
-    expect(prompt).toContain('if the callout endpoint/center is inside');
+    expect(prompt).toContain('if the marker center is inside');
     expect(prompt).toContain('empty region/gap');
     expect(prompt).toContain('Do not borrow the text, glyph, direction');
   });
@@ -247,7 +247,7 @@ describe('service.describe', () => {
       content.filter((item) => item.type === 'image_url')[0]?.image_url?.url,
     ).toBe('data:image/png;base64,boxed');
     expect(content.map((item) => item.text).filter(Boolean)).toEqual([
-      'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',
+      'Use these images together to describe the real UI target marked by the temporary target marker. Do not describe the marker itself.',
       'Image 1: full screenshot overview with the target marker, for page position and ownership context.',
       'Image 2: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.',
     ]);