/** * Build a semantic capture descriptor for an interacted element. * * Trace steps are an LLM *textbook*: each click must say what to look for * on screen (usually a short visible name). Tag-only noise like * `{ "tag": "div" }` / “点击div” is useless and must not be recorded. */ /** Content-script capture descriptor before VOM geometric matching. */ export interface CaptureTargetDescriptor { role?: string; name?: string; tag: string; name_attr?: string; placeholder?: string; nearby_label?: string; } /** Max length for a label that is still a useful “find this on screen” hint. */ const ACTIONABLE_LABEL_MAX = 48; function normalizeWhitespace(value: string): string { return value.replace(/\s+/g, " ").trim(); } function truncate(value: string, max = 80): string { const trimmed = normalizeWhitespace(value); if (trimmed.length <= max) return trimmed; return `${trimmed.slice(0, max - 1)}…`; } /** Short, human label an LLM can search for in a snapshot. */ export function isActionableLabel(name: string): boolean { const trimmed = normalizeWhitespace(name); if (!trimmed) return false; if (trimmed.length > ACTIONABLE_LABEL_MAX) return false; if (/^https?:\/\//i.test(trimmed)) return false; return true; } function labelledByText(el: Element): string | null { const ids = el.getAttribute("aria-labelledby"); if (!ids) return null; const parts = ids .split(/\s+/) .map((id) => document.getElementById(id)?.textContent ?? "") .map(normalizeWhitespace) .filter(Boolean); return parts.length ? truncate(parts.join(" ")) : null; } function associatedLabelText(el: Element): string | null { if (!(el instanceof HTMLElement) || !el.id) return null; const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); if (!label) return null; const text = normalizeWhitespace(label.textContent ?? ""); return text ? truncate(text) : null; } function wrappingLabelText(el: Element): string | null { const label = el.closest("label"); if (!label) return null; const clone = label.cloneNode(true) as HTMLElement; for (const control of clone.querySelectorAll("input,textarea,select,button")) { control.remove(); } const text = normalizeWhitespace(clone.textContent ?? ""); return text ? truncate(text) : null; } function placeholderOrTitle(el: Element): string | null { if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { const ph = normalizeWhitespace(el.placeholder); if (ph) return truncate(ph); } const title = normalizeWhitespace(el.getAttribute("title") ?? ""); return title ? truncate(title) : null; } function ownText(el: Element): string | null { if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { return null; } const text = normalizeWhitespace(el.textContent ?? ""); return text ? truncate(text) : null; } /** Prefer a compact title inside links (e.g. SERP result `

`), not the whole blob. */ function compactLinkName(el: Element): string | null { const heading = el.querySelector("h1, h2, h3, h4"); if (heading) { const text = normalizeWhitespace(heading.textContent ?? ""); if (text) return truncate(text); } const img = el.querySelector("img[alt]"); if (img instanceof HTMLImageElement) { const alt = normalizeWhitespace(img.alt); if (alt) return truncate(alt); } const clone = el.cloneNode(true) as HTMLElement; for (const junk of clone.querySelectorAll("cite, script, style, svg, noscript")) { junk.remove(); } const text = normalizeWhitespace(clone.textContent ?? ""); if (!text) return null; const withoutUrl = text.replace(/https?:\/\/\S+/g, "").trim(); return truncate(withoutUrl || text); } /** Visible accessible name preference order for LLM textbooks. */ export function accessibleName(el: Element): string | undefined { const ariaLabel = normalizeWhitespace(el.getAttribute("aria-label") ?? ""); if (ariaLabel) return truncate(ariaLabel); const labelledBy = labelledByText(el); if (labelledBy) return labelledBy; const forLabel = associatedLabelText(el); if (forLabel) return forLabel; const wrapLabel = wrappingLabelText(el); if (wrapLabel) return wrapLabel; const alt = el instanceof HTMLImageElement ? normalizeWhitespace(el.alt) : normalizeWhitespace(el.getAttribute("alt") ?? ""); if (alt) return truncate(alt); const ph = placeholderOrTitle(el); if (ph) return ph; const tag = el.tagName.toLowerCase(); if (tag === "a" || el.getAttribute("role") === "link") { const linkName = compactLinkName(el); if (linkName) return linkName; } const text = ownText(el); if (text) return text; if (el instanceof HTMLInputElement || el instanceof HTMLButtonElement) { const value = normalizeWhitespace(el.value); if ( value && (el instanceof HTMLButtonElement || el.type === "submit" || el.type === "button") ) { return truncate(value); } } return undefined; } const INTERACTIVE_SELECTOR = [ "a[href]", "button", "summary", "select", 'input:not([type="hidden"])', "textarea", '[role="button"]', '[role="link"]', '[role="tab"]', '[role="menuitem"]', '[role="menuitemcheckbox"]', '[role="menuitemradio"]', '[role="option"]', '[role="checkbox"]', '[role="radio"]', '[role="switch"]', '[role="treeitem"]', '[role="combobox"]', '[contenteditable]:not([contenteditable="false"])', ].join(","); const INTERACTIVE_ROLES = new Set([ "button", "link", "tab", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "checkbox", "radio", "switch", "treeitem", "combobox", "textbox", "searchbox", "slider", ]); function looksClickable(el: Element): boolean { if (!(el instanceof HTMLElement)) return false; if (el.matches(INTERACTIVE_SELECTOR)) return true; const role = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase(); if (role && INTERACTIVE_ROLES.has(role)) return true; if (el.tabIndex >= 0) return true; try { if (getComputedStyle(el).cursor === "pointer") return true; } catch { // happy-dom / detached nodes } const cls = typeof el.className === "string" ? el.className : ""; if (/\b(btn|button|link|clickable|action)\b/i.test(cls)) return true; return false; } /** * Resolve the element a click should describe for the LLM textbook. * Never falls back to an anonymous layout `div`. */ export function resolveClickableElement(target: Element): Element | null { const interactive = target.closest(INTERACTIVE_SELECTOR); if (interactive instanceof Element) return interactive; let node: Element | null = target; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 8) { const name = accessibleName(node); if (name && isActionableLabel(name) && looksClickable(node)) { return node; } node = node.parentElement; depth += 1; } return null; } export function resolveHoverElement(target: Element): Element | null { const clickable = resolveClickableElement(target); if (clickable) return clickable; let node: Element | null = target; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 8) { if (looksClickable(node)) return node; node = node.parentElement; depth += 1; } return null; } /** * Teachable clicks: the LLM must get a label it can later find via snapshot * (visible name), or at least a form `name_attr` for checkbox/radio. * Recording “点击div” with no name fails this bar. */ export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean { const name = target.name?.trim(); if (name && isActionableLabel(name)) return true; if ( (target.role === "checkbox" || target.role === "radio" || target.tag === "input") && target.name_attr ) { return true; } return false; } export function inferRole(el: Element): string | undefined { const explicit = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase(); if (explicit) return explicit; const tag = el.tagName.toLowerCase(); if (tag === "button") return "button"; if (tag === "a" && el.hasAttribute("href")) return "link"; if (tag === "select") return "combobox"; if (tag === "textarea") return "textbox"; if (tag === "img") return "img"; if (tag === "summary") return "button"; if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) { return "textbox"; } if (el instanceof HTMLInputElement) { const type = el.type.toLowerCase(); if (type === "checkbox") return "checkbox"; if (type === "radio") return "radio"; if (type === "submit" || type === "button" || type === "image" || type === "reset") { return "button"; } if (type === "range") return "slider"; return "textbox"; } return undefined; } /** * Nearby labels help the LLM name form fields. Only apply to fillable * controls — never to buttons/links, where a previous layout sibling * (e.g. a chrome `
`) would pollute the textbook target. */ function nearbyLabelText(el: Element): string | undefined { if ( !( el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement ) ) { return undefined; } if (el.id) { const byFor = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); const text = byFor ? normalizeWhitespace(byFor.textContent ?? "") : ""; if (text) return truncate(text, ACTIONABLE_LABEL_MAX); } const wrapped = el.closest("label"); if (wrapped) { const clone = wrapped.cloneNode(true) as HTMLLabelElement; for (const control of clone.querySelectorAll("input, textarea, select, button")) { control.remove(); } const text = normalizeWhitespace(clone.textContent ?? ""); if (text) return truncate(text, ACTIONABLE_LABEL_MAX); } const prev = el.previousElementSibling; if (prev && /^(LABEL|SPAN|DIV|P|DT)$/i.test(prev.tagName)) { const text = normalizeWhitespace(prev.textContent ?? ""); if (text && text.length <= ACTIONABLE_LABEL_MAX) return text; } return undefined; } export function describeTarget(el: Element): CaptureTargetDescriptor { const tag = el.tagName.toLowerCase(); const role = inferRole(el); const name = accessibleName(el); const nameAttr = el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement || el instanceof HTMLButtonElement ? normalizeWhitespace(el.name) || undefined : normalizeWhitespace(el.getAttribute("name") ?? "") || undefined; const placeholder = el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement ? normalizeWhitespace(el.placeholder) || undefined : normalizeWhitespace(el.getAttribute("placeholder") ?? "") || undefined; const nearby = nearbyLabelText(el); return { tag, ...(role ? { role } : {}), ...(name ? { name } : {}), ...(nameAttr ? { name_attr: nameAttr } : {}), ...(placeholder ? { placeholder } : {}), ...(nearby && nearby !== name ? { nearby_label: nearby } : {}), }; } export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null { if (!(target instanceof Element)) return null; const clickable = resolveClickableElement(target); if (!clickable) return null; const desc = describeTarget(clickable); return isMeaningfulClickTarget(desc) ? desc : null; }