From e069abaf2ea6624351312b2db566ea8d648fbdd6 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 28 Aug 2026 17:15:26 +0800 Subject: [PATCH] feat(vom): support canvas reading --- .../src/tools/__tests__/interaction.test.ts | 233 +++++++++++++++++- apps/extension/src/tools/interaction.ts | 221 +++++++++++++++-- apps/extension/src/transport/types.ts | 2 + .../test-fixtures/rendered-surfaces/README.md | 23 ++ .../rendered-surfaces/canvas-input.html | 20 ++ .../rendered-surfaces/frame.html | 28 +++ .../rendered-surfaces/input-frame.html | 30 +++ .../rendered-surfaces/input-frames.html | 23 ++ crates/bsk-cli/src/cli/render_error.rs | 32 +++ 9 files changed, 587 insertions(+), 25 deletions(-) create mode 100644 apps/extension/test-fixtures/rendered-surfaces/canvas-input.html create mode 100644 apps/extension/test-fixtures/rendered-surfaces/input-frame.html create mode 100644 apps/extension/test-fixtures/rendered-surfaces/input-frames.html diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index dffb491..c0e8b10 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -34,6 +34,15 @@ function makeFakeCdp(handlers: Record unknown>) { if (!h && method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; } + if ( + !h && + method === "Runtime.evaluate" && + String((params as { expression?: string })?.expression ?? "").includes( + "bsk-focus-target-probe", + ) + ) { + return { result: { value: { hasFocus: true, activeTag: "input" } } }; + } if (!h) throw new Error(`unexpected CDP call ${method}`); return h(params); }; @@ -45,6 +54,10 @@ function makeFakeCdp(handlers: Record unknown>) { params?: object, ) => Promise, trackSessionTab: vi.fn(), + getFrameGraph: vi.fn(async (tabId) => ({ + rootFrameId: "main", + frames: [{ frameId: "main", target: { tabId } }], + })), }; const tabsApi = { get: vi.fn( @@ -55,6 +68,16 @@ function makeFakeCdp(handlers: Record unknown>) { return { cdp, tabsApi, sent }; } +function fillRuntimeReadback(value: string) { + return (params: unknown) => { + const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? ""); + if (fn.includes("bsk-input-readback")) { + return { result: { value: { supported: true, value } } }; + } + return { result: { type: "undefined" } }; + }; +} + async function makeSurfaceClickFixture(onMouse?: (params: Record) => void) { const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const context = await manager.start("aa11"); @@ -733,7 +756,7 @@ describe("handleFill", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.focus": () => ({}), "DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }), - "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }), + "Runtime.callFunctionOn": fillRuntimeReadback("hello"), "Input.insertText": () => ({}), }); const res = await handleFill( @@ -766,6 +789,9 @@ describe("handleFill", () => { "DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }), "Runtime.callFunctionOn": (p) => { const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? ""; + if (fn.includes("bsk-input-readback")) { + return { result: { value: { supported: true, value: "existingx" } } }; + } // No "this.value = ''" clearing on a no-clear path; only the // post-input dispatchEvent. expect(fn).not.toMatch(/this\.value\s*=\s*''/); @@ -779,7 +805,7 @@ describe("handleFill", () => { { cdp: fake.cdp, tabsApi: fake.tabsApi }, ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(res.value_length).toBe(1); + expect(res.value_length).toBe(9); }); it("treats contenteditable=true as fillable", async () => { @@ -797,7 +823,7 @@ describe("handleFill", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.focus": () => ({}), "DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }), - "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }), + "Runtime.callFunctionOn": fillRuntimeReadback("rich"), "Input.insertText": () => ({}), }); const res = await handleFill( @@ -808,6 +834,88 @@ describe("handleFill", () => { expect("code" in res).toBe(false); }); + it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 77, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 12, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const childCalls: string[] = []; + fake.cdp.sendToTarget = vi.fn(async (target, method, params) => { + expect(target).toEqual({ tabId: 4, sessionId: "child-session" }); + childCalls.push(method); + if (method === "DOM.describeNode") { + return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } }; + } + if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {}; + if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } }; + if (method === "Input.insertText") { + expect(params).toEqual({ text: "中文输入" }); + return {}; + } + if (method === "Runtime.callFunctionOn") { + const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? ""); + return fn.includes("bsk-input-readback") + ? { result: { value: { supported: true, value: "中文输入" } } } + : { result: { type: "undefined" } }; + } + throw new Error(`unexpected child call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const res = await handleFill( + sm, + { session_id: "aa11", ref: "@e1", value: "中文输入" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(JSON.stringify(res)); + expect(res.value_length).toBe(4); + expect(childCalls).toContain("DOM.focus"); + expect(childCalls).toContain("Input.insertText"); + expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4); + expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false); + }); + + it("returns input_not_applied when a controlled input resets the value", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 8, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }), + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.focus": () => ({}), + "DOM.resolveNode": () => ({ object: { objectId: "controlled" } }), + "Input.insertText": () => ({}), + "Runtime.callFunctionOn": fillRuntimeReadback(""), + }); + + const res = await handleFill( + sm, + { session_id: "aa11", ref: "@e1", value: "rejected" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ + code: "cdp_failed", + data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 }, + }); + }); + it("stops before Input.insertText when abort fires after clearing", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); @@ -1002,6 +1110,125 @@ describe("handlePress", () => { expect(res.key).toBe("Enter"); }); + it("routes ref-targeted key events through the OOPIF session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 555, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 12, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const childCalls: string[] = []; + fake.cdp.sendToTarget = vi.fn(async (_target, method) => { + childCalls.push(method); + return {}; + }) as CdpRunner["sendToTarget"]; + + const res = await handlePress( + sm, + { session_id: "aa11", key: "Enter", ref: "@e1" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expectPressOk(res); + expect(childCalls).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.focus", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + ]); + expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false); + }); + + it("routes an untargeted press to the uniquely focused OOPIF session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const fake = makeFakeCdp({ + "Runtime.evaluate": () => ({ + result: { value: { hasFocus: true, activeTag: "iframe" } }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const childCalls: string[] = []; + fake.cdp.sendToTarget = vi.fn(async (_target, method) => { + childCalls.push(method); + if (method === "Runtime.evaluate") { + return { result: { value: { hasFocus: true, activeTag: "textarea" } } }; + } + return {}; + }) as CdpRunner["sendToTarget"]; + + const res = await handlePress( + sm, + { session_id: "aa11", key: "a" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expectPressOk(res); + expect(childCalls).toEqual([ + "Runtime.evaluate", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + ]); + }); + + it("refuses an untargeted press when focus ownership is ambiguous", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const fake = makeFakeCdp({ + "Runtime.evaluate": () => ({ + result: { value: { hasFocus: true, activeTag: "input" } }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { frameId: "child", target: { tabId: 4, sessionId: "child-session" } }, + ], + })); + fake.cdp.sendToTarget = vi.fn(async () => ({ + result: { value: { hasFocus: true, activeTag: "textarea" } }, + })) as CdpRunner["sendToTarget"]; + + const res = await handlePress( + sm, + { session_id: "aa11", key: "Enter" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ + code: "cdp_failed", + data: { reason: "focus_target_unresolved", candidate_count: 2 }, + }); + expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false); + }); + it("stops before focus when abort fires after press scroll", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index a196f19..bb7c189 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -11,7 +11,7 @@ // commands. import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; -import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { ClickParams, @@ -217,6 +217,108 @@ async function resolveBackendNode( } } +interface InputDispatchContext { + target: CdpTarget; + cdp: CdpRunner; + backendNodeId?: number; + frameId?: string; + usedRef?: string; + usedSelector?: string; +} + +function inputContextForNode( + cdp: CdpRunner, + node: Awaited>, +): (InputDispatchContext & { backendNodeId: number }) | RpcError { + if (isRpcError(node)) return node; + return { + target: node.cdpTarget, + cdp: cdpRunnerForTarget(cdp, node.cdpTarget), + backendNodeId: node.backendNodeId, + frameId: node.frameId, + usedRef: node.usedRef, + usedSelector: node.usedSelector, + }; +} + +interface FocusProbe { + hasFocus: boolean; + activeTag: string; +} + +/** Resolve keyboard input to the CDP target that currently owns focus. */ +async function resolveFocusedInputContext( + cdp: CdpRunner, + tabId: number, +): Promise { + if (!cdp.getFrameGraph) { + return rpcError( + "cdp_failed", + "focus_target_unresolved", + "press cannot determine the focused frame target", + ); + } + + let targets: CdpTarget[]; + try { + const graph = await cdp.getFrameGraph(tabId); + const unique = new Map(); + unique.set(cdpTargetKey({ tabId }), { tabId }); + for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target); + targets = [...unique.values()]; + } catch (err) { + return rpcError( + "cdp_failed", + "focus_target_unresolved", + err instanceof Error ? err.message : String(err), + ); + } + + const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = []; + await Promise.all( + targets.map(async (candidate) => { + try { + const reply = await cdpRunnerForTarget(cdp, candidate).send<{ + result?: { value?: FocusProbe }; + }>(tabId, "Runtime.evaluate", { + expression: `(() => { + /* bsk-focus-target-probe */ + const active = document.activeElement; + return { + hasFocus: document.hasFocus(), + activeTag: active instanceof Element ? active.tagName.toLowerCase() : '' + }; + })()`, + returnByValue: true, + }); + const probe = reply.result?.value; + if (probe?.hasFocus) focused.push({ target: candidate, probe }); + } catch { + // A target may navigate or detach while the graph is being probed. + } + }), + ); + + // A top document whose active element is an iframe delegates focus to + // that frame. Removing those delegating targets leaves the deepest OOPIF + // target while same-process frames continue to use the shared root target. + const owners = focused.filter( + ({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame", + ); + if (owners.length !== 1) { + return rpcError( + "cdp_failed", + "focus_target_unresolved", + owners.length === 0 + ? "press found no unique focused frame target" + : "press found multiple focused frame targets", + { candidate_count: owners.length }, + ); + } + const target = owners[0].target; + return { target, cdp: cdpRunnerForTarget(cdp, target) }; +} + // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- @@ -618,8 +720,9 @@ export async function handleFill( const dialogCursor = markDialogCursor(deps.cdp, target.tabId); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill"); - if (isRpcError(node)) return node; - const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + const input = inputContextForNode(deps.cdp, node); + if (isRpcError(input)) return input; + const nodeCdp = input.cdp; try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); @@ -627,7 +730,7 @@ export async function handleFill( target.tabId, "DOM.describeNode", { - backendNodeId: node.backendNodeId, + backendNodeId: input.backendNodeId, }, ); if (!described.node || !isFillable(described.node)) { @@ -640,15 +743,15 @@ export async function handleFill( const scrollErr = await scrollElementAndFramesIntoView( deps.cdp, target.tabId, - node.cdpTarget, - node.backendNodeId, - node.frameId, + input.target, + input.backendNodeId, + input.frameId, ); if (scrollErr) return scrollErr; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "fill aborted" }; } - await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId }); } catch (err) { return { code: "cdp_failed", @@ -660,7 +763,7 @@ export async function handleFill( return { code: "cancelled", message: "fill aborted" }; } - const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId); if (isRpcError(objectIdOrErr)) return objectIdOrErr; const objectId = objectIdOrErr; const clearBefore = params.clear_before ?? true; @@ -691,7 +794,7 @@ export async function handleFill( } // CDP `Input.insertText` handles IME / multi-byte input out of the // box, much more reliably than per-key `dispatchKeyEvent`. - await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value }); + await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value }); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "fill aborted" }; } @@ -704,6 +807,28 @@ export async function handleFill( }`, returnByValue: true, }); + // Give controlled inputs one bounded browser-render turn to accept or + // reject the dispatched value before reporting success. + await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { + /* bsk-input-sync */ + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(finish, 100); + if (typeof requestAnimationFrame !== 'function') { finish(); return; } + requestAnimationFrame(() => requestAnimationFrame(finish)); + }); + }`, + awaitPromise: true, + returnByValue: true, + }); } catch (err) { return { code: "cdp_failed", @@ -711,11 +836,56 @@ export async function handleFill( }; } + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "fill aborted" }; + } + + let actualValue: string; + try { + const readback = await nodeCdp.send<{ + result?: { value?: { supported?: boolean; value?: string } }; + }>(target.tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { + /* bsk-input-readback */ + if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) { + return { supported: true, value: this.value }; + } + if (this.isContentEditable) { + return { supported: true, value: this.innerText ?? this.textContent ?? '' }; + } + return { supported: false, value: '' }; + }`, + returnByValue: true, + }); + const state = readback.result?.value; + actualValue = typeof state?.value === "string" ? state.value : ""; + const applied = + state?.supported === true && + (clearBefore + ? actualValue === params.value + : params.value.length === 0 || actualValue.includes(params.value)); + if (!applied) { + return rpcError( + "cdp_failed", + "input_not_applied", + "the page did not retain the requested input value", + { expected_value_length: params.value.length, actual_value_length: actualValue.length }, + ); + } + } catch (err) { + return rpcError( + "cdp_failed", + "input_not_applied", + err instanceof Error ? err.message : String(err), + ); + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - value_length: params.value.length, + used_ref: input.usedRef, + used_selector: input.usedSelector, + value_length: actualValue.length, }); } @@ -890,31 +1060,38 @@ export async function handlePress( }; } + let input: InputDispatchContext | RpcError; // Optional focus before key dispatch. if (params.ref || params.selector) { const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press"); - if (isRpcError(node)) return node; - const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + const explicitInput = inputContextForNode(deps.cdp, node); + if (isRpcError(explicitInput)) return explicitInput; + input = explicitInput; try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const scrollErr = await scrollElementAndFramesIntoView( deps.cdp, target.tabId, - node.cdpTarget, - node.backendNodeId, - node.frameId, + explicitInput.target, + explicitInput.backendNodeId, + explicitInput.frameId, ); if (scrollErr) return scrollErr; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "press aborted" }; } - await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + await explicitInput.cdp.send(target.tabId, "DOM.focus", { + backendNodeId: explicitInput.backendNodeId, + }); } catch (err) { return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } + } else { + input = await resolveFocusedInputContext(deps.cdp, target.tabId); + if (isRpcError(input)) return input; } if (throwIfAborted(deps.signal)) { @@ -928,7 +1105,7 @@ export async function handlePress( try { let cancelled = false; deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { + await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: descriptor.key, code: descriptor.code, @@ -942,7 +1119,7 @@ export async function handlePress( typeof descriptor.text === "string" && descriptor.text.length > 0 ) { - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { + await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "char", key: descriptor.key, code: descriptor.code, @@ -959,7 +1136,7 @@ export async function handlePress( cancelled = true; } } - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { + await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp", key: descriptor.key, code: descriptor.code, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 9ec464a..0e6b6aa 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -27,6 +27,8 @@ export type RpcErrorReason = | "ref_not_found" | "selector_not_found" | "target_not_fillable" + | "focus_target_unresolved" + | "input_not_applied" | "target_not_select" | "option_not_found" | "single_select_value_count" diff --git a/apps/extension/test-fixtures/rendered-surfaces/README.md b/apps/extension/test-fixtures/rendered-surfaces/README.md index a377507..dafa382 100644 --- a/apps/extension/test-fixtures/rendered-surfaces/README.md +++ b/apps/extension/test-fixtures/rendered-surfaces/README.md @@ -93,6 +93,29 @@ Run with a `900x700` Agent Window. - `fixture-below-fold` is absent initially. - After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref. +### `input-frames.html` + +This page exercises generic frame-aware input routing. It contains a top-level +input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each +frame includes an input, textarea, contenteditable editor, and a controlled +input that resets its value on the next animation frame. + +- `fill` must route focus, text insertion, DOM events, and readback through the + ref's owning CDP target. +- `press --ref` must focus and dispatch every key event through that same target. +- Untargeted `press` must follow the actual focused target or refuse when focus + ownership cannot be determined uniquely. +- The rejecting input must return `input_not_applied`, never a successful + requested-value length. + +### `canvas-input.html` + +This is the PR3 end-to-end fixture. Its cross-origin frame contains a +visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and +focuses a standard textbox; ordinary `fill` must then write and read back its +real value through the textbox's owning target. The fixture deliberately does +not provide a Canvas-specific fill action. + ### `dynamic.html` Use the normal DOM buttons and re-observe after each action: diff --git a/apps/extension/test-fixtures/rendered-surfaces/canvas-input.html b/apps/extension/test-fixtures/rendered-surfaces/canvas-input.html new file mode 100644 index 0000000..9dd5b25 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/canvas-input.html @@ -0,0 +1,20 @@ + + + + + + Canvas to standard input fixture + + + +

Canvas to standard input fixture

+

Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.

+ + + diff --git a/apps/extension/test-fixtures/rendered-surfaces/frame.html b/apps/extension/test-fixtures/rendered-surfaces/frame.html index da17b81..3b1f02f 100644 --- a/apps/extension/test-fixtures/rendered-surfaces/frame.html +++ b/apps/extension/test-fixtures/rendered-surfaces/frame.html @@ -19,6 +19,18 @@ if (params.get("canvas") === "0") canvas.remove(); else { canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default"); + if (params.get("editor") === "1") { + const context = canvas.getContext("2d"); + context.font = "14px system-ui, sans-serif"; + context.textBaseline = "middle"; + context.strokeStyle = "#1e8449"; + context.fillStyle = "#145a32"; + for (let row = 0; row < 3; row += 1) { + const top = row * 30; + context.strokeRect(0.5, top + 0.5, 219, 29); + context.fillText(`Row ${row + 1}`, 12, top + 15); + } + } if (params.get("interactive") === "1") { const status = document.createElement("p"); status.id = "pointer-status"; @@ -39,6 +51,22 @@ canvas.addEventListener("click", () => { status.textContent += "; click received"; }); + if (params.get("editor") === "1") { + canvas.addEventListener("click", (event) => { + const rect = canvas.getBoundingClientRect(); + const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height; + const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30))); + status.textContent += `; selected row=${row + 1}`; + document.querySelector("#canvas-cell-editor")?.remove(); + if (row !== 2) return; + const editor = document.createElement("input"); + editor.id = "canvas-cell-editor"; + editor.setAttribute("aria-label", "selected canvas cell editor"); + editor.placeholder = "Type into selected row 3"; + canvas.insertAdjacentElement("afterend", editor); + editor.focus(); + }); + } } } if (params.get("nested") === "1") { diff --git a/apps/extension/test-fixtures/rendered-surfaces/input-frame.html b/apps/extension/test-fixtures/rendered-surfaces/input-frame.html new file mode 100644 index 0000000..528ea9e --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/input-frame.html @@ -0,0 +1,30 @@ + + + + + + + + + + +
+ + + + diff --git a/apps/extension/test-fixtures/rendered-surfaces/input-frames.html b/apps/extension/test-fixtures/rendered-surfaces/input-frames.html new file mode 100644 index 0000000..741f513 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/input-frames.html @@ -0,0 +1,23 @@ + + + + + + Frame input routing matrix + + + +

Frame input routing matrix

+ +
+ + + +
+ + diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 40933ae..0a31f99 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -44,6 +44,8 @@ pub mod reason { pub const REF_NOT_FOUND: &str = "ref_not_found"; pub const SELECTOR_NOT_FOUND: &str = "selector_not_found"; pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable"; + pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved"; + pub const INPUT_NOT_APPLIED: &str = "input_not_applied"; pub const TARGET_NOT_SELECT: &str = "target_not_select"; pub const OPTION_NOT_FOUND: &str = "option_not_found"; pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count"; @@ -234,6 +236,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, + (ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo { + summary: "focused frame target could not be determined", + hint: Some( + "pass an explicit --ref or --selector so the key event can be routed safely", + ), + exit_code: base.exit_code, + }, + (ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo { + summary: "the page did not retain the requested input", + hint: Some( + "refresh the snapshot and retry; the page may be rejecting or resetting controlled input", + ), + exit_code: base.exit_code, + }, (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo { summary: "snapshot ref does not support this operation", hint: Some( @@ -532,4 +548,20 @@ mod tests { ); assert!(info.hint.unwrap().contains("exactly one")); } + + #[test] + fn focus_target_unresolved_overrides_cdp_copy() { + let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&data)); + assert_eq!(info.summary, "focused frame target could not be determined"); + assert!(info.hint.unwrap().contains("--ref")); + } + + #[test] + fn input_not_applied_overrides_cdp_copy() { + let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&data)); + assert_eq!(info.summary, "the page did not retain the requested input"); + assert!(info.hint.unwrap().contains("rejecting")); + } }