diff --git a/apps/extension/src/tools/__tests__/editable-input.test.ts b/apps/extension/src/tools/__tests__/editable-input.test.ts new file mode 100644 index 0000000..9a01b62 --- /dev/null +++ b/apps/extension/src/tools/__tests__/editable-input.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it, vi } from "vitest"; +import { fillEditable } from "../editable-input"; +import type { CdpRunner } from "../shared"; + +interface FakeOptions { + initialValue?: string; + finalValue: string; + remount?: boolean; +} + +function editableState(value: string, connected = true, focused = true) { + return { + result: { + value: { supported: true, connected, focused, value, kind: "input" }, + }, + }; +} + +function makeEditableCdp(options: FakeOptions) { + const sent: Array<{ method: string; params?: object }> = []; + let inserted = false; + const send = vi.fn(async (_tabId: number, method: string, params?: object): Promise => { + sent.push({ method, params }); + if (method === "Input.insertText") { + inserted = true; + return {} as T; + } + if (method === "Input.dispatchKeyEvent") { + if ((params as { type?: string }).type === "rawKeyDown") inserted = true; + return {} as T; + } + if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`); + + const call = params as { objectId?: string; functionDeclaration?: string }; + const fn = call.functionDeclaration ?? ""; + if (fn.includes("bsk-editable-selection")) { + return { result: { value: { selected: true } } } as T; + } + if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T; + if (fn.includes("bsk-editable-live")) { + return { + result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" }, + } as T; + } + if (fn.includes("bsk-editable-state")) { + if (call.objectId === "replacement") return editableState(options.finalValue) as T; + if (options.remount && inserted) { + return editableState(options.finalValue, false, false) as T; + } + return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T; + } + throw new Error("unexpected Runtime.callFunctionOn function"); + }); + return { cdp: { send } as CdpRunner, sent }; +} + +describe("fillEditable", () => { + it("replaces text through the browser input path without synthetic events or value setters", async () => { + const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" }); + + const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true); + + expect(result).toMatchObject({ value: "新值", replacedNode: false }); + expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } }); + const declarations = fake.sent + .map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "") + .join("\n"); + expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/); + }); + + it("reacquires the focused editable after a framework remount", async () => { + const fake = makeEditableCdp({ finalValue: "persisted", remount: true }); + + const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true); + + expect(result).toMatchObject({ value: "persisted", replacedNode: true }); + }); + + it("reports a page rollback after the bounded settlement", async () => { + const fake = makeEditableCdp({ finalValue: "" }); + + const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { + reason: "input_not_applied", + expected_value_length: 8, + actual_value_length: 0, + phase: "readback", + }, + }); + }); + + it("clears to an empty value with a browser Backspace edit", async () => { + const fake = makeEditableCdp({ initialValue: "old", finalValue: "" }); + + const result = await fillEditable(fake.cdp, 4, "anchor", "", true); + + expect(result).toMatchObject({ value: "" }); + expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false); + expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2); + }); +}); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index c0e8b10..dbf44de 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -7,6 +7,7 @@ import { handleHover, handlePress, handleSelect, + handleType, modifiersBitfield, parseKeySpec, resolveKeyDescriptor, @@ -71,8 +72,21 @@ function makeFakeCdp(handlers: Record unknown>) { function fillRuntimeReadback(value: string) { return (params: unknown) => { const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? ""); - if (fn.includes("bsk-input-readback")) { - return { result: { value: { supported: true, value } } }; + if (fn.includes("bsk-editable-state")) { + return { + result: { + value: { + supported: true, + connected: true, + focused: true, + value, + kind: "input", + }, + }, + }; + } + if (fn.includes("bsk-editable-selection")) { + return { result: { value: { selected: true } } }; } return { result: { type: "undefined" } }; }; @@ -770,13 +784,13 @@ describe("handleFill", () => { expect(res.used_ref).toBe("e1"); const insert = fake.sent.find((c) => c.method === "Input.insertText"); expect(insert?.params).toEqual({ text: "hello" }); - // clear_before defaults to true → callFunctionOn invoked once to - // clear, once to fire input/change after typing. + // clear_before defaults to true, so the edit transaction prepares a + // replacement selection and performs a settled live-value readback. const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn"); expect(callFns.length).toBeGreaterThanOrEqual(2); }); - it("clear_before=false skips the wipe call", async () => { + it("clear_before=false skips replacement selection", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e1", 1, { tabId: 4 }); @@ -789,12 +803,20 @@ describe("handleFill", () => { "DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }), "Runtime.callFunctionOn": (p) => { const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? ""; - if (fn.includes("bsk-input-readback")) { - return { result: { value: { supported: true, value: "existingx" } } }; + if (fn.includes("bsk-editable-state")) { + return { + result: { + value: { + supported: true, + connected: true, + focused: true, + value: "existingx", + kind: "textarea", + }, + }, + }; } - // No "this.value = ''" clearing on a no-clear path; only the - // post-input dispatchEvent. - expect(fn).not.toMatch(/this\.value\s*=\s*''/); + expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/); return { result: { type: "undefined" } }; }, "Input.insertText": () => ({}), @@ -870,9 +892,23 @@ describe("handleFill", () => { } if (method === "Runtime.callFunctionOn") { const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? ""); - return fn.includes("bsk-input-readback") - ? { result: { value: { supported: true, value: "中文输入" } } } - : { result: { type: "undefined" } }; + if (fn.includes("bsk-editable-state")) { + return { + result: { + value: { + supported: true, + connected: true, + focused: true, + value: "中文输入", + kind: "input", + }, + }, + }; + } + if (fn.includes("bsk-editable-selection")) { + return { result: { value: { selected: true } } }; + } + return { result: { type: "undefined" } }; } throw new Error(`unexpected child call ${method}`); }) as CdpRunner["sendToTarget"]; @@ -916,7 +952,7 @@ describe("handleFill", () => { }); }); - it("stops before Input.insertText when abort fires after clearing", async () => { + it("stops before Input.insertText when abort fires after preparing replacement", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e1", 555, { tabId: 4 }); @@ -928,8 +964,25 @@ describe("handleFill", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.focus": () => ({}), "DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }), - "Runtime.callFunctionOn": () => { - abort.abort(); + "Runtime.callFunctionOn": (params) => { + const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? ""); + if (fn.includes("bsk-editable-state")) { + return { + result: { + value: { + supported: true, + connected: true, + focused: true, + value: "", + kind: "input", + }, + }, + }; + } + if (fn.includes("bsk-editable-selection")) { + abort.abort(); + return { result: { value: { selected: true } } }; + } return { result: { type: "undefined" } }; }, "Input.insertText": () => ({}), @@ -972,6 +1025,204 @@ describe("handleFill", () => { }); }); +describe("handleType", () => { + it("requires an explicit target mode instead of silently using focus", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const fake = makeFakeCdp({}); + + const missing = await handleType( + sm, + { session_id: "aa11", text: "x" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + expect(missing).toMatchObject({ + code: "invalid_params", + message: expect.stringContaining("focused=true"), + }); + + const mixed = await handleType( + sm, + { session_id: "aa11", ref: "@e1", focused: true, text: "x" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + expect(mixed).toMatchObject({ + code: "invalid_params", + message: expect.stringContaining("not both"), + }); + expect(fake.sent).toHaveLength(0); + }); + + it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 77, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 12, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const childCalls: Array<{ method: string; params?: object }> = []; + fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => { + childCalls.push({ method, params }); + if (method === "DOM.describeNode") { + return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } }; + } + return {}; + }) as CdpRunner["sendToTarget"]; + + const result = await handleType( + sm, + { session_id: "aa11", ref: "@e1", text: "测试" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(JSON.stringify(result)); + expect(result).toMatchObject({ + tab_id: 4, + used_ref: "e1", + text_length: 2, + verification: "dispatched", + }); + expect(childCalls.map((call) => call.method)).toEqual([ + "DOM.describeNode", + "DOM.scrollIntoViewIfNeeded", + "DOM.focus", + "Input.dispatchKeyEvent", + "Input.insertText", + "Input.dispatchKeyEvent", + ]); + expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false); + expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false); + }); + + it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const fake = makeFakeCdp({ + "Runtime.evaluate": () => ({ + result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const childCalls: string[] = []; + fake.cdp.sendToTarget = vi.fn(async (_target, method) => { + childCalls.push(method); + if (method === "Runtime.evaluate") { + return { + result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } }, + }; + } + return {}; + }) as CdpRunner["sendToTarget"]; + + const result = await handleType( + sm, + { session_id: "aa11", focused: true, text: "A" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(JSON.stringify(result)); + expect(result).toMatchObject({ text_length: 1, verification: "dispatched" }); + expect(childCalls).toEqual([ + "Runtime.evaluate", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + ]); + expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false); + }); + + it("refuses focused mode when focus is not owned by a text-entry target", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const fake = makeFakeCdp({ + "Runtime.evaluate": () => ({ + result: { value: { hasFocus: true, activeTag: "button", textEntry: false } }, + }), + }); + + const result = await handleType( + sm, + { session_id: "aa11", focused: true, text: "x" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "focus_target_unresolved", candidate_count: 0 }, + }); + expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false); + }); + + it("rejects explicit non-text input controls", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 77, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.describeNode": () => ({ + node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] }, + }), + }); + + const result = await handleType( + sm, + { session_id: "aa11", ref: "@e1", text: "x" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(result).toMatchObject({ + code: "invalid_params", + data: { reason: "target_not_fillable" }, + }); + expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false); + }); + + it("rejects a Surface ref instead of treating it as a text target", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e1", 77, { + tabId: 4, + kind: "surface", + capabilities: ["screenshot"], + }); + const fake = makeFakeCdp({}); + + const result = await handleType( + sm, + { session_id: "aa11", ref: "@e1", text: "x" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(result).toMatchObject({ + code: "permission_denied", + data: { reason: "ref_capability_denied", required_capability: "interact" }, + }); + }); +}); + describe("parseKeySpec", () => { it("splits compound expressions into modifiers + base key", () => { expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] }); diff --git a/apps/extension/src/tools/__tests__/text-input.test.ts b/apps/extension/src/tools/__tests__/text-input.test.ts new file mode 100644 index 0000000..522441a --- /dev/null +++ b/apps/extension/src/tools/__tests__/text-input.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpRunner } from "../shared"; +import { dispatchTextInput } from "../text-input"; + +function makeCdp(onSend?: (method: string, params?: object) => void) { + const sent: Array<{ method: string; params?: object }> = []; + const send = vi.fn(async (_tabId: number, method: string, params?: object): Promise => { + sent.push({ method, params }); + onSend?.(method, params); + return {} as T; + }); + return { cdp: { send } as CdpRunner, sent }; +} + +describe("dispatchTextInput", () => { + it("types ASCII through complete keydown, char, and keyup sequences", async () => { + const fake = makeCdp(); + + const result = await dispatchTextInput(fake.cdp, 4, "Ab"); + + expect(result).toBeNull(); + expect( + fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`), + ).toEqual([ + "Input.dispatchKeyEvent:rawKeyDown", + "Input.dispatchKeyEvent:char", + "Input.dispatchKeyEvent:keyUp", + "Input.dispatchKeyEvent:rawKeyDown", + "Input.dispatchKeyEvent:char", + "Input.dispatchKeyEvent:keyUp", + ]); + expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 }); + expect(fake.sent[1].params).toMatchObject({ text: "A" }); + }); + + it("types non-ASCII through the committed IME insertion path", async () => { + const fake = makeCdp(); + + const result = await dispatchTextInput(fake.cdp, 4, "测试"); + + expect(result).toBeNull(); + expect(fake.sent).toEqual([ + { + method: "Input.dispatchKeyEvent", + params: { + type: "rawKeyDown", + key: "Process", + code: "", + windowsVirtualKeyCode: 229, + nativeVirtualKeyCode: 229, + modifiers: 0, + }, + }, + { method: "Input.insertText", params: { text: "测试" } }, + { + method: "Input.dispatchKeyEvent", + params: { + type: "keyUp", + key: "Process", + code: "", + windowsVirtualKeyCode: 229, + nativeVirtualKeyCode: 229, + modifiers: 0, + }, + }, + ]); + }); + + it("preserves text order across keyboard and IME segments", async () => { + const fake = makeCdp(); + + const result = await dispatchTextInput(fake.cdp, 4, "A测试1"); + + expect(result).toBeNull(); + const meaningful = fake.sent.flatMap((call) => { + if (call.method === "Input.insertText") { + return [`ime:${(call.params as { text: string }).text}`]; + } + if ( + call.method === "Input.dispatchKeyEvent" && + (call.params as { type?: string }).type === "char" + ) { + return [`key:${(call.params as { text: string }).text}`]; + } + return []; + }); + expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]); + }); + + it("reports cancellation observed after an accepted IME insertion", async () => { + const abort = new AbortController(); + const fake = makeCdp((method) => { + if (method === "Input.insertText") abort.abort(); + }); + + const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal); + + expect(result).toMatchObject({ code: "cancelled" }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "Input.dispatchKeyEvent", + "Input.insertText", + "Input.dispatchKeyEvent", + ]); + expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" }); + }); + + it("releases the IME process key without committing when cancellation follows keydown", async () => { + const abort = new AbortController(); + const fake = makeCdp((method, params) => { + if ( + method === "Input.dispatchKeyEvent" && + (params as { type?: string }).type === "rawKeyDown" + ) { + abort.abort(); + } + }); + + const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal); + + expect(result).toMatchObject({ code: "cancelled" }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + ]); + expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" }); + expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" }); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index e0a5ac4..8687409 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -28,6 +28,7 @@ import type { ScreenshotParams, SelectParams, SnapshotParams, + TypeParams, WaitForNavigationParams, } from "@/transport/types"; import { isRequestFrame } from "@/transport/types"; @@ -35,7 +36,14 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; +import { + handleClick, + handleFill, + handleHover, + handlePress, + handleSelect, + handleType, +} from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -491,6 +499,17 @@ export class ToolDispatcher { ), signal, ); + case "tool.type": + return this.withHoverReleaseForRequest( + req.params as TypeParams, + () => + handleType( + this.sessions, + req.params as TypeParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.press": return this.withHoverReleaseForRequest( req.params as PressParams, @@ -717,6 +736,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.click": case "tool.hover": case "tool.fill": + case "tool.type": case "tool.press": case "tool.select": case "tool.evaluate": diff --git a/apps/extension/src/tools/editable-input.ts b/apps/extension/src/tools/editable-input.ts new file mode 100644 index 0000000..283b574 --- /dev/null +++ b/apps/extension/src/tools/editable-input.ts @@ -0,0 +1,282 @@ +// Browser-native editing transaction for input, textarea, and contenteditable +// targets. The CDP target/session is selected by interaction.ts; this module +// owns the shorter-lived editable identity inside that target. + +import type { RpcError } from "@/transport/types"; +import { rpcError } from "./errors"; +import type { CdpRunner } from "./shared"; + +interface EditableState { + supported: boolean; + connected: boolean; + focused: boolean; + value: string; + kind: "input" | "textarea" | "contenteditable" | "unsupported"; +} + +export interface EditableFillResult { + value: string; + replacedNode: boolean; +} + +function cancelled(signal: AbortSignal | undefined): RpcError | null { + return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null; +} + +async function inspectEditable( + cdp: CdpRunner, + tabId: number, + objectId: string, +): Promise { + const reply = await cdp.send<{ + result?: { value?: EditableState }; + }>(tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { + /* bsk-editable-state */ + const doc = this.ownerDocument; + const focused = !!doc && doc.activeElement === this; + if (this instanceof HTMLInputElement) { + return { supported: true, connected: this.isConnected, focused, + value: this.value, kind: 'input' }; + } + if (this instanceof HTMLTextAreaElement) { + return { supported: true, connected: this.isConnected, focused, + value: this.value, kind: 'textarea' }; + } + if (this.isContentEditable) { + return { supported: true, connected: this.isConnected, focused, + value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' }; + } + return { supported: false, connected: this.isConnected === true, focused, + value: '', kind: 'unsupported' }; + }`, + returnByValue: true, + }); + const state = reply.result?.value; + if (!state || typeof state.value !== "string") { + throw new Error("editable state probe returned no value"); + } + return state; +} + +/** + * Return the active editable from the anchor's own document. Using the + * anchor's execution context also works for same-process nested frames, where + * a target-level Runtime.evaluate would otherwise see the top document. + */ +async function focusedEditableFromAnchor( + cdp: CdpRunner, + tabId: number, + anchorObjectId: string, +): Promise { + const reply = await cdp.send<{ + result?: { objectId?: string; subtype?: string }; + }>(tabId, "Runtime.callFunctionOn", { + objectId: anchorObjectId, + functionDeclaration: `function() { + /* bsk-editable-live */ + const active = this.ownerDocument?.activeElement; + if (!active) return null; + const editable = active instanceof HTMLInputElement || + active instanceof HTMLTextAreaElement || active.isContentEditable; + return editable ? active : null; + }`, + returnByValue: false, + }); + if (reply.result?.subtype === "null") return null; + return typeof reply.result?.objectId === "string" ? reply.result.objectId : null; +} + +async function resolveLiveEditable( + cdp: CdpRunner, + tabId: number, + anchorObjectId: string, +): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> { + const anchor = await inspectEditable(cdp, tabId, anchorObjectId); + if (anchor.supported && anchor.connected && anchor.focused) { + return { objectId: anchorObjectId, state: anchor, replaced: false }; + } + + const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId); + if (!focusedObjectId) { + return rpcError( + "cdp_failed", + "input_not_applied", + "the target editable lost focus or detached before input", + { phase: "resolve_live_editable" }, + ); + } + const focused = await inspectEditable(cdp, tabId, focusedObjectId); + if (!focused.supported || !focused.connected || !focused.focused) { + return rpcError( + "cdp_failed", + "input_not_applied", + "the replacement editable was not live and focused", + { phase: "resolve_live_editable" }, + ); + } + return { objectId: focusedObjectId, state: focused, replaced: true }; +} + +async function selectEditableContents( + cdp: CdpRunner, + tabId: number, + objectId: string, +): Promise { + const reply = await cdp.send<{ + result?: { value?: { selected?: boolean } }; + }>(tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { + /* bsk-editable-selection */ + if (!this.isConnected || this.ownerDocument?.activeElement !== this) { + return { selected: false }; + } + if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) { + try { + this.setSelectionRange(0, this.value.length); + } catch { + if (typeof this.select !== 'function') return { selected: false }; + this.select(); + } + return { selected: true }; + } + if (this.isContentEditable) { + const selection = this.ownerDocument.getSelection(); + if (!selection) return { selected: false }; + const range = this.ownerDocument.createRange(); + range.selectNodeContents(this); + selection.removeAllRanges(); + selection.addRange(range); + return { selected: true }; + } + return { selected: false }; + }`, + returnByValue: true, + }); + if (reply.result?.value?.selected !== true) { + throw new Error("could not select the live editable contents"); + } +} + +async function waitForEditableSettlement( + cdp: CdpRunner, + tabId: number, + anchorObjectId: string, +): Promise { + await cdp.send(tabId, "Runtime.callFunctionOn", { + objectId: anchorObjectId, + functionDeclaration: `function() { + /* bsk-editable-sync */ + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(finish, 100); + const view = this.ownerDocument?.defaultView; + if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; } + view.requestAnimationFrame(() => view.requestAnimationFrame(finish)); + }); + }`, + awaitPromise: true, + returnByValue: true, + }); +} + +async function deleteSelection(cdp: CdpRunner, tabId: number): Promise { + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "rawKeyDown", + key: "Backspace", + code: "Backspace", + windowsVirtualKeyCode: 8, + }); + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "keyUp", + key: "Backspace", + code: "Backspace", + windowsVirtualKeyCode: 8, + }); +} + +/** + * Replace or append text through Chromium's editing pipeline. No value setter + * or synthetic DOM events are used: controlled components observe the same + * beforeinput/input path as a real browser edit. The live editable is resolved + * again after settlement so a framework may remount it without producing a + * false failure or a stale-object success. + */ +export async function fillEditable( + cdp: CdpRunner, + tabId: number, + anchorObjectId: string, + value: string, + clearBefore: boolean, + signal?: AbortSignal, +): Promise { + try { + const abortedBefore = cancelled(signal); + if (abortedBefore) return abortedBefore; + + const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId); + if ("code" in initial) return initial; + let replacedNode = initial.replaced; + + if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId); + const abortedAfterSelection = cancelled(signal); + if (abortedAfterSelection) return abortedAfterSelection; + + if (value.length > 0) { + await cdp.send(tabId, "Input.insertText", { text: value }); + } else if (clearBefore) { + await deleteSelection(cdp, tabId); + } + + const abortedAfterInput = cancelled(signal); + if (abortedAfterInput) return abortedAfterInput; + await waitForEditableSettlement(cdp, tabId, initial.objectId); + + const abortedAfterSettlement = cancelled(signal); + if (abortedAfterSettlement) return abortedAfterSettlement; + + let finalObjectId = initial.objectId; + let finalState = await inspectEditable(cdp, tabId, finalObjectId); + if (!finalState.connected || !finalState.focused) { + const live = await resolveLiveEditable(cdp, tabId, initial.objectId); + if ("code" in live) return live; + finalObjectId = live.objectId; + finalState = live.state; + replacedNode ||= live.replaced || finalObjectId !== initial.objectId; + } + + const applied = clearBefore + ? finalState.value === value + : value.length === 0 || finalState.value.includes(value); + if (!finalState.supported || !applied) { + return rpcError( + "cdp_failed", + "input_not_applied", + "the page did not retain the requested input value", + { + expected_value_length: value.length, + actual_value_length: finalState.value.length, + phase: "readback", + editable_replaced: replacedNode, + }, + ); + } + + return { value: finalState.value, replacedNode }; + } catch (err) { + return rpcError( + "cdp_failed", + "input_not_applied", + err instanceof Error ? err.message : String(err), + { phase: "edit_transaction" }, + ); + } +} diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index bb7c189..b17634d 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -1,5 +1,5 @@ -// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and -// `tool.select`. +// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`, +// `tool.press`, and `tool.select`. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). @@ -27,8 +27,11 @@ import type { RpcError, SelectParams, SelectResult, + TypeParams, + TypeResult, } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; +import { fillEditable } from "./editable-input"; import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; @@ -45,6 +48,7 @@ import { } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSurfacePointerTarget } from "./surface-target"; +import { dispatchTextInput } from "./text-input"; export interface InteractionDeps { cdp: CdpRunner; @@ -244,18 +248,20 @@ function inputContextForNode( interface FocusProbe { hasFocus: boolean; activeTag: string; + textEntry: boolean; } /** Resolve keyboard input to the CDP target that currently owns focus. */ async function resolveFocusedInputContext( cdp: CdpRunner, tabId: number, + mode: "press" | "type", ): Promise { if (!cdp.getFrameGraph) { return rpcError( "cdp_failed", "focus_target_unresolved", - "press cannot determine the focused frame target", + `${mode} cannot determine the focused frame target`, ); } @@ -284,9 +290,21 @@ async function resolveFocusedInputContext( expression: `(() => { /* bsk-focus-target-probe */ const active = document.activeElement; + const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : ''; + const nonTextInputTypes = new Set([ + 'button', 'checkbox', 'color', 'date', 'datetime-local', 'file', + 'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit', + 'time', 'week' + ]); + const role = active instanceof Element ? active.getAttribute('role') : null; return { hasFocus: document.hasFocus(), - activeTag: active instanceof Element ? active.tagName.toLowerCase() : '' + activeTag: active instanceof Element ? active.tagName.toLowerCase() : '', + textEntry: + (active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) || + active instanceof HTMLTextAreaElement || + (active instanceof HTMLElement && active.isContentEditable) || + role === 'textbox' || role === 'searchbox' }; })()`, returnByValue: true, @@ -302,16 +320,17 @@ async function resolveFocusedInputContext( // A top document whose active element is an iframe delegates focus to // that frame. Removing those delegating targets leaves the deepest OOPIF // target while same-process frames continue to use the shared root target. - const owners = focused.filter( - ({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame", - ); + const owners = focused.filter(({ probe }) => { + if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false; + return mode !== "type" || probe.textEntry; + }); if (owners.length !== 1) { return rpcError( "cdp_failed", "focus_target_unresolved", owners.length === 0 - ? "press found no unique focused frame target" - : "press found multiple focused frame targets", + ? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target` + : `${mode} found multiple focused frame targets`, { candidate_count: owners.length }, ); } @@ -682,6 +701,47 @@ function isFillable(node: DescribedNode): boolean { return false; } +/** `type` also supports custom text-entry widgets that expose an ARIA role. */ +function isTypeTarget(node: DescribedNode): boolean { + const tag = (node.nodeName ?? "").toUpperCase(); + const attrs = node.attributes ?? []; + const attributes = new Map(); + for (let i = 0; i + 1 < attrs.length; i += 2) { + attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase()); + } + if (tag === "TEXTAREA") return true; + if (tag === "INPUT") { + const type = attributes.get("type") ?? "text"; + return !new Set([ + "button", + "checkbox", + "color", + "date", + "datetime-local", + "file", + "hidden", + "image", + "month", + "radio", + "range", + "reset", + "submit", + "time", + "week", + ]).has(type); + } + const contentEditable = attributes.get("contenteditable"); + if ( + contentEditable === "" || + contentEditable === "true" || + contentEditable === "plaintext-only" + ) { + return true; + } + const role = attributes.get("role"); + return role === "textbox" || role === "searchbox"; +} + type SelectMutationResult = | { ok: true; @@ -767,125 +827,114 @@ export async function handleFill( if (isRpcError(objectIdOrErr)) return objectIdOrErr; const objectId = objectIdOrErr; const clearBefore = params.clear_before ?? true; - - try { - if (clearBefore) { - // Clear input/textarea value or wipe contenteditable innerText, - // then fire `input` so frameworks observe the empty state. - await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: `function() { - if (this.isContentEditable) { this.textContent = ''; } - else { - const proto = this instanceof HTMLTextAreaElement - ? HTMLTextAreaElement.prototype - : HTMLInputElement.prototype; - const descriptor = Object.getOwnPropertyDescriptor(proto, 'value'); - if (descriptor && descriptor.set) descriptor.set.call(this, ''); - else this.value = ''; - } - this.dispatchEvent(new Event('input', { bubbles: true })); - }`, - returnByValue: true, - }); - } - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } - // CDP `Input.insertText` handles IME / multi-byte input out of the - // box, much more reliably than per-key `dispatchKeyEvent`. - await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value }); - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } - // Fire `input` + `change` so React / Vue controlled inputs commit. - await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: `function() { - this.dispatchEvent(new Event('input', { bubbles: true })); - this.dispatchEvent(new Event('change', { bubbles: true })); - }`, - returnByValue: true, - }); - // Give controlled inputs one bounded browser-render turn to accept or - // reject the dispatched value before reporting success. - await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: `function() { - /* bsk-input-sync */ - return new Promise((resolve) => { - let settled = false; - const finish = () => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(); - }; - const timer = setTimeout(finish, 100); - if (typeof requestAnimationFrame !== 'function') { finish(); return; } - requestAnimationFrame(() => requestAnimationFrame(finish)); - }); - }`, - awaitPromise: true, - returnByValue: true, - }); - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } - - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } - - let actualValue: string; - try { - const readback = await nodeCdp.send<{ - result?: { value?: { supported?: boolean; value?: string } }; - }>(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: `function() { - /* bsk-input-readback */ - if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) { - return { supported: true, value: this.value }; - } - if (this.isContentEditable) { - return { supported: true, value: this.innerText ?? this.textContent ?? '' }; - } - return { supported: false, value: '' }; - }`, - returnByValue: true, - }); - const state = readback.result?.value; - actualValue = typeof state?.value === "string" ? state.value : ""; - const applied = - state?.supported === true && - (clearBefore - ? actualValue === params.value - : params.value.length === 0 || actualValue.includes(params.value)); - if (!applied) { - return rpcError( - "cdp_failed", - "input_not_applied", - "the page did not retain the requested input value", - { expected_value_length: params.value.length, actual_value_length: actualValue.length }, - ); - } - } catch (err) { - return rpcError( - "cdp_failed", - "input_not_applied", - err instanceof Error ? err.message : String(err), - ); - } + const fillResult = await fillEditable( + nodeCdp, + target.tabId, + objectId, + params.value, + clearBefore, + deps.signal, + ); + if (isRpcError(fillResult)) return fillResult; return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: input.usedRef, used_selector: input.usedSelector, - value_length: actualValue.length, + value_length: fillResult.value.length, + }); +} + +// --------------------------------------------------------------------------- +// tool.type +// --------------------------------------------------------------------------- + +export async function handleType( + manager: SessionManager, + params: TypeParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + if (!params || typeof params.text !== "string") { + return { code: "invalid_params", message: "type requires a text string" }; + } + if (params.focused !== undefined && typeof params.focused !== "boolean") { + return { code: "invalid_params", message: "type focused must be a boolean" }; + } + const hasExplicitTarget = + (typeof params.ref === "string" && params.ref.length > 0) || + (typeof params.selector === "string" && params.selector.length > 0); + if (params.focused === true && hasExplicitTarget) { + return { + code: "invalid_params", + message: "type: pass a ref/selector or focused=true, not both", + }; + } + if (params.focused !== true && !hasExplicitTarget) { + return { + code: "invalid_params", + message: "type requires a ref/selector or focused=true", + }; + } + const ctxOrErr = lookupSession(manager, params, "type"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" }; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "type"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + let input: InputDispatchContext | RpcError; + if (hasExplicitTarget) { + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type"); + const explicitInput = inputContextForNode(deps.cdp, node); + if (isRpcError(explicitInput)) return explicitInput; + input = explicitInput; + try { + const described = await explicitInput.cdp.send<{ node?: DescribedNode }>( + target.tabId, + "DOM.describeNode", + { backendNodeId: explicitInput.backendNodeId }, + ); + if (!described.node || !isTypeTarget(described.node)) { + return rpcError( + "invalid_params", + "target_not_fillable", + `element ${described.node?.nodeName ?? "?"} not text-editable`, + ); + } + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollElementAndFramesIntoView( + deps.cdp, + target.tabId, + explicitInput.target, + explicitInput.backendNodeId, + explicitInput.frameId, + ); + if (scrollErr) return scrollErr; + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" }; + await explicitInput.cdp.send(target.tabId, "DOM.focus", { + backendNodeId: explicitInput.backendNodeId, + }); + } catch (err) { + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } + } else { + input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type"); + if (isRpcError(input)) return input; + } + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal); + if (dispatchError) return dispatchError; + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: input.usedRef, + used_selector: input.usedSelector, + text_length: params.text.length, + verification: "dispatched", }); } @@ -1090,7 +1139,7 @@ export async function handlePress( }; } } else { - input = await resolveFocusedInputContext(deps.cdp, target.tabId); + input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press"); if (isRpcError(input)) return input; } diff --git a/apps/extension/src/tools/text-input.ts b/apps/extension/src/tools/text-input.ts new file mode 100644 index 0000000..40ed5ee --- /dev/null +++ b/apps/extension/src/tools/text-input.ts @@ -0,0 +1,194 @@ +// Browser-native text entry for live editing sessions. Unlike fill, this +// module does not assume that the focused editable retains a DOM value: a +// Canvas editor may use an input/textarea only as a transient keyboard/IME +// sink and clear it after consuming each event. + +import type { RpcError } from "@/transport/types"; +import type { CdpRunner } from "./shared"; + +function cancelled(signal: AbortSignal | undefined): RpcError | null { + return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null; +} + +interface KeyDescriptor { + key: string; + code: string; + windowsVirtualKeyCode: number; + nativeVirtualKeyCode?: number; + modifiers: number; +} + +function asciiKeyDescriptor(character: string): KeyDescriptor { + if (/^[a-z]$/.test(character)) { + return { + key: character, + code: `Key${character.toUpperCase()}`, + windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0), + modifiers: 0, + }; + } + if (/^[A-Z]$/.test(character)) { + return { + key: character, + code: `Key${character}`, + windowsVirtualKeyCode: character.charCodeAt(0), + modifiers: 8, + }; + } + if (/^[0-9]$/.test(character)) { + return { + key: character, + code: `Digit${character}`, + windowsVirtualKeyCode: character.charCodeAt(0), + modifiers: 0, + }; + } + if (character === " ") { + return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 }; + } + return { + key: character, + code: "", + windowsVirtualKeyCode: character.charCodeAt(0), + modifiers: 0, + }; +} + +/** + * Run one complete browser key transaction and always release the key. The + * commit callback owns the text-producing phase: a `char` event for a + * keyboard character, or `Input.insertText` for committed IME text. + */ +async function dispatchKeyTransaction( + cdp: CdpRunner, + tabId: number, + descriptor: KeyDescriptor, + commit: () => Promise, + signal?: AbortSignal, +): Promise { + let keyDown = false; + try { + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "rawKeyDown", + ...descriptor, + }); + keyDown = true; + const abortedAfterDown = cancelled(signal); + if (!abortedAfterDown) await commit(); + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "keyUp", + ...descriptor, + }); + keyDown = false; + return abortedAfterDown ?? cancelled(signal); + } catch (err) { + if (keyDown) { + try { + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "keyUp", + ...descriptor, + }); + } catch { + // Best-effort cleanup keeps the page from observing a stuck key. + } + } + throw err; + } +} + +async function typeAsciiCharacter( + cdp: CdpRunner, + tabId: number, + character: string, + signal?: AbortSignal, +): Promise { + const descriptor = asciiKeyDescriptor(character); + return dispatchKeyTransaction( + cdp, + tabId, + descriptor, + async () => { + await cdp.send(tabId, "Input.dispatchKeyEvent", { + type: "char", + ...descriptor, + text: character, + unmodifiedText: character, + }); + }, + signal, + ); +} + +async function typeImeSegment( + cdp: CdpRunner, + tabId: number, + text: string, + signal?: AbortSignal, +): Promise { + const abortedBefore = cancelled(signal); + if (abortedBefore) return abortedBefore; + + // A committed IME value is still part of a keyboard transaction. The + // Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an + // IME owns the keystroke. It lets selection-based editors enter text mode + // before insertText commits the Unicode value, without inventing a visible + // ASCII prefix or leaving a composition transaction open. + const processKey: KeyDescriptor = { + key: "Process", + code: "", + windowsVirtualKeyCode: 229, + nativeVirtualKeyCode: 229, + modifiers: 0, + }; + return dispatchKeyTransaction( + cdp, + tabId, + processKey, + () => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined), + signal, + ); +} + +/** + * Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are + * delivered as physical-key-shaped events; non-ASCII runs use CDP's committed + * IME/software-keyboard insertion path. Successful return means the browser + * accepted the input commands, not that a DOM value retained the text. + */ +export async function dispatchTextInput( + cdp: CdpRunner, + tabId: number, + text: string, + signal?: AbortSignal, +): Promise { + const abortedBefore = cancelled(signal); + if (abortedBefore) return abortedBefore; + + const segments: Array<{ ime: boolean; text: string }> = []; + for (const character of text) { + const ime = character.codePointAt(0)! > 0x7f; + const previous = segments.at(-1); + if (previous?.ime === ime) previous.text += character; + else segments.push({ ime, text: character }); + } + + try { + for (const segment of segments) { + if (segment.ime) { + const error = await typeImeSegment(cdp, tabId, segment.text, signal); + if (error) return error; + continue; + } + for (const character of segment.text) { + const error = await typeAsciiCharacter(cdp, tabId, character, signal); + if (error) return error; + } + } + return null; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 0e6b6aa..8c07f7b 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -484,6 +484,26 @@ export interface FillResult { dialogs?: JavaScriptDialogInfo[]; } +export interface TypeParams { + session_id: string; + text: string; + ref?: string; + selector?: string; + /** Explicitly dispatch to the unique focused text-entry target. */ + focused?: boolean; + tab_id?: number; + timeout_ms?: number; +} + +export interface TypeResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + text_length: number; + verification: "dispatched"; + dialogs?: JavaScriptDialogInfo[]; +} + export interface PressParams { session_id: string; key: string; diff --git a/apps/extension/test-fixtures/rendered-surfaces/README.md b/apps/extension/test-fixtures/rendered-surfaces/README.md index dafa382..10a8ff4 100644 --- a/apps/extension/test-fixtures/rendered-surfaces/README.md +++ b/apps/extension/test-fixtures/rendered-surfaces/README.md @@ -97,14 +97,25 @@ Run with a `900x700` Agent Window. This page exercises generic frame-aware input routing. It contains a top-level input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each -frame includes an input, textarea, contenteditable editor, and a controlled -input that resets its value on the next animation frame. +frame includes an input, textarea, contenteditable editor, an input that +replaces its DOM node after accepted input, an input that rejects synthetic +DOM events, a transient keyboard/IME input proxy that consumes and clears its +DOM value, and a controlled input that resets its value on the next animation +frame. - `fill` must route focus, text insertion, DOM events, and readback through the ref's owning CDP target. - `press --ref` must focus and dispatch every key event through that same target. - Untargeted `press` must follow the actual focused target or refuse when focus ownership cannot be determined uniquely. +- The remounting input must retain the value through its replacement node. +- The trusted input must be filled without synthetic `input` / `change` events. +- `type` must deliver ASCII key events and committed non-ASCII IME/software + keyboard insertion to the input proxy. Its DOM value remains empty while + `proxy output` retains the consumed text; the result reports + `verification=dispatched`, not a retained DOM value. +- `type` must reject an omitted target unless `focused=true` is explicit, and + focused mode must reject focus that is not owned by a text-entry target. - The rejecting input must return `input_not_applied`, never a successful requested-value length. diff --git a/apps/extension/test-fixtures/rendered-surfaces/input-frame.html b/apps/extension/test-fixtures/rendered-surfaces/input-frame.html index 528ea9e..a629c2f 100644 --- a/apps/extension/test-fixtures/rendered-surfaces/input-frame.html +++ b/apps/extension/test-fixtures/rendered-surfaces/input-frame.html @@ -14,9 +14,38 @@
+ + + + proxy output: empty