feat(vom): support canvas reading

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -34,6 +34,15 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
     if (!h && method === "Page.getLayoutMetrics") {
       return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
     }
+    if (
+      !h &&
+      method === "Runtime.evaluate" &&
+      String((params as { expression?: string })?.expression ?? "").includes(
+        "bsk-focus-target-probe",
+      )
+    ) {
+      return { result: { value: { hasFocus: true, activeTag: "input" } } };
+    }
     if (!h) throw new Error(`unexpected CDP call ${method}`);
     return h(params);
   };
@@ -45,6 +54,10 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
       params?: object,
     ) => Promise<T>,
     trackSessionTab: vi.fn(),
+    getFrameGraph: vi.fn(async (tabId) => ({
+      rootFrameId: "main",
+      frames: [{ frameId: "main", target: { tabId } }],
+    })),
   };
   const tabsApi = {
     get: vi.fn(
@@ -55,6 +68,16 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
   return { cdp, tabsApi, sent };
 }
 
+function fillRuntimeReadback(value: string) {
+  return (params: unknown) => {
+    const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
+    if (fn.includes("bsk-input-readback")) {
+      return { result: { value: { supported: true, value } } };
+    }
+    return { result: { type: "undefined" } };
+  };
+}
+
 async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
   const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
   const context = await manager.start("aa11");
@@ -733,7 +756,7 @@ describe("handleFill", () => {
       "DOM.scrollIntoViewIfNeeded": () => ({}),
       "DOM.focus": () => ({}),
       "DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
-      "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
+      "Runtime.callFunctionOn": fillRuntimeReadback("hello"),
       "Input.insertText": () => ({}),
     });
     const res = await handleFill(
@@ -766,6 +789,9 @@ describe("handleFill", () => {
       "DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
       "Runtime.callFunctionOn": (p) => {
         const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
+        if (fn.includes("bsk-input-readback")) {
+          return { result: { value: { supported: true, value: "existingx" } } };
+        }
         // No "this.value = ''" clearing on a no-clear path; only the
         // post-input dispatchEvent.
         expect(fn).not.toMatch(/this\.value\s*=\s*''/);
@@ -779,7 +805,7 @@ describe("handleFill", () => {
       { cdp: fake.cdp, tabsApi: fake.tabsApi },
     );
     if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
-    expect(res.value_length).toBe(1);
+    expect(res.value_length).toBe(9);
   });
 
   it("treats contenteditable=true as fillable", async () => {
@@ -797,7 +823,7 @@ describe("handleFill", () => {
       "DOM.scrollIntoViewIfNeeded": () => ({}),
       "DOM.focus": () => ({}),
       "DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
-      "Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
+      "Runtime.callFunctionOn": fillRuntimeReadback("rich"),
       "Input.insertText": () => ({}),
     });
     const res = await handleFill(
@@ -808,6 +834,88 @@ describe("handleFill", () => {
     expect("code" in res).toBe(false);
   });
 
+  it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e1", 77, {
+      tabId: 4,
+      frameId: "child-frame",
+      cdpSessionId: "child-session",
+    });
+    const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
+    fake.cdp.getFrameGraph = vi.fn(async () => ({
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child-frame",
+          parentFrameId: "main",
+          ownerBackendNodeId: 12,
+          target: { tabId: 4, sessionId: "child-session" },
+        },
+      ],
+    }));
+    const childCalls: string[] = [];
+    fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
+      expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
+      childCalls.push(method);
+      if (method === "DOM.describeNode") {
+        return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
+      }
+      if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
+      if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
+      if (method === "Input.insertText") {
+        expect(params).toEqual({ text: "中文输入" });
+        return {};
+      }
+      if (method === "Runtime.callFunctionOn") {
+        const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
+        return fn.includes("bsk-input-readback")
+          ? { result: { value: { supported: true, value: "中文输入" } } }
+          : { result: { type: "undefined" } };
+      }
+      throw new Error(`unexpected child call ${method}`);
+    }) as CdpRunner["sendToTarget"];
+
+    const res = await handleFill(
+      sm,
+      { session_id: "aa11", ref: "@e1", value: "中文输入" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    if ("code" in res) throw new Error(JSON.stringify(res));
+    expect(res.value_length).toBe(4);
+    expect(childCalls).toContain("DOM.focus");
+    expect(childCalls).toContain("Input.insertText");
+    expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
+    expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
+  });
+
+  it("returns input_not_applied when a controlled input resets the value", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e1", 8, { tabId: 4 });
+    const fake = makeFakeCdp({
+      "DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
+      "DOM.scrollIntoViewIfNeeded": () => ({}),
+      "DOM.focus": () => ({}),
+      "DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
+      "Input.insertText": () => ({}),
+      "Runtime.callFunctionOn": fillRuntimeReadback(""),
+    });
+
+    const res = await handleFill(
+      sm,
+      { session_id: "aa11", ref: "@e1", value: "rejected" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(res).toMatchObject({
+      code: "cdp_failed",
+      data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
+    });
+  });
+
   it("stops before Input.insertText when abort fires after clearing", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
@@ -1002,6 +1110,125 @@ describe("handlePress", () => {
     expect(res.key).toBe("Enter");
   });
 
+  it("routes ref-targeted key events through the OOPIF session", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e1", 555, {
+      tabId: 4,
+      frameId: "child-frame",
+      cdpSessionId: "child-session",
+    });
+    const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
+    fake.cdp.getFrameGraph = vi.fn(async () => ({
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child-frame",
+          parentFrameId: "main",
+          ownerBackendNodeId: 12,
+          target: { tabId: 4, sessionId: "child-session" },
+        },
+      ],
+    }));
+    const childCalls: string[] = [];
+    fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
+      childCalls.push(method);
+      return {};
+    }) as CdpRunner["sendToTarget"];
+
+    const res = await handlePress(
+      sm,
+      { session_id: "aa11", key: "Enter", ref: "@e1" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expectPressOk(res);
+    expect(childCalls).toEqual([
+      "DOM.scrollIntoViewIfNeeded",
+      "DOM.focus",
+      "Input.dispatchKeyEvent",
+      "Input.dispatchKeyEvent",
+      "Input.dispatchKeyEvent",
+    ]);
+    expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
+  });
+
+  it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const fake = makeFakeCdp({
+      "Runtime.evaluate": () => ({
+        result: { value: { hasFocus: true, activeTag: "iframe" } },
+      }),
+    });
+    fake.cdp.getFrameGraph = vi.fn(async () => ({
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child-frame",
+          parentFrameId: "main",
+          target: { tabId: 4, sessionId: "child-session" },
+        },
+      ],
+    }));
+    const childCalls: string[] = [];
+    fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
+      childCalls.push(method);
+      if (method === "Runtime.evaluate") {
+        return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
+      }
+      return {};
+    }) as CdpRunner["sendToTarget"];
+
+    const res = await handlePress(
+      sm,
+      { session_id: "aa11", key: "a" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expectPressOk(res);
+    expect(childCalls).toEqual([
+      "Runtime.evaluate",
+      "Input.dispatchKeyEvent",
+      "Input.dispatchKeyEvent",
+      "Input.dispatchKeyEvent",
+    ]);
+  });
+
+  it("refuses an untargeted press when focus ownership is ambiguous", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await sm.start("aa11");
+    const fake = makeFakeCdp({
+      "Runtime.evaluate": () => ({
+        result: { value: { hasFocus: true, activeTag: "input" } },
+      }),
+    });
+    fake.cdp.getFrameGraph = vi.fn(async () => ({
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        { frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
+      ],
+    }));
+    fake.cdp.sendToTarget = vi.fn(async () => ({
+      result: { value: { hasFocus: true, activeTag: "textarea" } },
+    })) as CdpRunner["sendToTarget"];
+
+    const res = await handlePress(
+      sm,
+      { session_id: "aa11", key: "Enter" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(res).toMatchObject({
+      code: "cdp_failed",
+      data: { reason: "focus_target_unresolved", candidate_count: 2 },
+    });
+    expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
+  });
+
   it("stops before focus when abort fires after press scroll", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");

apps/extension/src/tools/interaction.ts

@@ -11,7 +11,7 @@
 //    commands.
 
 import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
-import type { CdpTarget } from "@/browser-driver/frame-graph";
+import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
 import type { SessionContext, SessionManager } from "@/session-manager/manager";
 import type {
   ClickParams,
@@ -217,6 +217,108 @@ async function resolveBackendNode(
   }
 }
 
+interface InputDispatchContext {
+  target: CdpTarget;
+  cdp: CdpRunner;
+  backendNodeId?: number;
+  frameId?: string;
+  usedRef?: string;
+  usedSelector?: string;
+}
+
+function inputContextForNode(
+  cdp: CdpRunner,
+  node: Awaited<ReturnType<typeof resolveBackendNode>>,
+): (InputDispatchContext & { backendNodeId: number }) | RpcError {
+  if (isRpcError(node)) return node;
+  return {
+    target: node.cdpTarget,
+    cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
+    backendNodeId: node.backendNodeId,
+    frameId: node.frameId,
+    usedRef: node.usedRef,
+    usedSelector: node.usedSelector,
+  };
+}
+
+interface FocusProbe {
+  hasFocus: boolean;
+  activeTag: string;
+}
+
+/** Resolve keyboard input to the CDP target that currently owns focus. */
+async function resolveFocusedInputContext(
+  cdp: CdpRunner,
+  tabId: number,
+): Promise<InputDispatchContext | RpcError> {
+  if (!cdp.getFrameGraph) {
+    return rpcError(
+      "cdp_failed",
+      "focus_target_unresolved",
+      "press cannot determine the focused frame target",
+    );
+  }
+
+  let targets: CdpTarget[];
+  try {
+    const graph = await cdp.getFrameGraph(tabId);
+    const unique = new Map<string, CdpTarget>();
+    unique.set(cdpTargetKey({ tabId }), { tabId });
+    for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
+    targets = [...unique.values()];
+  } catch (err) {
+    return rpcError(
+      "cdp_failed",
+      "focus_target_unresolved",
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
+  const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
+  await Promise.all(
+    targets.map(async (candidate) => {
+      try {
+        const reply = await cdpRunnerForTarget(cdp, candidate).send<{
+          result?: { value?: FocusProbe };
+        }>(tabId, "Runtime.evaluate", {
+          expression: `(() => {
+            /* bsk-focus-target-probe */
+            const active = document.activeElement;
+            return {
+              hasFocus: document.hasFocus(),
+              activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
+            };
+          })()`,
+          returnByValue: true,
+        });
+        const probe = reply.result?.value;
+        if (probe?.hasFocus) focused.push({ target: candidate, probe });
+      } catch {
+        // A target may navigate or detach while the graph is being probed.
+      }
+    }),
+  );
+
+  // A top document whose active element is an iframe delegates focus to
+  // that frame. Removing those delegating targets leaves the deepest OOPIF
+  // target while same-process frames continue to use the shared root target.
+  const owners = focused.filter(
+    ({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
+  );
+  if (owners.length !== 1) {
+    return rpcError(
+      "cdp_failed",
+      "focus_target_unresolved",
+      owners.length === 0
+        ? "press found no unique focused frame target"
+        : "press found multiple focused frame targets",
+      { candidate_count: owners.length },
+    );
+  }
+  const target = owners[0].target;
+  return { target, cdp: cdpRunnerForTarget(cdp, target) };
+}
+
 // ---------------------------------------------------------------------------
 // tool.click
 // ---------------------------------------------------------------------------
@@ -618,8 +720,9 @@ export async function handleFill(
   const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
 
   const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
-  if (isRpcError(node)) return node;
+  const input = inputContextForNode(deps.cdp, node);
-  const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
+  if (isRpcError(input)) return input;
+  const nodeCdp = input.cdp;
 
   try {
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -627,7 +730,7 @@ export async function handleFill(
       target.tabId,
       "DOM.describeNode",
       {
-        backendNodeId: node.backendNodeId,
+        backendNodeId: input.backendNodeId,
       },
     );
     if (!described.node || !isFillable(described.node)) {
@@ -640,15 +743,15 @@ export async function handleFill(
     const scrollErr = await scrollElementAndFramesIntoView(
       deps.cdp,
       target.tabId,
-      node.cdpTarget,
+      input.target,
-      node.backendNodeId,
+      input.backendNodeId,
-      node.frameId,
+      input.frameId,
     );
     if (scrollErr) return scrollErr;
     if (throwIfAborted(deps.signal)) {
       return { code: "cancelled", message: "fill aborted" };
     }
-    await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
+    await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -660,7 +763,7 @@ export async function handleFill(
     return { code: "cancelled", message: "fill aborted" };
   }
 
-  const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
+  const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
   if (isRpcError(objectIdOrErr)) return objectIdOrErr;
   const objectId = objectIdOrErr;
   const clearBefore = params.clear_before ?? true;
@@ -691,7 +794,7 @@ export async function handleFill(
     }
     // CDP `Input.insertText` handles IME / multi-byte input out of the
     // box, much more reliably than per-key `dispatchKeyEvent`.
-    await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
+    await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
     if (throwIfAborted(deps.signal)) {
       return { code: "cancelled", message: "fill aborted" };
     }
@@ -704,6 +807,28 @@ export async function handleFill(
       }`,
       returnByValue: true,
     });
+    // Give controlled inputs one bounded browser-render turn to accept or
+    // reject the dispatched value before reporting success.
+    await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
+      objectId,
+      functionDeclaration: `function() {
+        /* bsk-input-sync */
+        return new Promise((resolve) => {
+          let settled = false;
+          const finish = () => {
+            if (settled) return;
+            settled = true;
+            clearTimeout(timer);
+            resolve();
+          };
+          const timer = setTimeout(finish, 100);
+          if (typeof requestAnimationFrame !== 'function') { finish(); return; }
+          requestAnimationFrame(() => requestAnimationFrame(finish));
+        });
+      }`,
+      awaitPromise: true,
+      returnByValue: true,
+    });
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -711,11 +836,56 @@ export async function handleFill(
     };
   }
 
+  if (throwIfAborted(deps.signal)) {
+    return { code: "cancelled", message: "fill aborted" };
+  }
+
+  let actualValue: string;
+  try {
+    const readback = await nodeCdp.send<{
+      result?: { value?: { supported?: boolean; value?: string } };
+    }>(target.tabId, "Runtime.callFunctionOn", {
+      objectId,
+      functionDeclaration: `function() {
+        /* bsk-input-readback */
+        if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
+          return { supported: true, value: this.value };
+        }
+        if (this.isContentEditable) {
+          return { supported: true, value: this.innerText ?? this.textContent ?? '' };
+        }
+        return { supported: false, value: '' };
+      }`,
+      returnByValue: true,
+    });
+    const state = readback.result?.value;
+    actualValue = typeof state?.value === "string" ? state.value : "";
+    const applied =
+      state?.supported === true &&
+      (clearBefore
+        ? actualValue === params.value
+        : params.value.length === 0 || actualValue.includes(params.value));
+    if (!applied) {
+      return rpcError(
+        "cdp_failed",
+        "input_not_applied",
+        "the page did not retain the requested input value",
+        { expected_value_length: params.value.length, actual_value_length: actualValue.length },
+      );
+    }
+  } catch (err) {
+    return rpcError(
+      "cdp_failed",
+      "input_not_applied",
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+
   return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
     tab_id: target.tabId,
-    used_ref: node.usedRef,
+    used_ref: input.usedRef,
-    used_selector: node.usedSelector,
+    used_selector: input.usedSelector,
-    value_length: params.value.length,
+    value_length: actualValue.length,
   });
 }
 
@@ -890,31 +1060,38 @@ export async function handlePress(
     };
   }
 
+  let input: InputDispatchContext | RpcError;
   // Optional focus before key dispatch.
   if (params.ref || params.selector) {
     const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
-    if (isRpcError(node)) return node;
+    const explicitInput = inputContextForNode(deps.cdp, node);
-    const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
+    if (isRpcError(explicitInput)) return explicitInput;
+    input = explicitInput;
     try {
       deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
       const scrollErr = await scrollElementAndFramesIntoView(
         deps.cdp,
         target.tabId,
-        node.cdpTarget,
+        explicitInput.target,
-        node.backendNodeId,
+        explicitInput.backendNodeId,
-        node.frameId,
+        explicitInput.frameId,
       );
       if (scrollErr) return scrollErr;
       if (throwIfAborted(deps.signal)) {
         return { code: "cancelled", message: "press aborted" };
       }
-      await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
+      await explicitInput.cdp.send(target.tabId, "DOM.focus", {
+        backendNodeId: explicitInput.backendNodeId,
+      });
     } catch (err) {
       return {
         code: "cdp_failed",
         message: err instanceof Error ? err.message : String(err),
       };
     }
+  } else {
+    input = await resolveFocusedInputContext(deps.cdp, target.tabId);
+    if (isRpcError(input)) return input;
   }
 
   if (throwIfAborted(deps.signal)) {
@@ -928,7 +1105,7 @@ export async function handlePress(
   try {
     let cancelled = false;
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-    await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+    await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
       type: "rawKeyDown",
       key: descriptor.key,
       code: descriptor.code,
@@ -942,7 +1119,7 @@ export async function handlePress(
       typeof descriptor.text === "string" &&
       descriptor.text.length > 0
     ) {
-      await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+      await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
         type: "char",
         key: descriptor.key,
         code: descriptor.code,
@@ -959,7 +1136,7 @@ export async function handlePress(
         cancelled = true;
       }
     }
-    await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
+    await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
       type: "keyUp",
       key: descriptor.key,
       code: descriptor.code,

apps/extension/src/transport/types.ts

@@ -27,6 +27,8 @@ export type RpcErrorReason =
   | "ref_not_found"
   | "selector_not_found"
   | "target_not_fillable"
+  | "focus_target_unresolved"
+  | "input_not_applied"
   | "target_not_select"
   | "option_not_found"
   | "single_select_value_count"

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -93,6 +93,29 @@ Run with a `900x700` Agent Window.
 - `fixture-below-fold` is absent initially.
 - After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
 
+### `input-frames.html`
+
+This page exercises generic frame-aware input routing. It contains a top-level
+input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
+frame includes an input, textarea, contenteditable editor, and a controlled
+input that resets its value on the next animation frame.
+
+- `fill` must route focus, text insertion, DOM events, and readback through the
+  ref's owning CDP target.
+- `press --ref` must focus and dispatch every key event through that same target.
+- Untargeted `press` must follow the actual focused target or refuse when focus
+  ownership cannot be determined uniquely.
+- The rejecting input must return `input_not_applied`, never a successful
+  requested-value length.
+
+### `canvas-input.html`
+
+This is the PR3 end-to-end fixture. Its cross-origin frame contains a
+visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
+focuses a standard textbox; ordinary `fill` must then write and read back its
+real value through the textbox's owning target. The fixture deliberately does
+not provide a Canvas-specific fill action.
+
 ### `dynamic.html`
 
 Use the normal DOM buttons and re-observe after each action:

apps/extension/test-fixtures/rendered-surfaces/canvas-input.html

@@ -0,0 +1,20 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Canvas to standard input fixture</title>
+    <style>
+      body { padding: 16px; font: 14px system-ui, sans-serif; }
+      iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
+    </style>
+  </head>
+  <body>
+    <h1>Canvas to standard input fixture</h1>
+    <p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
+    <iframe
+      title="cross-origin Canvas editor"
+      src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
+    ></iframe>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/frame.html

@@ -19,6 +19,18 @@
       if (params.get("canvas") === "0") canvas.remove();
       else {
         canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
+        if (params.get("editor") === "1") {
+          const context = canvas.getContext("2d");
+          context.font = "14px system-ui, sans-serif";
+          context.textBaseline = "middle";
+          context.strokeStyle = "#1e8449";
+          context.fillStyle = "#145a32";
+          for (let row = 0; row < 3; row += 1) {
+            const top = row * 30;
+            context.strokeRect(0.5, top + 0.5, 219, 29);
+            context.fillText(`Row ${row + 1}`, 12, top + 15);
+          }
+        }
         if (params.get("interactive") === "1") {
           const status = document.createElement("p");
           status.id = "pointer-status";
@@ -39,6 +51,22 @@
           canvas.addEventListener("click", () => {
             status.textContent += "; click received";
           });
+          if (params.get("editor") === "1") {
+            canvas.addEventListener("click", (event) => {
+              const rect = canvas.getBoundingClientRect();
+              const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
+              const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
+              status.textContent += `; selected row=${row + 1}`;
+              document.querySelector("#canvas-cell-editor")?.remove();
+              if (row !== 2) return;
+              const editor = document.createElement("input");
+              editor.id = "canvas-cell-editor";
+              editor.setAttribute("aria-label", "selected canvas cell editor");
+              editor.placeholder = "Type into selected row 3";
+              canvas.insertAdjacentElement("afterend", editor);
+              editor.focus();
+            });
+          }
         }
       }
       if (params.get("nested") === "1") {

apps/extension/test-fixtures/rendered-surfaces/input-frame.html

@@ -0,0 +1,30 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <style>
+      body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
+      label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
+      input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
+      iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
+    </style>
+  </head>
+  <body>
+    <label>Frame input <input aria-label="frame input" /></label>
+    <label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
+    <div contenteditable="true" aria-label="frame editor"></div>
+    <label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
+    <script>
+      const params = new URLSearchParams(location.search);
+      const rejecting = document.querySelector("#rejecting");
+      rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
+      if (params.get("nested") === "1") {
+        const iframe = document.createElement("iframe");
+        iframe.title = "nested input frame";
+        iframe.src = "input-frame.html";
+        document.body.append(iframe);
+      }
+    </script>
+  </body>
+</html>

apps/extension/test-fixtures/rendered-surfaces/input-frames.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Frame input routing matrix</title>
+    <style>
+      body { padding: 16px; font: 14px system-ui, sans-serif; }
+      .frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
+      iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
+      input { padding: 6px; }
+    </style>
+  </head>
+  <body>
+    <h1>Frame input routing matrix</h1>
+    <label>Top input <input aria-label="top input" /></label>
+    <div class="frames">
+      <iframe title="same-origin inputs" src="input-frame.html"></iframe>
+      <iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
+      <iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
+    </div>
+  </body>
+</html>

crates/bsk-cli/src/cli/render_error.rs

@@ -44,6 +44,8 @@ pub mod reason {
     pub const REF_NOT_FOUND: &str = "ref_not_found";
     pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
     pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
+    pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
+    pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
     pub const TARGET_NOT_SELECT: &str = "target_not_select";
     pub const OPTION_NOT_FOUND: &str = "option_not_found";
     pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
@@ -234,6 +236,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
             ),
             exit_code: base.exit_code,
         },
+        (ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
+            summary: "focused frame target could not be determined",
+            hint: Some(
+                "pass an explicit --ref or --selector so the key event can be routed safely",
+            ),
+            exit_code: base.exit_code,
+        },
+        (ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
+            summary: "the page did not retain the requested input",
+            hint: Some(
+                "refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
+            ),
+            exit_code: base.exit_code,
+        },
         (ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
             summary: "snapshot ref does not support this operation",
             hint: Some(
@@ -532,4 +548,20 @@ mod tests {
         );
         assert!(info.hint.unwrap().contains("exactly one"));
     }
+
+    #[test]
+    fn focus_target_unresolved_overrides_cdp_copy() {
+        let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
+        let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
+        assert_eq!(info.summary, "focused frame target could not be determined");
+        assert!(info.hint.unwrap().contains("--ref"));
+    }
+
+    #[test]
+    fn input_not_applied_overrides_cdp_copy() {
+        let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
+        let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
+        assert_eq!(info.summary, "the page did not retain the requested input");
+        assert!(info.hint.unwrap().contains("rejecting"));
+    }
 }