mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a1904f8141 | ||
|
|
e069abaf2e |
No files matched your search
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fillEditable } from "../editable-input";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
interface FakeOptions {
|
||||
initialValue?: string;
|
||||
finalValue: string;
|
||||
remount?: boolean;
|
||||
}
|
||||
|
||||
function editableState(value: string, connected = true, focused = true) {
|
||||
return {
|
||||
result: {
|
||||
value: { supported: true, connected, focused, value, kind: "input" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeEditableCdp(options: FakeOptions) {
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
let inserted = false;
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Input.insertText") {
|
||||
inserted = true;
|
||||
return {} as T;
|
||||
}
|
||||
if (method === "Input.dispatchKeyEvent") {
|
||||
if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
|
||||
return {} as T;
|
||||
}
|
||||
if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
|
||||
|
||||
const call = params as { objectId?: string; functionDeclaration?: string };
|
||||
const fn = call.functionDeclaration ?? "";
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } } as T;
|
||||
}
|
||||
if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
|
||||
if (fn.includes("bsk-editable-live")) {
|
||||
return {
|
||||
result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
|
||||
} as T;
|
||||
}
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
if (call.objectId === "replacement") return editableState(options.finalValue) as T;
|
||||
if (options.remount && inserted) {
|
||||
return editableState(options.finalValue, false, false) as T;
|
||||
}
|
||||
return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
|
||||
}
|
||||
throw new Error("unexpected Runtime.callFunctionOn function");
|
||||
});
|
||||
return { cdp: { send } as CdpRunner, sent };
|
||||
}
|
||||
|
||||
describe("fillEditable", () => {
|
||||
it("replaces text through the browser input path without synthetic events or value setters", async () => {
|
||||
const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "新值", replacedNode: false });
|
||||
expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
|
||||
const declarations = fake.sent
|
||||
.map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
|
||||
.join("\n");
|
||||
expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||
});
|
||||
|
||||
it("reacquires the focused editable after a framework remount", async () => {
|
||||
const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "persisted", replacedNode: true });
|
||||
});
|
||||
|
||||
it("reports a page rollback after the bounded settlement", async () => {
|
||||
const fake = makeEditableCdp({ finalValue: "" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: {
|
||||
reason: "input_not_applied",
|
||||
expected_value_length: 8,
|
||||
actual_value_length: 0,
|
||||
phase: "readback",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("clears to an empty value with a browser Backspace edit", async () => {
|
||||
const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "" });
|
||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
handleType,
|
||||
modifiersBitfield,
|
||||
parseKeySpec,
|
||||
resolveKeyDescriptor,
|
||||
@@ -34,6 +35,15 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (
|
||||
!h &&
|
||||
method === "Runtime.evaluate" &&
|
||||
String((params as { expression?: string })?.expression ?? "").includes(
|
||||
"bsk-focus-target-probe",
|
||||
)
|
||||
) {
|
||||
return { result: { value: { hasFocus: true, activeTag: "input" } } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -45,6 +55,10 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
getFrameGraph: vi.fn(async (tabId) => ({
|
||||
rootFrameId: "main",
|
||||
frames: [{ frameId: "main", target: { tabId } }],
|
||||
})),
|
||||
};
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
@@ -55,6 +69,29 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
return { cdp, tabsApi, sent };
|
||||
}
|
||||
|
||||
function fillRuntimeReadback(value: string) {
|
||||
return (params: unknown) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value,
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
};
|
||||
}
|
||||
|
||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const context = await manager.start("aa11");
|
||||
@@ -733,7 +770,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("hello"),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -747,13 +784,13 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
// clear_before defaults to true, so the edit transaction prepares a
|
||||
// replacement selection and performs a settled live-value readback.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
it("clear_before=false skips replacement selection", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -766,9 +803,20 @@ describe("handleFill", () => {
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "existingx",
|
||||
kind: "textarea",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||
return { result: { type: "undefined" } };
|
||||
},
|
||||
"Input.insertText": () => ({}),
|
||||
@@ -779,7 +827,7 @@ describe("handleFill", () => {
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.value_length).toBe(1);
|
||||
expect(res.value_length).toBe(9);
|
||||
});
|
||||
|
||||
it("treats contenteditable=true as fillable", async () => {
|
||||
@@ -797,7 +845,7 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback("rich"),
|
||||
"Input.insertText": () => ({}),
|
||||
});
|
||||
const res = await handleFill(
|
||||
@@ -808,7 +856,103 @@ describe("handleFill", () => {
|
||||
expect("code" in res).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
||||
it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
childCalls.push(method);
|
||||
if (method === "DOM.describeNode") {
|
||||
return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
|
||||
}
|
||||
if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
|
||||
if (method === "Input.insertText") {
|
||||
expect(params).toEqual({ text: "中文输入" });
|
||||
return {};
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "中文输入",
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
}
|
||||
throw new Error(`unexpected child call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "中文输入" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(JSON.stringify(res));
|
||||
expect(res.value_length).toBe(4);
|
||||
expect(childCalls).toContain("DOM.focus");
|
||||
expect(childCalls).toContain("Input.insertText");
|
||||
expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
|
||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns input_not_applied when a controlled input resets the value", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 8, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
|
||||
"Input.insertText": () => ({}),
|
||||
"Runtime.callFunctionOn": fillRuntimeReadback(""),
|
||||
});
|
||||
|
||||
const res = await handleFill(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", value: "rejected" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it("stops before Input.insertText when abort fires after preparing replacement", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 555, { tabId: 4 });
|
||||
@@ -820,8 +964,25 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => {
|
||||
abort.abort();
|
||||
"Runtime.callFunctionOn": (params) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "",
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
abort.abort();
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
},
|
||||
"Input.insertText": () => ({}),
|
||||
@@ -864,6 +1025,204 @@ describe("handleFill", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleType", () => {
|
||||
it("requires an explicit target mode instead of silently using focus", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const missing = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(missing).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: expect.stringContaining("focused=true"),
|
||||
});
|
||||
|
||||
const mixed = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", focused: true, text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(mixed).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: expect.stringContaining("not both"),
|
||||
});
|
||||
expect(fake.sent).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.describeNode") {
|
||||
return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "测试" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
used_ref: "e1",
|
||||
text_length: 2,
|
||||
verification: "dispatched",
|
||||
});
|
||||
expect(childCalls.map((call) => call.method)).toEqual([
|
||||
"DOM.describeNode",
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.focus",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.insertText",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
|
||||
});
|
||||
|
||||
it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
|
||||
};
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", focused: true, text: "A" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
|
||||
expect(childCalls).toEqual([
|
||||
"Runtime.evaluate",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses focused mode when focus is not owned by a text-entry target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", focused: true, text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "focus_target_unresolved", candidate_count: 0 },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects explicit non-text input controls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.describeNode": () => ({
|
||||
node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a Surface ref instead of treating it as a text target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseKeySpec", () => {
|
||||
it("splits compound expressions into modifiers + base key", () => {
|
||||
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
||||
@@ -1002,6 +1361,125 @@ describe("handlePress", () => {
|
||||
expect(res.key).toBe("Enter");
|
||||
});
|
||||
|
||||
it("routes ref-targeted key events through the OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 555, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter", ref: "@e1" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.focus",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "iframe" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "a" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expectPressOk(res);
|
||||
expect(childCalls).toEqual([
|
||||
"Runtime.evaluate",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
});
|
||||
|
||||
it("refuses an untargeted press when focus ownership is ambiguous", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "input" } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
|
||||
],
|
||||
}));
|
||||
fake.cdp.sendToTarget = vi.fn(async () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "textarea" } },
|
||||
})) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handlePress(
|
||||
sm,
|
||||
{ session_id: "aa11", key: "Enter" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "focus_target_unresolved", candidate_count: 2 },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before focus when abort fires after press scroll", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { dispatchTextInput } from "../text-input";
|
||||
|
||||
function makeCdp(onSend?: (method: string, params?: object) => void) {
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||
sent.push({ method, params });
|
||||
onSend?.(method, params);
|
||||
return {} as T;
|
||||
});
|
||||
return { cdp: { send } as CdpRunner, sent };
|
||||
}
|
||||
|
||||
describe("dispatchTextInput", () => {
|
||||
it("types ASCII through complete keydown, char, and keyup sequences", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "Ab");
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(
|
||||
fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
|
||||
).toEqual([
|
||||
"Input.dispatchKeyEvent:rawKeyDown",
|
||||
"Input.dispatchKeyEvent:char",
|
||||
"Input.dispatchKeyEvent:keyUp",
|
||||
"Input.dispatchKeyEvent:rawKeyDown",
|
||||
"Input.dispatchKeyEvent:char",
|
||||
"Input.dispatchKeyEvent:keyUp",
|
||||
]);
|
||||
expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
|
||||
expect(fake.sent[1].params).toMatchObject({ text: "A" });
|
||||
});
|
||||
|
||||
it("types non-ASCII through the committed IME insertion path", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试");
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(fake.sent).toEqual([
|
||||
{
|
||||
method: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "rawKeyDown",
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
},
|
||||
},
|
||||
{ method: "Input.insertText", params: { text: "测试" } },
|
||||
{
|
||||
method: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves text order across keyboard and IME segments", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
|
||||
|
||||
expect(result).toBeNull();
|
||||
const meaningful = fake.sent.flatMap((call) => {
|
||||
if (call.method === "Input.insertText") {
|
||||
return [`ime:${(call.params as { text: string }).text}`];
|
||||
}
|
||||
if (
|
||||
call.method === "Input.dispatchKeyEvent" &&
|
||||
(call.params as { type?: string }).type === "char"
|
||||
) {
|
||||
return [`key:${(call.params as { text: string }).text}`];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
|
||||
});
|
||||
|
||||
it("reports cancellation observed after an accepted IME insertion", async () => {
|
||||
const abort = new AbortController();
|
||||
const fake = makeCdp((method) => {
|
||||
if (method === "Input.insertText") abort.abort();
|
||||
});
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.insertText",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||
});
|
||||
|
||||
it("releases the IME process key without committing when cancellation follows keydown", async () => {
|
||||
const abort = new AbortController();
|
||||
const fake = makeCdp((method, params) => {
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type?: string }).type === "rawKeyDown"
|
||||
) {
|
||||
abort.abort();
|
||||
}
|
||||
});
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
|
||||
expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||
});
|
||||
});
|
||||
@@ -28,6 +28,7 @@ import type {
|
||||
ScreenshotParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
TypeParams,
|
||||
WaitForNavigationParams,
|
||||
} from "@/transport/types";
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
@@ -35,7 +36,14 @@ import { handleConsole } from "./console";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleClick,
|
||||
handleFill,
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
handleType,
|
||||
} from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
handleNavigateBack,
|
||||
@@ -491,6 +499,17 @@ export class ToolDispatcher {
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.type":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TypeParams,
|
||||
() =>
|
||||
handleType(
|
||||
this.sessions,
|
||||
req.params as TypeParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.press":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as PressParams,
|
||||
@@ -717,6 +736,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.fill":
|
||||
case "tool.type":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.evaluate":
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
// Browser-native editing transaction for input, textarea, and contenteditable
|
||||
// targets. The CDP target/session is selected by interaction.ts; this module
|
||||
// owns the shorter-lived editable identity inside that target.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
interface EditableState {
|
||||
supported: boolean;
|
||||
connected: boolean;
|
||||
focused: boolean;
|
||||
value: string;
|
||||
kind: "input" | "textarea" | "contenteditable" | "unsupported";
|
||||
}
|
||||
|
||||
export interface EditableFillResult {
|
||||
value: string;
|
||||
replacedNode: boolean;
|
||||
}
|
||||
|
||||
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
|
||||
}
|
||||
|
||||
async function inspectEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
objectId: string,
|
||||
): Promise<EditableState> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { value?: EditableState };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-state */
|
||||
const doc = this.ownerDocument;
|
||||
const focused = !!doc && doc.activeElement === this;
|
||||
if (this instanceof HTMLInputElement) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.value, kind: 'input' };
|
||||
}
|
||||
if (this instanceof HTMLTextAreaElement) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.value, kind: 'textarea' };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
|
||||
}
|
||||
return { supported: false, connected: this.isConnected === true, focused,
|
||||
value: '', kind: 'unsupported' };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const state = reply.result?.value;
|
||||
if (!state || typeof state.value !== "string") {
|
||||
throw new Error("editable state probe returned no value");
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the active editable from the anchor's own document. Using the
|
||||
* anchor's execution context also works for same-process nested frames, where
|
||||
* a target-level Runtime.evaluate would otherwise see the top document.
|
||||
*/
|
||||
async function focusedEditableFromAnchor(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<string | null> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { objectId?: string; subtype?: string };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId: anchorObjectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-live */
|
||||
const active = this.ownerDocument?.activeElement;
|
||||
if (!active) return null;
|
||||
const editable = active instanceof HTMLInputElement ||
|
||||
active instanceof HTMLTextAreaElement || active.isContentEditable;
|
||||
return editable ? active : null;
|
||||
}`,
|
||||
returnByValue: false,
|
||||
});
|
||||
if (reply.result?.subtype === "null") return null;
|
||||
return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
|
||||
}
|
||||
|
||||
async function resolveLiveEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
|
||||
const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
|
||||
if (anchor.supported && anchor.connected && anchor.focused) {
|
||||
return { objectId: anchorObjectId, state: anchor, replaced: false };
|
||||
}
|
||||
|
||||
const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
|
||||
if (!focusedObjectId) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the target editable lost focus or detached before input",
|
||||
{ phase: "resolve_live_editable" },
|
||||
);
|
||||
}
|
||||
const focused = await inspectEditable(cdp, tabId, focusedObjectId);
|
||||
if (!focused.supported || !focused.connected || !focused.focused) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the replacement editable was not live and focused",
|
||||
{ phase: "resolve_live_editable" },
|
||||
);
|
||||
}
|
||||
return { objectId: focusedObjectId, state: focused, replaced: true };
|
||||
}
|
||||
|
||||
async function selectEditableContents(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
objectId: string,
|
||||
): Promise<void> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { value?: { selected?: boolean } };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-selection */
|
||||
if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
|
||||
return { selected: false };
|
||||
}
|
||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||
try {
|
||||
this.setSelectionRange(0, this.value.length);
|
||||
} catch {
|
||||
if (typeof this.select !== 'function') return { selected: false };
|
||||
this.select();
|
||||
}
|
||||
return { selected: true };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
const selection = this.ownerDocument.getSelection();
|
||||
if (!selection) return { selected: false };
|
||||
const range = this.ownerDocument.createRange();
|
||||
range.selectNodeContents(this);
|
||||
selection.removeAllRanges();
|
||||
selection.addRange(range);
|
||||
return { selected: true };
|
||||
}
|
||||
return { selected: false };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (reply.result?.value?.selected !== true) {
|
||||
throw new Error("could not select the live editable contents");
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForEditableSettlement(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<void> {
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId: anchorObjectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-sync */
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, 100);
|
||||
const view = this.ownerDocument?.defaultView;
|
||||
if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
|
||||
view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
|
||||
});
|
||||
}`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
key: "Backspace",
|
||||
code: "Backspace",
|
||||
windowsVirtualKeyCode: 8,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: "Backspace",
|
||||
code: "Backspace",
|
||||
windowsVirtualKeyCode: 8,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace or append text through Chromium's editing pipeline. No value setter
|
||||
* or synthetic DOM events are used: controlled components observe the same
|
||||
* beforeinput/input path as a real browser edit. The live editable is resolved
|
||||
* again after settlement so a framework may remount it without producing a
|
||||
* false failure or a stale-object success.
|
||||
*/
|
||||
export async function fillEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
value: string,
|
||||
clearBefore: boolean,
|
||||
signal?: AbortSignal,
|
||||
): Promise<EditableFillResult | RpcError> {
|
||||
try {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
|
||||
if ("code" in initial) return initial;
|
||||
let replacedNode = initial.replaced;
|
||||
|
||||
if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
|
||||
const abortedAfterSelection = cancelled(signal);
|
||||
if (abortedAfterSelection) return abortedAfterSelection;
|
||||
|
||||
if (value.length > 0) {
|
||||
await cdp.send(tabId, "Input.insertText", { text: value });
|
||||
} else if (clearBefore) {
|
||||
await deleteSelection(cdp, tabId);
|
||||
}
|
||||
|
||||
const abortedAfterInput = cancelled(signal);
|
||||
if (abortedAfterInput) return abortedAfterInput;
|
||||
await waitForEditableSettlement(cdp, tabId, initial.objectId);
|
||||
|
||||
const abortedAfterSettlement = cancelled(signal);
|
||||
if (abortedAfterSettlement) return abortedAfterSettlement;
|
||||
|
||||
let finalObjectId = initial.objectId;
|
||||
let finalState = await inspectEditable(cdp, tabId, finalObjectId);
|
||||
if (!finalState.connected || !finalState.focused) {
|
||||
const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
|
||||
if ("code" in live) return live;
|
||||
finalObjectId = live.objectId;
|
||||
finalState = live.state;
|
||||
replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
|
||||
}
|
||||
|
||||
const applied = clearBefore
|
||||
? finalState.value === value
|
||||
: value.length === 0 || finalState.value.includes(value);
|
||||
if (!finalState.supported || !applied) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the page did not retain the requested input value",
|
||||
{
|
||||
expected_value_length: value.length,
|
||||
actual_value_length: finalState.value.length,
|
||||
phase: "readback",
|
||||
editable_replaced: replacedNode,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { value: finalState.value, replacedNode };
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
{ phase: "edit_transaction" },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
|
||||
// `tool.select`.
|
||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
|
||||
// `tool.press`, and `tool.select`.
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
@@ -11,7 +11,7 @@
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -27,8 +27,11 @@ import type {
|
||||
RpcError,
|
||||
SelectParams,
|
||||
SelectResult,
|
||||
TypeParams,
|
||||
TypeResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { fillEditable } from "./editable-input";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
@@ -45,6 +48,7 @@ import {
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||
import { dispatchTextInput } from "./text-input";
|
||||
|
||||
export interface InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
@@ -217,6 +221,123 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
interface InputDispatchContext {
|
||||
target: CdpTarget;
|
||||
cdp: CdpRunner;
|
||||
backendNodeId?: number;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
function inputContextForNode(
|
||||
cdp: CdpRunner,
|
||||
node: Awaited<ReturnType<typeof resolveBackendNode>>,
|
||||
): (InputDispatchContext & { backendNodeId: number }) | RpcError {
|
||||
if (isRpcError(node)) return node;
|
||||
return {
|
||||
target: node.cdpTarget,
|
||||
cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
|
||||
backendNodeId: node.backendNodeId,
|
||||
frameId: node.frameId,
|
||||
usedRef: node.usedRef,
|
||||
usedSelector: node.usedSelector,
|
||||
};
|
||||
}
|
||||
|
||||
interface FocusProbe {
|
||||
hasFocus: boolean;
|
||||
activeTag: string;
|
||||
textEntry: boolean;
|
||||
}
|
||||
|
||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||
async function resolveFocusedInputContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
mode: "press" | "type",
|
||||
): Promise<InputDispatchContext | RpcError> {
|
||||
if (!cdp.getFrameGraph) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
`${mode} cannot determine the focused frame target`,
|
||||
);
|
||||
}
|
||||
|
||||
let targets: CdpTarget[];
|
||||
try {
|
||||
const graph = await cdp.getFrameGraph(tabId);
|
||||
const unique = new Map<string, CdpTarget>();
|
||||
unique.set(cdpTargetKey({ tabId }), { tabId });
|
||||
for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
|
||||
targets = [...unique.values()];
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
|
||||
const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
|
||||
await Promise.all(
|
||||
targets.map(async (candidate) => {
|
||||
try {
|
||||
const reply = await cdpRunnerForTarget(cdp, candidate).send<{
|
||||
result?: { value?: FocusProbe };
|
||||
}>(tabId, "Runtime.evaluate", {
|
||||
expression: `(() => {
|
||||
/* bsk-focus-target-probe */
|
||||
const active = document.activeElement;
|
||||
const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
|
||||
const nonTextInputTypes = new Set([
|
||||
'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
|
||||
'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
|
||||
'time', 'week'
|
||||
]);
|
||||
const role = active instanceof Element ? active.getAttribute('role') : null;
|
||||
return {
|
||||
hasFocus: document.hasFocus(),
|
||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
|
||||
textEntry:
|
||||
(active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
|
||||
active instanceof HTMLTextAreaElement ||
|
||||
(active instanceof HTMLElement && active.isContentEditable) ||
|
||||
role === 'textbox' || role === 'searchbox'
|
||||
};
|
||||
})()`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const probe = reply.result?.value;
|
||||
if (probe?.hasFocus) focused.push({ target: candidate, probe });
|
||||
} catch {
|
||||
// A target may navigate or detach while the graph is being probed.
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// A top document whose active element is an iframe delegates focus to
|
||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||
// target while same-process frames continue to use the shared root target.
|
||||
const owners = focused.filter(({ probe }) => {
|
||||
if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
|
||||
return mode !== "type" || probe.textEntry;
|
||||
});
|
||||
if (owners.length !== 1) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
owners.length === 0
|
||||
? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
|
||||
: `${mode} found multiple focused frame targets`,
|
||||
{ candidate_count: owners.length },
|
||||
);
|
||||
}
|
||||
const target = owners[0].target;
|
||||
return { target, cdp: cdpRunnerForTarget(cdp, target) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -580,6 +701,47 @@ function isFillable(node: DescribedNode): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** `type` also supports custom text-entry widgets that expose an ARIA role. */
|
||||
function isTypeTarget(node: DescribedNode): boolean {
|
||||
const tag = (node.nodeName ?? "").toUpperCase();
|
||||
const attrs = node.attributes ?? [];
|
||||
const attributes = new Map<string, string>();
|
||||
for (let i = 0; i + 1 < attrs.length; i += 2) {
|
||||
attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
|
||||
}
|
||||
if (tag === "TEXTAREA") return true;
|
||||
if (tag === "INPUT") {
|
||||
const type = attributes.get("type") ?? "text";
|
||||
return !new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"color",
|
||||
"date",
|
||||
"datetime-local",
|
||||
"file",
|
||||
"hidden",
|
||||
"image",
|
||||
"month",
|
||||
"radio",
|
||||
"range",
|
||||
"reset",
|
||||
"submit",
|
||||
"time",
|
||||
"week",
|
||||
]).has(type);
|
||||
}
|
||||
const contentEditable = attributes.get("contenteditable");
|
||||
if (
|
||||
contentEditable === "" ||
|
||||
contentEditable === "true" ||
|
||||
contentEditable === "plaintext-only"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const role = attributes.get("role");
|
||||
return role === "textbox" || role === "searchbox";
|
||||
}
|
||||
|
||||
type SelectMutationResult =
|
||||
| {
|
||||
ok: true;
|
||||
@@ -618,8 +780,9 @@ export async function handleFill(
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
const input = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(input)) return input;
|
||||
const nodeCdp = input.cdp;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
@@ -627,7 +790,7 @@ export async function handleFill(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
backendNodeId: node.backendNodeId,
|
||||
backendNodeId: input.backendNodeId,
|
||||
},
|
||||
);
|
||||
if (!described.node || !isFillable(described.node)) {
|
||||
@@ -640,15 +803,15 @@ export async function handleFill(
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
input.target,
|
||||
input.backendNodeId,
|
||||
input.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -660,62 +823,118 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
const fillResult = await fillEditable(
|
||||
nodeCdp,
|
||||
target.tabId,
|
||||
objectId,
|
||||
params.value,
|
||||
clearBefore,
|
||||
deps.signal,
|
||||
);
|
||||
if (isRpcError(fillResult)) return fillResult;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
value_length: params.value.length,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
value_length: fillResult.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function handleType(
|
||||
manager: SessionManager,
|
||||
params: TypeParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<TypeResult | RpcError> {
|
||||
if (!params || typeof params.text !== "string") {
|
||||
return { code: "invalid_params", message: "type requires a text string" };
|
||||
}
|
||||
if (params.focused !== undefined && typeof params.focused !== "boolean") {
|
||||
return { code: "invalid_params", message: "type focused must be a boolean" };
|
||||
}
|
||||
const hasExplicitTarget =
|
||||
(typeof params.ref === "string" && params.ref.length > 0) ||
|
||||
(typeof params.selector === "string" && params.selector.length > 0);
|
||||
if (params.focused === true && hasExplicitTarget) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "type: pass a ref/selector or focused=true, not both",
|
||||
};
|
||||
}
|
||||
if (params.focused !== true && !hasExplicitTarget) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "type requires a ref/selector or focused=true",
|
||||
};
|
||||
}
|
||||
const ctxOrErr = lookupSession(manager, params, "type");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "type");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
let input: InputDispatchContext | RpcError;
|
||||
if (hasExplicitTarget) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
|
||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(explicitInput)) return explicitInput;
|
||||
input = explicitInput;
|
||||
try {
|
||||
const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: explicitInput.backendNodeId },
|
||||
);
|
||||
if (!described.node || !isTypeTarget(described.node)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"target_not_fillable",
|
||||
`element ${described.node?.nodeName ?? "?"} not text-editable`,
|
||||
);
|
||||
}
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
explicitInput.target,
|
||||
explicitInput.backendNodeId,
|
||||
explicitInput.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||
backendNodeId: explicitInput.backendNodeId,
|
||||
});
|
||||
} catch (err) {
|
||||
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
|
||||
if (dispatchError) return dispatchError;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
text_length: params.text.length,
|
||||
verification: "dispatched",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -890,31 +1109,38 @@ export async function handlePress(
|
||||
};
|
||||
}
|
||||
|
||||
let input: InputDispatchContext | RpcError;
|
||||
// Optional focus before key dispatch.
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(explicitInput)) return explicitInput;
|
||||
input = explicitInput;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
explicitInput.target,
|
||||
explicitInput.backendNodeId,
|
||||
explicitInput.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||
backendNodeId: explicitInput.backendNodeId,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
@@ -928,7 +1154,7 @@ export async function handlePress(
|
||||
try {
|
||||
let cancelled = false;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -942,7 +1168,7 @@ export async function handlePress(
|
||||
typeof descriptor.text === "string" &&
|
||||
descriptor.text.length > 0
|
||||
) {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "char",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
@@ -959,7 +1185,7 @@ export async function handlePress(
|
||||
cancelled = true;
|
||||
}
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: descriptor.key,
|
||||
code: descriptor.code,
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// Browser-native text entry for live editing sessions. Unlike fill, this
|
||||
// module does not assume that the focused editable retains a DOM value: a
|
||||
// Canvas editor may use an input/textarea only as a transient keyboard/IME
|
||||
// sink and clear it after consuming each event.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
|
||||
}
|
||||
|
||||
interface KeyDescriptor {
|
||||
key: string;
|
||||
code: string;
|
||||
windowsVirtualKeyCode: number;
|
||||
nativeVirtualKeyCode?: number;
|
||||
modifiers: number;
|
||||
}
|
||||
|
||||
function asciiKeyDescriptor(character: string): KeyDescriptor {
|
||||
if (/^[a-z]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Key${character.toUpperCase()}`,
|
||||
windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
if (/^[A-Z]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Key${character}`,
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 8,
|
||||
};
|
||||
}
|
||||
if (/^[0-9]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Digit${character}`,
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
if (character === " ") {
|
||||
return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
|
||||
}
|
||||
return {
|
||||
key: character,
|
||||
code: "",
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one complete browser key transaction and always release the key. The
|
||||
* commit callback owns the text-producing phase: a `char` event for a
|
||||
* keyboard character, or `Input.insertText` for committed IME text.
|
||||
*/
|
||||
async function dispatchKeyTransaction(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
descriptor: KeyDescriptor,
|
||||
commit: () => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
let keyDown = false;
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
...descriptor,
|
||||
});
|
||||
keyDown = true;
|
||||
const abortedAfterDown = cancelled(signal);
|
||||
if (!abortedAfterDown) await commit();
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
...descriptor,
|
||||
});
|
||||
keyDown = false;
|
||||
return abortedAfterDown ?? cancelled(signal);
|
||||
} catch (err) {
|
||||
if (keyDown) {
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
...descriptor,
|
||||
});
|
||||
} catch {
|
||||
// Best-effort cleanup keeps the page from observing a stuck key.
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function typeAsciiCharacter(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
character: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const descriptor = asciiKeyDescriptor(character);
|
||||
return dispatchKeyTransaction(
|
||||
cdp,
|
||||
tabId,
|
||||
descriptor,
|
||||
async () => {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "char",
|
||||
...descriptor,
|
||||
text: character,
|
||||
unmodifiedText: character,
|
||||
});
|
||||
},
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
async function typeImeSegment(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
text: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
// A committed IME value is still part of a keyboard transaction. The
|
||||
// Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
|
||||
// IME owns the keystroke. It lets selection-based editors enter text mode
|
||||
// before insertText commits the Unicode value, without inventing a visible
|
||||
// ASCII prefix or leaving a composition transaction open.
|
||||
const processKey: KeyDescriptor = {
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
};
|
||||
return dispatchKeyTransaction(
|
||||
cdp,
|
||||
tabId,
|
||||
processKey,
|
||||
() => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
|
||||
* delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
|
||||
* IME/software-keyboard insertion path. Successful return means the browser
|
||||
* accepted the input commands, not that a DOM value retained the text.
|
||||
*/
|
||||
export async function dispatchTextInput(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
text: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
const segments: Array<{ ime: boolean; text: string }> = [];
|
||||
for (const character of text) {
|
||||
const ime = character.codePointAt(0)! > 0x7f;
|
||||
const previous = segments.at(-1);
|
||||
if (previous?.ime === ime) previous.text += character;
|
||||
else segments.push({ ime, text: character });
|
||||
}
|
||||
|
||||
try {
|
||||
for (const segment of segments) {
|
||||
if (segment.ime) {
|
||||
const error = await typeImeSegment(cdp, tabId, segment.text, signal);
|
||||
if (error) return error;
|
||||
continue;
|
||||
}
|
||||
for (const character of segment.text) {
|
||||
const error = await typeAsciiCharacter(cdp, tabId, character, signal);
|
||||
if (error) return error;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,8 @@ export type RpcErrorReason =
|
||||
| "ref_not_found"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "focus_target_unresolved"
|
||||
| "input_not_applied"
|
||||
| "target_not_select"
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
@@ -482,6 +484,26 @@ export interface FillResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface TypeParams {
|
||||
session_id: string;
|
||||
text: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
/** Explicitly dispatch to the unique focused text-entry target. */
|
||||
focused?: boolean;
|
||||
tab_id?: number;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface TypeResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
text_length: number;
|
||||
verification: "dispatched";
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface PressParams {
|
||||
session_id: string;
|
||||
key: string;
|
||||
|
||||
@@ -93,6 +93,40 @@ Run with a `900x700` Agent Window.
|
||||
- `fixture-below-fold` is absent initially.
|
||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||
|
||||
### `input-frames.html`
|
||||
|
||||
This page exercises generic frame-aware input routing. It contains a top-level
|
||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||
frame includes an input, textarea, contenteditable editor, an input that
|
||||
replaces its DOM node after accepted input, an input that rejects synthetic
|
||||
DOM events, a transient keyboard/IME input proxy that consumes and clears its
|
||||
DOM value, and a controlled input that resets its value on the next animation
|
||||
frame.
|
||||
|
||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||
ref's owning CDP target.
|
||||
- `press --ref` must focus and dispatch every key event through that same target.
|
||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||
ownership cannot be determined uniquely.
|
||||
- The remounting input must retain the value through its replacement node.
|
||||
- The trusted input must be filled without synthetic `input` / `change` events.
|
||||
- `type` must deliver ASCII key events and committed non-ASCII IME/software
|
||||
keyboard insertion to the input proxy. Its DOM value remains empty while
|
||||
`proxy output` retains the consumed text; the result reports
|
||||
`verification=dispatched`, not a retained DOM value.
|
||||
- `type` must reject an omitted target unless `focused=true` is explicit, and
|
||||
focused mode must reject focus that is not owned by a text-entry target.
|
||||
- The rejecting input must return `input_not_applied`, never a successful
|
||||
requested-value length.
|
||||
|
||||
### `canvas-input.html`
|
||||
|
||||
This is the PR3 end-to-end fixture. Its cross-origin frame contains a
|
||||
visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
|
||||
focuses a standard textbox; ordinary `fill` must then write and read back its
|
||||
real value through the textbox's owning target. The fixture deliberately does
|
||||
not provide a Canvas-specific fill action.
|
||||
|
||||
### `dynamic.html`
|
||||
|
||||
Use the normal DOM buttons and re-observe after each action:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Canvas to standard input fixture</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Canvas to standard input fixture</h1>
|
||||
<p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
|
||||
<iframe
|
||||
title="cross-origin Canvas editor"
|
||||
src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
|
||||
></iframe>
|
||||
</body>
|
||||
</html>
|
||||
@@ -19,6 +19,18 @@
|
||||
if (params.get("canvas") === "0") canvas.remove();
|
||||
else {
|
||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||
if (params.get("editor") === "1") {
|
||||
const context = canvas.getContext("2d");
|
||||
context.font = "14px system-ui, sans-serif";
|
||||
context.textBaseline = "middle";
|
||||
context.strokeStyle = "#1e8449";
|
||||
context.fillStyle = "#145a32";
|
||||
for (let row = 0; row < 3; row += 1) {
|
||||
const top = row * 30;
|
||||
context.strokeRect(0.5, top + 0.5, 219, 29);
|
||||
context.fillText(`Row ${row + 1}`, 12, top + 15);
|
||||
}
|
||||
}
|
||||
if (params.get("interactive") === "1") {
|
||||
const status = document.createElement("p");
|
||||
status.id = "pointer-status";
|
||||
@@ -39,6 +51,22 @@
|
||||
canvas.addEventListener("click", () => {
|
||||
status.textContent += "; click received";
|
||||
});
|
||||
if (params.get("editor") === "1") {
|
||||
canvas.addEventListener("click", (event) => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
|
||||
const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
|
||||
status.textContent += `; selected row=${row + 1}`;
|
||||
document.querySelector("#canvas-cell-editor")?.remove();
|
||||
if (row !== 2) return;
|
||||
const editor = document.createElement("input");
|
||||
editor.id = "canvas-cell-editor";
|
||||
editor.setAttribute("aria-label", "selected canvas cell editor");
|
||||
editor.placeholder = "Type into selected row 3";
|
||||
canvas.insertAdjacentElement("afterend", editor);
|
||||
editor.focus();
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.get("nested") === "1") {
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style>
|
||||
body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
|
||||
label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
|
||||
input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
|
||||
iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<label>Frame input <input aria-label="frame input" /></label>
|
||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||
<div contenteditable="true" aria-label="frame editor"></div>
|
||||
<label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
|
||||
<label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
|
||||
<label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
|
||||
<output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
|
||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const remountHost = document.querySelector("#remount-host");
|
||||
const armRemount = (input) => {
|
||||
input.addEventListener("input", () => {
|
||||
const replacement = document.createElement("input");
|
||||
replacement.setAttribute("aria-label", "remounting input");
|
||||
replacement.value = input.value;
|
||||
armRemount(replacement);
|
||||
input.replaceWith(replacement);
|
||||
replacement.focus();
|
||||
}, { once: true });
|
||||
};
|
||||
armRemount(remountHost.querySelector("input"));
|
||||
const trusted = document.querySelector("#trusted");
|
||||
trusted.addEventListener("input", (event) => {
|
||||
if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
|
||||
});
|
||||
const proxy = document.querySelector("#proxy");
|
||||
const proxyOutput = document.querySelector("#proxy-output");
|
||||
let proxyValue = "";
|
||||
proxy.addEventListener("input", (event) => {
|
||||
if (event.isComposing) return;
|
||||
proxyValue += event.data ?? proxy.value;
|
||||
proxy.value = "";
|
||||
proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
|
||||
});
|
||||
const rejecting = document.querySelector("#rejecting");
|
||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||
if (params.get("nested") === "1") {
|
||||
const iframe = document.createElement("iframe");
|
||||
iframe.title = "nested input frame";
|
||||
iframe.src = "input-frame.html";
|
||||
document.body.append(iframe);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Frame input routing matrix</title>
|
||||
<style>
|
||||
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||
iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
|
||||
input { padding: 6px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Frame input routing matrix</h1>
|
||||
<label>Top input <input aria-label="top input" /></label>
|
||||
<div class="frames">
|
||||
<iframe title="same-origin inputs" src="input-frame.html"></iframe>
|
||||
<iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
|
||||
<iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
### Choosing `fill`, `type`, and `press`
|
||||
|
||||
Choose by the intended postcondition, not by whether observation happens to
|
||||
render `textbox [empty]`:
|
||||
|
||||
- `fill` sets a field's retained value. Use it when success means the target
|
||||
value must equal the requested text; it replaces existing content by default.
|
||||
- `type` inserts text at the active caret/selection in a live editing session.
|
||||
Use it when the application must consume keyboard/IME text; it never clears
|
||||
first and reports dispatch only.
|
||||
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||
Do not encode special keys inside `type --text`.
|
||||
|
||||
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||
available, or pass `--focused` only after an immediately preceding action in
|
||||
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||
application may already have consumed the text before clearing its proxy DOM
|
||||
value. Re-observe first. After `type`, verify the application output with one
|
||||
observation or screenshot; `verification=dispatched` is not business success.
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -95,6 +95,27 @@ fn split_target(
|
||||
}
|
||||
}
|
||||
|
||||
fn split_type_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
focused: bool,
|
||||
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||
if focused {
|
||||
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"pass a target or --focused, not both"
|
||||
)));
|
||||
}
|
||||
return Ok((None, None));
|
||||
}
|
||||
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||
CliError::Local(anyhow::anyhow!(
|
||||
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
#[group(id = "type_destination", required = true, multiple = false)]
|
||||
pub struct TypeDestinationArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||
#[arg(value_name = "TARGET")]
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||
#[arg(long)]
|
||||
pub focused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct TypeArgs {
|
||||
#[command(flatten)]
|
||||
pub destination: TypeDestinationArgs,
|
||||
|
||||
/// Text to dispatch through the live keyboard/IME editing session.
|
||||
#[arg(long)]
|
||||
pub text: String,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||
let (ref_, selector) = split_type_target(
|
||||
args.destination.target.clone(),
|
||||
args.destination.ref_.clone(),
|
||||
args.destination.selector.clone(),
|
||||
args.destination.focused,
|
||||
)?;
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let params = TypeParams {
|
||||
session_id: args.session.clone(),
|
||||
text: args.text.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
focused: args.destination.focused,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: TypeResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolType,
|
||||
params,
|
||||
"type-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target = if args.destination.focused {
|
||||
"focused".into()
|
||||
} else {
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||
};
|
||||
println!(
|
||||
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||
reply.tab_id, reply.text_length
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -659,6 +769,52 @@ mod tests {
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||
.expect("type args should parse");
|
||||
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||
assert_eq!(parsed.args.text, "测试");
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||
.expect("focused type args should parse");
|
||||
assert!(parsed.args.destination.focused);
|
||||
assert!(parsed.args.destination.target.is_none());
|
||||
|
||||
let focused = split_type_target(None, None, None, true).unwrap();
|
||||
assert_eq!(focused, (None, None));
|
||||
|
||||
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||
assert!(missing.to_string().contains("--focused"));
|
||||
|
||||
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||
assert!(mixed.to_string().contains("not both"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||
use clap::Parser;
|
||||
|
||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -171,6 +171,9 @@ pub enum Command {
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||
Type(TypeArgs),
|
||||
|
||||
/// Dispatch a keyboard key combo.
|
||||
Press(PressArgs),
|
||||
|
||||
|
||||
@@ -44,6 +44,8 @@ pub mod reason {
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
|
||||
pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||
@@ -234,6 +236,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
|
||||
summary: "focused frame target could not be determined",
|
||||
hint: Some(
|
||||
"pass an explicit --ref or --selector so the key event can be routed safely",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
|
||||
summary: "the page did not retain the requested input",
|
||||
hint: Some(
|
||||
"refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
@@ -532,4 +548,20 @@ mod tests {
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("exactly one"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn focus_target_unresolved_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "focused frame target could not be determined");
|
||||
assert!(info.hint.unwrap().contains("--ref"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_not_applied_overrides_cdp_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||
assert_eq!(info.summary, "the page did not retain the requested input");
|
||||
assert!(info.hint.unwrap().contains("rejecting"));
|
||||
}
|
||||
}
|
||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
|
||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||
WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn type_round_trips_dispatched_verification() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolType);
|
||||
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||
assert_eq!(p.text, "测试");
|
||||
assert!(!p.focused);
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(TypeResult {
|
||||
tab_id: 12,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: TypeResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id,
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("type ok");
|
||||
assert_eq!(result.text_length, 2);
|
||||
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn press_round_trips_compound_key() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::CdpFailed,
|
||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
.unwrap_err();
|
||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||
|
||||
let type_result = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id: session_id.clone(),
|
||||
text: "hello".into(),
|
||||
ref_: Some("@e1".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||
|
||||
let press = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolPress,
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
|
||||
"type": "string"
|
||||
},
|
||||
"truncated": {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id",
|
||||
"text"
|
||||
],
|
||||
"properties": {
|
||||
"focused": {
|
||||
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"ref": {
|
||||
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
|
||||
"type": "string"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tab_id",
|
||||
"text_length",
|
||||
"verification"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"text_length": {
|
||||
"description": "UTF-16 code-unit length of text accepted for dispatch.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"verification": {
|
||||
"$ref": "#/definitions/TypeVerification"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"TypeVerification": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"dispatched"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,8 @@ fn main() {
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(TypeParams, "tool_type_params");
|
||||
dump!(TypeResult, "tool_type_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
dump!(PressResult, "tool_press_result");
|
||||
dump!(SelectParams, "tool_select_params");
|
||||
|
||||
@@ -74,6 +74,8 @@ pub enum Method {
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.type")]
|
||||
ToolType,
|
||||
#[serde(rename = "tool.press")]
|
||||
ToolPress,
|
||||
#[serde(rename = "tool.select")]
|
||||
@@ -154,6 +156,7 @@ impl Method {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
@@ -246,6 +249,13 @@ mod tests {
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_method_round_trips() {
|
||||
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
|
||||
assert_eq!(method, Method::ToolType);
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_params_and_result_round_trip() {
|
||||
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
||||
@@ -284,6 +294,7 @@ mod tests {
|
||||
assert!(Method::ToolReload.is_mutating());
|
||||
assert!(Method::ToolClick.is_mutating());
|
||||
assert!(Method::ToolFill.is_mutating());
|
||||
assert!(Method::ToolType.is_mutating());
|
||||
assert!(Method::ToolPress.is_mutating());
|
||||
assert!(Method::ToolSelect.is_mutating());
|
||||
assert!(Method::ToolEvaluate.is_mutating());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
|
||||
//! `tool.select`).
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
|
||||
//! `tool.press`, `tool.select`).
|
||||
//!
|
||||
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
||||
//! normalised against the session's RefStore) **or** a CSS selector
|
||||
@@ -188,6 +188,60 @@ pub struct FillResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeParams {
|
||||
pub session_id: String,
|
||||
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
|
||||
/// does not require the focused DOM editable to retain this text.
|
||||
pub text: String,
|
||||
/// Target to focus before dispatching text. Exactly one targeting mode
|
||||
/// must be selected: `ref` / `selector`, or `focused=true`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Explicitly dispatch to the unique focused text-entry target instead
|
||||
/// of focusing a ref or selector.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub focused: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TypeVerification {
|
||||
/// Chromium accepted the complete keyboard/IME dispatch transaction.
|
||||
/// The caller must observe the application output to verify business state.
|
||||
Dispatched,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// UTF-16 code-unit length of text accepted for dispatch.
|
||||
pub text_length: u32,
|
||||
pub verification: TypeVerification,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -351,6 +405,50 @@ mod tests {
|
||||
assert_eq!(v, json!("meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_payloads_distinguish_dispatch_from_value_verification() {
|
||||
let params = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: Some(4),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let encoded = serde_json::to_value(¶ms).unwrap();
|
||||
assert_eq!(encoded["ref"], json!("@e3"));
|
||||
assert_eq!(encoded["text"], json!("测试"));
|
||||
assert!(encoded.get("focused").is_none());
|
||||
|
||||
let focused = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "hello".into(),
|
||||
ref_: None,
|
||||
selector: None,
|
||||
focused: true,
|
||||
tab_id: None,
|
||||
timeout_ms: None,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(focused).unwrap()["focused"],
|
||||
json!(true)
|
||||
);
|
||||
|
||||
let result = TypeResult {
|
||||
tab_id: 4,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(result).unwrap()["verification"],
|
||||
json!("dispatched")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn press_result_round_trips() {
|
||||
let r = PressResult {
|
||||
|
||||
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SnapshotResult {
|
||||
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill`
|
||||
/// selectors.
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
|
||||
/// `tool.type` targets.
|
||||
pub text: String,
|
||||
/// Number of `@e<N>` refs registered for this session by this
|
||||
/// snapshot. Equivalent to the size of the new ref-store map.
|
||||
|
||||
Reference in new issue
Block a user