mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a1904f8141 | ||
|
|
e069abaf2e |
No files matched your search
@@ -0,0 +1,104 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { fillEditable } from "../editable-input";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
interface FakeOptions {
|
||||||
|
initialValue?: string;
|
||||||
|
finalValue: string;
|
||||||
|
remount?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function editableState(value: string, connected = true, focused = true) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { supported: true, connected, focused, value, kind: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeEditableCdp(options: FakeOptions) {
|
||||||
|
const sent: Array<{ method: string; params?: object }> = [];
|
||||||
|
let inserted = false;
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||||
|
sent.push({ method, params });
|
||||||
|
if (method === "Input.insertText") {
|
||||||
|
inserted = true;
|
||||||
|
return {} as T;
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchKeyEvent") {
|
||||||
|
if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
|
||||||
|
return {} as T;
|
||||||
|
}
|
||||||
|
if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
|
||||||
|
|
||||||
|
const call = params as { objectId?: string; functionDeclaration?: string };
|
||||||
|
const fn = call.functionDeclaration ?? "";
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } } as T;
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
|
||||||
|
if (fn.includes("bsk-editable-live")) {
|
||||||
|
return {
|
||||||
|
result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
|
||||||
|
} as T;
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
if (call.objectId === "replacement") return editableState(options.finalValue) as T;
|
||||||
|
if (options.remount && inserted) {
|
||||||
|
return editableState(options.finalValue, false, false) as T;
|
||||||
|
}
|
||||||
|
return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
|
||||||
|
}
|
||||||
|
throw new Error("unexpected Runtime.callFunctionOn function");
|
||||||
|
});
|
||||||
|
return { cdp: { send } as CdpRunner, sent };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("fillEditable", () => {
|
||||||
|
it("replaces text through the browser input path without synthetic events or value setters", async () => {
|
||||||
|
const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "新值", replacedNode: false });
|
||||||
|
expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
|
||||||
|
const declarations = fake.sent
|
||||||
|
.map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
|
||||||
|
.join("\n");
|
||||||
|
expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reacquires the focused editable after a framework remount", async () => {
|
||||||
|
const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "persisted", replacedNode: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a page rollback after the bounded settlement", async () => {
|
||||||
|
const fake = makeEditableCdp({ finalValue: "" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: {
|
||||||
|
reason: "input_not_applied",
|
||||||
|
expected_value_length: 8,
|
||||||
|
actual_value_length: 0,
|
||||||
|
phase: "readback",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears to an empty value with a browser Backspace edit", async () => {
|
||||||
|
const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "" });
|
||||||
|
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||||
|
expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
handleHover,
|
handleHover,
|
||||||
handlePress,
|
handlePress,
|
||||||
handleSelect,
|
handleSelect,
|
||||||
|
handleType,
|
||||||
modifiersBitfield,
|
modifiersBitfield,
|
||||||
parseKeySpec,
|
parseKeySpec,
|
||||||
resolveKeyDescriptor,
|
resolveKeyDescriptor,
|
||||||
@@ -34,6 +35,15 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
if (!h && method === "Page.getLayoutMetrics") {
|
if (!h && method === "Page.getLayoutMetrics") {
|
||||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
!h &&
|
||||||
|
method === "Runtime.evaluate" &&
|
||||||
|
String((params as { expression?: string })?.expression ?? "").includes(
|
||||||
|
"bsk-focus-target-probe",
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return { result: { value: { hasFocus: true, activeTag: "input" } } };
|
||||||
|
}
|
||||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||||
return h(params);
|
return h(params);
|
||||||
};
|
};
|
||||||
@@ -45,6 +55,10 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
params?: object,
|
params?: object,
|
||||||
) => Promise<T>,
|
) => Promise<T>,
|
||||||
trackSessionTab: vi.fn(),
|
trackSessionTab: vi.fn(),
|
||||||
|
getFrameGraph: vi.fn(async (tabId) => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [{ frameId: "main", target: { tabId } }],
|
||||||
|
})),
|
||||||
};
|
};
|
||||||
const tabsApi = {
|
const tabsApi = {
|
||||||
get: vi.fn(
|
get: vi.fn(
|
||||||
@@ -55,6 +69,29 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
return { cdp, tabsApi, sent };
|
return { cdp, tabsApi, sent };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function fillRuntimeReadback(value: string) {
|
||||||
|
return (params: unknown) => {
|
||||||
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value,
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
|
}
|
||||||
|
return { result: { type: "undefined" } };
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
|
||||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const context = await manager.start("aa11");
|
const context = await manager.start("aa11");
|
||||||
@@ -733,7 +770,7 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
"Runtime.callFunctionOn": fillRuntimeReadback("hello"),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -747,13 +784,13 @@ describe("handleFill", () => {
|
|||||||
expect(res.used_ref).toBe("e1");
|
expect(res.used_ref).toBe("e1");
|
||||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||||
expect(insert?.params).toEqual({ text: "hello" });
|
expect(insert?.params).toEqual({ text: "hello" });
|
||||||
// clear_before defaults to true → callFunctionOn invoked once to
|
// clear_before defaults to true, so the edit transaction prepares a
|
||||||
// clear, once to fire input/change after typing.
|
// replacement selection and performs a settled live-value readback.
|
||||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clear_before=false skips the wipe call", async () => {
|
it("clear_before=false skips replacement selection", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||||
@@ -766,9 +803,20 @@ describe("handleFill", () => {
|
|||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||||
"Runtime.callFunctionOn": (p) => {
|
"Runtime.callFunctionOn": (p) => {
|
||||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
// No "this.value = ''" clearing on a no-clear path; only the
|
if (fn.includes("bsk-editable-state")) {
|
||||||
// post-input dispatchEvent.
|
return {
|
||||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "existingx",
|
||||||
|
kind: "textarea",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||||
return { result: { type: "undefined" } };
|
return { result: { type: "undefined" } };
|
||||||
},
|
},
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
@@ -779,7 +827,7 @@ describe("handleFill", () => {
|
|||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
);
|
);
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
expect(res.value_length).toBe(1);
|
expect(res.value_length).toBe(9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("treats contenteditable=true as fillable", async () => {
|
it("treats contenteditable=true as fillable", async () => {
|
||||||
@@ -797,7 +845,7 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-3" } }),
|
||||||
"Runtime.callFunctionOn": () => ({ result: { type: "undefined" } }),
|
"Runtime.callFunctionOn": fillRuntimeReadback("rich"),
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
});
|
});
|
||||||
const res = await handleFill(
|
const res = await handleFill(
|
||||||
@@ -808,7 +856,103 @@ describe("handleFill", () => {
|
|||||||
expect("code" in res).toBe(false);
|
expect("code" in res).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
it("routes OOPIF focus, text, DOM events, and readback through the node target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: string[] = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (target, method, params) => {
|
||||||
|
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||||
|
childCalls.push(method);
|
||||||
|
if (method === "DOM.describeNode") {
|
||||||
|
return { node: { backendNodeId: 77, nodeName: "INPUT", attributes: [] } };
|
||||||
|
}
|
||||||
|
if (method === "DOM.scrollIntoViewIfNeeded" || method === "DOM.focus") return {};
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "child-input" } };
|
||||||
|
if (method === "Input.insertText") {
|
||||||
|
expect(params).toEqual({ text: "中文输入" });
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "中文输入",
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
|
}
|
||||||
|
return { result: { type: "undefined" } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child call ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handleFill(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", value: "中文输入" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(JSON.stringify(res));
|
||||||
|
expect(res.value_length).toBe(4);
|
||||||
|
expect(childCalls).toContain("DOM.focus");
|
||||||
|
expect(childCalls).toContain("Input.insertText");
|
||||||
|
expect(childCalls.filter((method) => method === "Runtime.callFunctionOn")).toHaveLength(4);
|
||||||
|
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns input_not_applied when a controlled input resets the value", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 8, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.describeNode": () => ({ node: { backendNodeId: 8, nodeName: "INPUT" } }),
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.focus": () => ({}),
|
||||||
|
"DOM.resolveNode": () => ({ object: { objectId: "controlled" } }),
|
||||||
|
"Input.insertText": () => ({}),
|
||||||
|
"Runtime.callFunctionOn": fillRuntimeReadback(""),
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleFill(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", value: "rejected" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "input_not_applied", expected_value_length: 8, actual_value_length: 0 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops before Input.insertText when abort fires after preparing replacement", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e1", 555, { tabId: 4 });
|
ctx.refStore.set("e1", 555, { tabId: 4 });
|
||||||
@@ -820,8 +964,25 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||||
"Runtime.callFunctionOn": () => {
|
"Runtime.callFunctionOn": (params) => {
|
||||||
abort.abort();
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "",
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
abort.abort();
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
|
}
|
||||||
return { result: { type: "undefined" } };
|
return { result: { type: "undefined" } };
|
||||||
},
|
},
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
@@ -864,6 +1025,204 @@ describe("handleFill", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleType", () => {
|
||||||
|
it("requires an explicit target mode instead of silently using focus", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({});
|
||||||
|
|
||||||
|
const missing = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
expect(missing).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: expect.stringContaining("focused=true"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const mixed = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", focused: true, text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
expect(mixed).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: expect.stringContaining("not both"),
|
||||||
|
});
|
||||||
|
expect(fake.sent).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
|
||||||
|
childCalls.push({ method, params });
|
||||||
|
if (method === "DOM.describeNode") {
|
||||||
|
return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "测试" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
tab_id: 4,
|
||||||
|
used_ref: "e1",
|
||||||
|
text_length: 2,
|
||||||
|
verification: "dispatched",
|
||||||
|
});
|
||||||
|
expect(childCalls.map((call) => call.method)).toEqual([
|
||||||
|
"DOM.describeNode",
|
||||||
|
"DOM.scrollIntoViewIfNeeded",
|
||||||
|
"DOM.focus",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.insertText",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: string[] = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
childCalls.push(method);
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
return {
|
||||||
|
result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", focused: true, text: "A" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||||
|
expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
|
||||||
|
expect(childCalls).toEqual([
|
||||||
|
"Runtime.evaluate",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses focused mode when focus is not owned by a text-entry target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", focused: true, text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "focus_target_unresolved", candidate_count: 0 },
|
||||||
|
});
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects explicit non-text input controls", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.describeNode": () => ({
|
||||||
|
node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
data: { reason: "target_not_fillable" },
|
||||||
|
});
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a Surface ref instead of treating it as a text target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, {
|
||||||
|
tabId: 4,
|
||||||
|
kind: "surface",
|
||||||
|
capabilities: ["screenshot"],
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "permission_denied",
|
||||||
|
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("parseKeySpec", () => {
|
describe("parseKeySpec", () => {
|
||||||
it("splits compound expressions into modifiers + base key", () => {
|
it("splits compound expressions into modifiers + base key", () => {
|
||||||
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
||||||
@@ -1002,6 +1361,125 @@ describe("handlePress", () => {
|
|||||||
expect(res.key).toBe("Enter");
|
expect(res.key).toBe("Enter");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("routes ref-targeted key events through the OOPIF session", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 555, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: string[] = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
childCalls.push(method);
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handlePress(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", key: "Enter", ref: "@e1" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expectPressOk(res);
|
||||||
|
expect(childCalls).toEqual([
|
||||||
|
"DOM.scrollIntoViewIfNeeded",
|
||||||
|
"DOM.focus",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes an untargeted press to the uniquely focused OOPIF session", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "iframe" } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: string[] = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
childCalls.push(method);
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
return { result: { value: { hasFocus: true, activeTag: "textarea" } } };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handlePress(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", key: "a" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expectPressOk(res);
|
||||||
|
expect(childCalls).toEqual([
|
||||||
|
"Runtime.evaluate",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses an untargeted press when focus ownership is ambiguous", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "input" } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{ frameId: "child", target: { tabId: 4, sessionId: "child-session" } },
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "textarea" } },
|
||||||
|
})) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handlePress(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", key: "Enter" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "focus_target_unresolved", candidate_count: 2 },
|
||||||
|
});
|
||||||
|
expect(fake.sent.some((call) => call.method === "Input.dispatchKeyEvent")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("stops before focus when abort fires after press scroll", async () => {
|
it("stops before focus when abort fires after press scroll", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { dispatchTextInput } from "../text-input";
|
||||||
|
|
||||||
|
function makeCdp(onSend?: (method: string, params?: object) => void) {
|
||||||
|
const sent: Array<{ method: string; params?: object }> = [];
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||||
|
sent.push({ method, params });
|
||||||
|
onSend?.(method, params);
|
||||||
|
return {} as T;
|
||||||
|
});
|
||||||
|
return { cdp: { send } as CdpRunner, sent };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("dispatchTextInput", () => {
|
||||||
|
it("types ASCII through complete keydown, char, and keyup sequences", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "Ab");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(
|
||||||
|
fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
|
||||||
|
).toEqual([
|
||||||
|
"Input.dispatchKeyEvent:rawKeyDown",
|
||||||
|
"Input.dispatchKeyEvent:char",
|
||||||
|
"Input.dispatchKeyEvent:keyUp",
|
||||||
|
"Input.dispatchKeyEvent:rawKeyDown",
|
||||||
|
"Input.dispatchKeyEvent:char",
|
||||||
|
"Input.dispatchKeyEvent:keyUp",
|
||||||
|
]);
|
||||||
|
expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
|
||||||
|
expect(fake.sent[1].params).toMatchObject({ text: "A" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("types non-ASCII through the committed IME insertion path", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(fake.sent).toEqual([
|
||||||
|
{
|
||||||
|
method: "Input.dispatchKeyEvent",
|
||||||
|
params: {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ method: "Input.insertText", params: { text: "测试" } },
|
||||||
|
{
|
||||||
|
method: "Input.dispatchKeyEvent",
|
||||||
|
params: {
|
||||||
|
type: "keyUp",
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves text order across keyboard and IME segments", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
const meaningful = fake.sent.flatMap((call) => {
|
||||||
|
if (call.method === "Input.insertText") {
|
||||||
|
return [`ime:${(call.params as { text: string }).text}`];
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
call.method === "Input.dispatchKeyEvent" &&
|
||||||
|
(call.params as { type?: string }).type === "char"
|
||||||
|
) {
|
||||||
|
return [`key:${(call.params as { text: string }).text}`];
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports cancellation observed after an accepted IME insertion", async () => {
|
||||||
|
const abort = new AbortController();
|
||||||
|
const fake = makeCdp((method) => {
|
||||||
|
if (method === "Input.insertText") abort.abort();
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.insertText",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the IME process key without committing when cancellation follows keydown", async () => {
|
||||||
|
const abort = new AbortController();
|
||||||
|
const fake = makeCdp((method, params) => {
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchKeyEvent" &&
|
||||||
|
(params as { type?: string }).type === "rawKeyDown"
|
||||||
|
) {
|
||||||
|
abort.abort();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
|
||||||
|
expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -28,6 +28,7 @@ import type {
|
|||||||
ScreenshotParams,
|
ScreenshotParams,
|
||||||
SelectParams,
|
SelectParams,
|
||||||
SnapshotParams,
|
SnapshotParams,
|
||||||
|
TypeParams,
|
||||||
WaitForNavigationParams,
|
WaitForNavigationParams,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { isRequestFrame } from "@/transport/types";
|
import { isRequestFrame } from "@/transport/types";
|
||||||
@@ -35,7 +36,14 @@ import { handleConsole } from "./console";
|
|||||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
import {
|
||||||
|
handleClick,
|
||||||
|
handleFill,
|
||||||
|
handleHover,
|
||||||
|
handlePress,
|
||||||
|
handleSelect,
|
||||||
|
handleType,
|
||||||
|
} from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
handleNavigateBack,
|
handleNavigateBack,
|
||||||
@@ -491,6 +499,17 @@ export class ToolDispatcher {
|
|||||||
),
|
),
|
||||||
signal,
|
signal,
|
||||||
);
|
);
|
||||||
|
case "tool.type":
|
||||||
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as TypeParams,
|
||||||
|
() =>
|
||||||
|
handleType(
|
||||||
|
this.sessions,
|
||||||
|
req.params as TypeParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
return this.withHoverReleaseForRequest(
|
return this.withHoverReleaseForRequest(
|
||||||
req.params as PressParams,
|
req.params as PressParams,
|
||||||
@@ -717,6 +736,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
case "tool.click":
|
case "tool.click":
|
||||||
case "tool.hover":
|
case "tool.hover":
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
|
case "tool.type":
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
case "tool.evaluate":
|
case "tool.evaluate":
|
||||||
|
|||||||
@@ -0,0 +1,282 @@
|
|||||||
|
// Browser-native editing transaction for input, textarea, and contenteditable
|
||||||
|
// targets. The CDP target/session is selected by interaction.ts; this module
|
||||||
|
// owns the shorter-lived editable identity inside that target.
|
||||||
|
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { rpcError } from "./errors";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
|
||||||
|
interface EditableState {
|
||||||
|
supported: boolean;
|
||||||
|
connected: boolean;
|
||||||
|
focused: boolean;
|
||||||
|
value: string;
|
||||||
|
kind: "input" | "textarea" | "contenteditable" | "unsupported";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EditableFillResult {
|
||||||
|
value: string;
|
||||||
|
replacedNode: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||||
|
return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function inspectEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<EditableState> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { value?: EditableState };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-state */
|
||||||
|
const doc = this.ownerDocument;
|
||||||
|
const focused = !!doc && doc.activeElement === this;
|
||||||
|
if (this instanceof HTMLInputElement) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.value, kind: 'input' };
|
||||||
|
}
|
||||||
|
if (this instanceof HTMLTextAreaElement) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.value, kind: 'textarea' };
|
||||||
|
}
|
||||||
|
if (this.isContentEditable) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
|
||||||
|
}
|
||||||
|
return { supported: false, connected: this.isConnected === true, focused,
|
||||||
|
value: '', kind: 'unsupported' };
|
||||||
|
}`,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const state = reply.result?.value;
|
||||||
|
if (!state || typeof state.value !== "string") {
|
||||||
|
throw new Error("editable state probe returned no value");
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the active editable from the anchor's own document. Using the
|
||||||
|
* anchor's execution context also works for same-process nested frames, where
|
||||||
|
* a target-level Runtime.evaluate would otherwise see the top document.
|
||||||
|
*/
|
||||||
|
async function focusedEditableFromAnchor(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<string | null> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { objectId?: string; subtype?: string };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId: anchorObjectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-live */
|
||||||
|
const active = this.ownerDocument?.activeElement;
|
||||||
|
if (!active) return null;
|
||||||
|
const editable = active instanceof HTMLInputElement ||
|
||||||
|
active instanceof HTMLTextAreaElement || active.isContentEditable;
|
||||||
|
return editable ? active : null;
|
||||||
|
}`,
|
||||||
|
returnByValue: false,
|
||||||
|
});
|
||||||
|
if (reply.result?.subtype === "null") return null;
|
||||||
|
return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveLiveEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
|
||||||
|
const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
|
||||||
|
if (anchor.supported && anchor.connected && anchor.focused) {
|
||||||
|
return { objectId: anchorObjectId, state: anchor, replaced: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
|
||||||
|
if (!focusedObjectId) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the target editable lost focus or detached before input",
|
||||||
|
{ phase: "resolve_live_editable" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const focused = await inspectEditable(cdp, tabId, focusedObjectId);
|
||||||
|
if (!focused.supported || !focused.connected || !focused.focused) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the replacement editable was not live and focused",
|
||||||
|
{ phase: "resolve_live_editable" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { objectId: focusedObjectId, state: focused, replaced: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function selectEditableContents(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { value?: { selected?: boolean } };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-selection */
|
||||||
|
if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
|
||||||
|
return { selected: false };
|
||||||
|
}
|
||||||
|
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||||
|
try {
|
||||||
|
this.setSelectionRange(0, this.value.length);
|
||||||
|
} catch {
|
||||||
|
if (typeof this.select !== 'function') return { selected: false };
|
||||||
|
this.select();
|
||||||
|
}
|
||||||
|
return { selected: true };
|
||||||
|
}
|
||||||
|
if (this.isContentEditable) {
|
||||||
|
const selection = this.ownerDocument.getSelection();
|
||||||
|
if (!selection) return { selected: false };
|
||||||
|
const range = this.ownerDocument.createRange();
|
||||||
|
range.selectNodeContents(this);
|
||||||
|
selection.removeAllRanges();
|
||||||
|
selection.addRange(range);
|
||||||
|
return { selected: true };
|
||||||
|
}
|
||||||
|
return { selected: false };
|
||||||
|
}`,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
if (reply.result?.value?.selected !== true) {
|
||||||
|
throw new Error("could not select the live editable contents");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForEditableSettlement(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId: anchorObjectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-sync */
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
let settled = false;
|
||||||
|
const finish = () => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const timer = setTimeout(finish, 100);
|
||||||
|
const view = this.ownerDocument?.defaultView;
|
||||||
|
if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
|
||||||
|
view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
|
||||||
|
});
|
||||||
|
}`,
|
||||||
|
awaitPromise: true,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
key: "Backspace",
|
||||||
|
code: "Backspace",
|
||||||
|
windowsVirtualKeyCode: 8,
|
||||||
|
});
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
key: "Backspace",
|
||||||
|
code: "Backspace",
|
||||||
|
windowsVirtualKeyCode: 8,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replace or append text through Chromium's editing pipeline. No value setter
|
||||||
|
* or synthetic DOM events are used: controlled components observe the same
|
||||||
|
* beforeinput/input path as a real browser edit. The live editable is resolved
|
||||||
|
* again after settlement so a framework may remount it without producing a
|
||||||
|
* false failure or a stale-object success.
|
||||||
|
*/
|
||||||
|
export async function fillEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
value: string,
|
||||||
|
clearBefore: boolean,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<EditableFillResult | RpcError> {
|
||||||
|
try {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
|
||||||
|
if ("code" in initial) return initial;
|
||||||
|
let replacedNode = initial.replaced;
|
||||||
|
|
||||||
|
if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
|
||||||
|
const abortedAfterSelection = cancelled(signal);
|
||||||
|
if (abortedAfterSelection) return abortedAfterSelection;
|
||||||
|
|
||||||
|
if (value.length > 0) {
|
||||||
|
await cdp.send(tabId, "Input.insertText", { text: value });
|
||||||
|
} else if (clearBefore) {
|
||||||
|
await deleteSelection(cdp, tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const abortedAfterInput = cancelled(signal);
|
||||||
|
if (abortedAfterInput) return abortedAfterInput;
|
||||||
|
await waitForEditableSettlement(cdp, tabId, initial.objectId);
|
||||||
|
|
||||||
|
const abortedAfterSettlement = cancelled(signal);
|
||||||
|
if (abortedAfterSettlement) return abortedAfterSettlement;
|
||||||
|
|
||||||
|
let finalObjectId = initial.objectId;
|
||||||
|
let finalState = await inspectEditable(cdp, tabId, finalObjectId);
|
||||||
|
if (!finalState.connected || !finalState.focused) {
|
||||||
|
const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
|
||||||
|
if ("code" in live) return live;
|
||||||
|
finalObjectId = live.objectId;
|
||||||
|
finalState = live.state;
|
||||||
|
replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const applied = clearBefore
|
||||||
|
? finalState.value === value
|
||||||
|
: value.length === 0 || finalState.value.includes(value);
|
||||||
|
if (!finalState.supported || !applied) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the page did not retain the requested input value",
|
||||||
|
{
|
||||||
|
expected_value_length: value.length,
|
||||||
|
actual_value_length: finalState.value.length,
|
||||||
|
phase: "readback",
|
||||||
|
editable_replaced: replacedNode,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { value: finalState.value, replacedNode };
|
||||||
|
} catch (err) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
{ phase: "edit_transaction" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
|
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
|
||||||
// `tool.select`.
|
// `tool.press`, and `tool.select`.
|
||||||
//
|
//
|
||||||
// All interaction tools:
|
// All interaction tools:
|
||||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||||
@@ -11,7 +11,7 @@
|
|||||||
// commands.
|
// commands.
|
||||||
|
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
ClickParams,
|
ClickParams,
|
||||||
@@ -27,8 +27,11 @@ import type {
|
|||||||
RpcError,
|
RpcError,
|
||||||
SelectParams,
|
SelectParams,
|
||||||
SelectResult,
|
SelectResult,
|
||||||
|
TypeParams,
|
||||||
|
TypeResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
|
import { fillEditable } from "./editable-input";
|
||||||
import { backendNodeToObject } from "./element-geometry";
|
import { backendNodeToObject } from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||||
@@ -45,6 +48,7 @@ import {
|
|||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||||
|
import { dispatchTextInput } from "./text-input";
|
||||||
|
|
||||||
export interface InteractionDeps {
|
export interface InteractionDeps {
|
||||||
cdp: CdpRunner;
|
cdp: CdpRunner;
|
||||||
@@ -217,6 +221,123 @@ async function resolveBackendNode(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface InputDispatchContext {
|
||||||
|
target: CdpTarget;
|
||||||
|
cdp: CdpRunner;
|
||||||
|
backendNodeId?: number;
|
||||||
|
frameId?: string;
|
||||||
|
usedRef?: string;
|
||||||
|
usedSelector?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function inputContextForNode(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
node: Awaited<ReturnType<typeof resolveBackendNode>>,
|
||||||
|
): (InputDispatchContext & { backendNodeId: number }) | RpcError {
|
||||||
|
if (isRpcError(node)) return node;
|
||||||
|
return {
|
||||||
|
target: node.cdpTarget,
|
||||||
|
cdp: cdpRunnerForTarget(cdp, node.cdpTarget),
|
||||||
|
backendNodeId: node.backendNodeId,
|
||||||
|
frameId: node.frameId,
|
||||||
|
usedRef: node.usedRef,
|
||||||
|
usedSelector: node.usedSelector,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FocusProbe {
|
||||||
|
hasFocus: boolean;
|
||||||
|
activeTag: string;
|
||||||
|
textEntry: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||||
|
async function resolveFocusedInputContext(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
mode: "press" | "type",
|
||||||
|
): Promise<InputDispatchContext | RpcError> {
|
||||||
|
if (!cdp.getFrameGraph) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"focus_target_unresolved",
|
||||||
|
`${mode} cannot determine the focused frame target`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let targets: CdpTarget[];
|
||||||
|
try {
|
||||||
|
const graph = await cdp.getFrameGraph(tabId);
|
||||||
|
const unique = new Map<string, CdpTarget>();
|
||||||
|
unique.set(cdpTargetKey({ tabId }), { tabId });
|
||||||
|
for (const frame of graph.frames) unique.set(cdpTargetKey(frame.target), frame.target);
|
||||||
|
targets = [...unique.values()];
|
||||||
|
} catch (err) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"focus_target_unresolved",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const focused: Array<{ target: CdpTarget; probe: FocusProbe }> = [];
|
||||||
|
await Promise.all(
|
||||||
|
targets.map(async (candidate) => {
|
||||||
|
try {
|
||||||
|
const reply = await cdpRunnerForTarget(cdp, candidate).send<{
|
||||||
|
result?: { value?: FocusProbe };
|
||||||
|
}>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: `(() => {
|
||||||
|
/* bsk-focus-target-probe */
|
||||||
|
const active = document.activeElement;
|
||||||
|
const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
|
||||||
|
const nonTextInputTypes = new Set([
|
||||||
|
'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
|
||||||
|
'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
|
||||||
|
'time', 'week'
|
||||||
|
]);
|
||||||
|
const role = active instanceof Element ? active.getAttribute('role') : null;
|
||||||
|
return {
|
||||||
|
hasFocus: document.hasFocus(),
|
||||||
|
activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
|
||||||
|
textEntry:
|
||||||
|
(active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
|
||||||
|
active instanceof HTMLTextAreaElement ||
|
||||||
|
(active instanceof HTMLElement && active.isContentEditable) ||
|
||||||
|
role === 'textbox' || role === 'searchbox'
|
||||||
|
};
|
||||||
|
})()`,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const probe = reply.result?.value;
|
||||||
|
if (probe?.hasFocus) focused.push({ target: candidate, probe });
|
||||||
|
} catch {
|
||||||
|
// A target may navigate or detach while the graph is being probed.
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
// A top document whose active element is an iframe delegates focus to
|
||||||
|
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||||
|
// target while same-process frames continue to use the shared root target.
|
||||||
|
const owners = focused.filter(({ probe }) => {
|
||||||
|
if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
|
||||||
|
return mode !== "type" || probe.textEntry;
|
||||||
|
});
|
||||||
|
if (owners.length !== 1) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"focus_target_unresolved",
|
||||||
|
owners.length === 0
|
||||||
|
? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
|
||||||
|
: `${mode} found multiple focused frame targets`,
|
||||||
|
{ candidate_count: owners.length },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const target = owners[0].target;
|
||||||
|
return { target, cdp: cdpRunnerForTarget(cdp, target) };
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// tool.click
|
// tool.click
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -580,6 +701,47 @@ function isFillable(node: DescribedNode): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** `type` also supports custom text-entry widgets that expose an ARIA role. */
|
||||||
|
function isTypeTarget(node: DescribedNode): boolean {
|
||||||
|
const tag = (node.nodeName ?? "").toUpperCase();
|
||||||
|
const attrs = node.attributes ?? [];
|
||||||
|
const attributes = new Map<string, string>();
|
||||||
|
for (let i = 0; i + 1 < attrs.length; i += 2) {
|
||||||
|
attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
|
||||||
|
}
|
||||||
|
if (tag === "TEXTAREA") return true;
|
||||||
|
if (tag === "INPUT") {
|
||||||
|
const type = attributes.get("type") ?? "text";
|
||||||
|
return !new Set([
|
||||||
|
"button",
|
||||||
|
"checkbox",
|
||||||
|
"color",
|
||||||
|
"date",
|
||||||
|
"datetime-local",
|
||||||
|
"file",
|
||||||
|
"hidden",
|
||||||
|
"image",
|
||||||
|
"month",
|
||||||
|
"radio",
|
||||||
|
"range",
|
||||||
|
"reset",
|
||||||
|
"submit",
|
||||||
|
"time",
|
||||||
|
"week",
|
||||||
|
]).has(type);
|
||||||
|
}
|
||||||
|
const contentEditable = attributes.get("contenteditable");
|
||||||
|
if (
|
||||||
|
contentEditable === "" ||
|
||||||
|
contentEditable === "true" ||
|
||||||
|
contentEditable === "plaintext-only"
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const role = attributes.get("role");
|
||||||
|
return role === "textbox" || role === "searchbox";
|
||||||
|
}
|
||||||
|
|
||||||
type SelectMutationResult =
|
type SelectMutationResult =
|
||||||
| {
|
| {
|
||||||
ok: true;
|
ok: true;
|
||||||
@@ -618,8 +780,9 @@ export async function handleFill(
|
|||||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||||
if (isRpcError(node)) return node;
|
const input = inputContextForNode(deps.cdp, node);
|
||||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
if (isRpcError(input)) return input;
|
||||||
|
const nodeCdp = input.cdp;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
@@ -627,7 +790,7 @@ export async function handleFill(
|
|||||||
target.tabId,
|
target.tabId,
|
||||||
"DOM.describeNode",
|
"DOM.describeNode",
|
||||||
{
|
{
|
||||||
backendNodeId: node.backendNodeId,
|
backendNodeId: input.backendNodeId,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
if (!described.node || !isFillable(described.node)) {
|
if (!described.node || !isFillable(described.node)) {
|
||||||
@@ -640,15 +803,15 @@ export async function handleFill(
|
|||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
deps.cdp,
|
deps.cdp,
|
||||||
target.tabId,
|
target.tabId,
|
||||||
node.cdpTarget,
|
input.target,
|
||||||
node.backendNodeId,
|
input.backendNodeId,
|
||||||
node.frameId,
|
input.frameId,
|
||||||
);
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: input.backendNodeId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
@@ -660,62 +823,118 @@ export async function handleFill(
|
|||||||
return { code: "cancelled", message: "fill aborted" };
|
return { code: "cancelled", message: "fill aborted" };
|
||||||
}
|
}
|
||||||
|
|
||||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, input.backendNodeId);
|
||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
|
const fillResult = await fillEditable(
|
||||||
try {
|
nodeCdp,
|
||||||
if (clearBefore) {
|
target.tabId,
|
||||||
// Clear input/textarea value or wipe contenteditable innerText,
|
objectId,
|
||||||
// then fire `input` so frameworks observe the empty state.
|
params.value,
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
clearBefore,
|
||||||
objectId,
|
deps.signal,
|
||||||
functionDeclaration: `function() {
|
);
|
||||||
if (this.isContentEditable) { this.textContent = ''; }
|
if (isRpcError(fillResult)) return fillResult;
|
||||||
else {
|
|
||||||
const proto = this instanceof HTMLTextAreaElement
|
|
||||||
? HTMLTextAreaElement.prototype
|
|
||||||
: HTMLInputElement.prototype;
|
|
||||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
|
||||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
|
||||||
else this.value = '';
|
|
||||||
}
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
|
||||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
|
||||||
await deps.cdp.send(target.tabId, "Input.insertText", { text: params.value });
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
return {
|
|
||||||
code: "cdp_failed",
|
|
||||||
message: err instanceof Error ? err.message : String(err),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
used_ref: node.usedRef,
|
used_ref: input.usedRef,
|
||||||
used_selector: node.usedSelector,
|
used_selector: input.usedSelector,
|
||||||
value_length: params.value.length,
|
value_length: fillResult.value.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tool.type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function handleType(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: TypeParams,
|
||||||
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
|
): Promise<TypeResult | RpcError> {
|
||||||
|
if (!params || typeof params.text !== "string") {
|
||||||
|
return { code: "invalid_params", message: "type requires a text string" };
|
||||||
|
}
|
||||||
|
if (params.focused !== undefined && typeof params.focused !== "boolean") {
|
||||||
|
return { code: "invalid_params", message: "type focused must be a boolean" };
|
||||||
|
}
|
||||||
|
const hasExplicitTarget =
|
||||||
|
(typeof params.ref === "string" && params.ref.length > 0) ||
|
||||||
|
(typeof params.selector === "string" && params.selector.length > 0);
|
||||||
|
if (params.focused === true && hasExplicitTarget) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "type: pass a ref/selector or focused=true, not both",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (params.focused !== true && !hasExplicitTarget) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "type requires a ref/selector or focused=true",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "type");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "type");
|
||||||
|
if (denied) return denied;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
|
let input: InputDispatchContext | RpcError;
|
||||||
|
if (hasExplicitTarget) {
|
||||||
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
|
||||||
|
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||||
|
if (isRpcError(explicitInput)) return explicitInput;
|
||||||
|
input = explicitInput;
|
||||||
|
try {
|
||||||
|
const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
|
||||||
|
target.tabId,
|
||||||
|
"DOM.describeNode",
|
||||||
|
{ backendNodeId: explicitInput.backendNodeId },
|
||||||
|
);
|
||||||
|
if (!described.node || !isTypeTarget(described.node)) {
|
||||||
|
return rpcError(
|
||||||
|
"invalid_params",
|
||||||
|
"target_not_fillable",
|
||||||
|
`element ${described.node?.nodeName ?? "?"} not text-editable`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
explicitInput.target,
|
||||||
|
explicitInput.backendNodeId,
|
||||||
|
explicitInput.frameId,
|
||||||
|
);
|
||||||
|
if (scrollErr) return scrollErr;
|
||||||
|
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||||
|
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||||
|
backendNodeId: explicitInput.backendNodeId,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
|
||||||
|
if (isRpcError(input)) return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
|
||||||
|
if (dispatchError) return dispatchError;
|
||||||
|
|
||||||
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: input.usedRef,
|
||||||
|
used_selector: input.usedSelector,
|
||||||
|
text_length: params.text.length,
|
||||||
|
verification: "dispatched",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -890,31 +1109,38 @@ export async function handlePress(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let input: InputDispatchContext | RpcError;
|
||||||
// Optional focus before key dispatch.
|
// Optional focus before key dispatch.
|
||||||
if (params.ref || params.selector) {
|
if (params.ref || params.selector) {
|
||||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||||
if (isRpcError(node)) return node;
|
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
if (isRpcError(explicitInput)) return explicitInput;
|
||||||
|
input = explicitInput;
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
const scrollErr = await scrollElementAndFramesIntoView(
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
deps.cdp,
|
deps.cdp,
|
||||||
target.tabId,
|
target.tabId,
|
||||||
node.cdpTarget,
|
explicitInput.target,
|
||||||
node.backendNodeId,
|
explicitInput.backendNodeId,
|
||||||
node.frameId,
|
explicitInput.frameId,
|
||||||
);
|
);
|
||||||
if (scrollErr) return scrollErr;
|
if (scrollErr) return scrollErr;
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
return { code: "cancelled", message: "press aborted" };
|
return { code: "cancelled", message: "press aborted" };
|
||||||
}
|
}
|
||||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||||
|
backendNodeId: explicitInput.backendNodeId,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
|
||||||
|
if (isRpcError(input)) return input;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
if (throwIfAborted(deps.signal)) {
|
||||||
@@ -928,7 +1154,7 @@ export async function handlePress(
|
|||||||
try {
|
try {
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "rawKeyDown",
|
type: "rawKeyDown",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
@@ -942,7 +1168,7 @@ export async function handlePress(
|
|||||||
typeof descriptor.text === "string" &&
|
typeof descriptor.text === "string" &&
|
||||||
descriptor.text.length > 0
|
descriptor.text.length > 0
|
||||||
) {
|
) {
|
||||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "char",
|
type: "char",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
@@ -959,7 +1185,7 @@ export async function handlePress(
|
|||||||
cancelled = true;
|
cancelled = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
await input.cdp.send(target.tabId, "Input.dispatchKeyEvent", {
|
||||||
type: "keyUp",
|
type: "keyUp",
|
||||||
key: descriptor.key,
|
key: descriptor.key,
|
||||||
code: descriptor.code,
|
code: descriptor.code,
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
// Browser-native text entry for live editing sessions. Unlike fill, this
|
||||||
|
// module does not assume that the focused editable retains a DOM value: a
|
||||||
|
// Canvas editor may use an input/textarea only as a transient keyboard/IME
|
||||||
|
// sink and clear it after consuming each event.
|
||||||
|
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
|
||||||
|
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||||
|
return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface KeyDescriptor {
|
||||||
|
key: string;
|
||||||
|
code: string;
|
||||||
|
windowsVirtualKeyCode: number;
|
||||||
|
nativeVirtualKeyCode?: number;
|
||||||
|
modifiers: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function asciiKeyDescriptor(character: string): KeyDescriptor {
|
||||||
|
if (/^[a-z]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Key${character.toUpperCase()}`,
|
||||||
|
windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (/^[A-Z]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Key${character}`,
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 8,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (/^[0-9]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Digit${character}`,
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (character === " ") {
|
||||||
|
return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run one complete browser key transaction and always release the key. The
|
||||||
|
* commit callback owns the text-producing phase: a `char` event for a
|
||||||
|
* keyboard character, or `Input.insertText` for committed IME text.
|
||||||
|
*/
|
||||||
|
async function dispatchKeyTransaction(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
descriptor: KeyDescriptor,
|
||||||
|
commit: () => Promise<void>,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
let keyDown = false;
|
||||||
|
try {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
keyDown = true;
|
||||||
|
const abortedAfterDown = cancelled(signal);
|
||||||
|
if (!abortedAfterDown) await commit();
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
keyDown = false;
|
||||||
|
return abortedAfterDown ?? cancelled(signal);
|
||||||
|
} catch (err) {
|
||||||
|
if (keyDown) {
|
||||||
|
try {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup keeps the page from observing a stuck key.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function typeAsciiCharacter(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
character: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const descriptor = asciiKeyDescriptor(character);
|
||||||
|
return dispatchKeyTransaction(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
descriptor,
|
||||||
|
async () => {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "char",
|
||||||
|
...descriptor,
|
||||||
|
text: character,
|
||||||
|
unmodifiedText: character,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function typeImeSegment(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
text: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
// A committed IME value is still part of a keyboard transaction. The
|
||||||
|
// Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
|
||||||
|
// IME owns the keystroke. It lets selection-based editors enter text mode
|
||||||
|
// before insertText commits the Unicode value, without inventing a visible
|
||||||
|
// ASCII prefix or leaving a composition transaction open.
|
||||||
|
const processKey: KeyDescriptor = {
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
return dispatchKeyTransaction(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
processKey,
|
||||||
|
() => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
|
||||||
|
* delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
|
||||||
|
* IME/software-keyboard insertion path. Successful return means the browser
|
||||||
|
* accepted the input commands, not that a DOM value retained the text.
|
||||||
|
*/
|
||||||
|
export async function dispatchTextInput(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
text: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
const segments: Array<{ ime: boolean; text: string }> = [];
|
||||||
|
for (const character of text) {
|
||||||
|
const ime = character.codePointAt(0)! > 0x7f;
|
||||||
|
const previous = segments.at(-1);
|
||||||
|
if (previous?.ime === ime) previous.text += character;
|
||||||
|
else segments.push({ ime, text: character });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const segment of segments) {
|
||||||
|
if (segment.ime) {
|
||||||
|
const error = await typeImeSegment(cdp, tabId, segment.text, signal);
|
||||||
|
if (error) return error;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (const character of segment.text) {
|
||||||
|
const error = await typeAsciiCharacter(cdp, tabId, character, signal);
|
||||||
|
if (error) return error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,6 +27,8 @@ export type RpcErrorReason =
|
|||||||
| "ref_not_found"
|
| "ref_not_found"
|
||||||
| "selector_not_found"
|
| "selector_not_found"
|
||||||
| "target_not_fillable"
|
| "target_not_fillable"
|
||||||
|
| "focus_target_unresolved"
|
||||||
|
| "input_not_applied"
|
||||||
| "target_not_select"
|
| "target_not_select"
|
||||||
| "option_not_found"
|
| "option_not_found"
|
||||||
| "single_select_value_count"
|
| "single_select_value_count"
|
||||||
@@ -482,6 +484,26 @@ export interface FillResult {
|
|||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TypeParams {
|
||||||
|
session_id: string;
|
||||||
|
text: string;
|
||||||
|
ref?: string;
|
||||||
|
selector?: string;
|
||||||
|
/** Explicitly dispatch to the unique focused text-entry target. */
|
||||||
|
focused?: boolean;
|
||||||
|
tab_id?: number;
|
||||||
|
timeout_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TypeResult {
|
||||||
|
tab_id: number;
|
||||||
|
used_ref?: string;
|
||||||
|
used_selector?: string;
|
||||||
|
text_length: number;
|
||||||
|
verification: "dispatched";
|
||||||
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface PressParams {
|
export interface PressParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
key: string;
|
key: string;
|
||||||
|
|||||||
@@ -93,6 +93,40 @@ Run with a `900x700` Agent Window.
|
|||||||
- `fixture-below-fold` is absent initially.
|
- `fixture-below-fold` is absent initially.
|
||||||
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
|
||||||
|
|
||||||
|
### `input-frames.html`
|
||||||
|
|
||||||
|
This page exercises generic frame-aware input routing. It contains a top-level
|
||||||
|
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||||
|
frame includes an input, textarea, contenteditable editor, an input that
|
||||||
|
replaces its DOM node after accepted input, an input that rejects synthetic
|
||||||
|
DOM events, a transient keyboard/IME input proxy that consumes and clears its
|
||||||
|
DOM value, and a controlled input that resets its value on the next animation
|
||||||
|
frame.
|
||||||
|
|
||||||
|
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||||
|
ref's owning CDP target.
|
||||||
|
- `press --ref` must focus and dispatch every key event through that same target.
|
||||||
|
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||||
|
ownership cannot be determined uniquely.
|
||||||
|
- The remounting input must retain the value through its replacement node.
|
||||||
|
- The trusted input must be filled without synthetic `input` / `change` events.
|
||||||
|
- `type` must deliver ASCII key events and committed non-ASCII IME/software
|
||||||
|
keyboard insertion to the input proxy. Its DOM value remains empty while
|
||||||
|
`proxy output` retains the consumed text; the result reports
|
||||||
|
`verification=dispatched`, not a retained DOM value.
|
||||||
|
- `type` must reject an omitted target unless `focused=true` is explicit, and
|
||||||
|
focused mode must reject focus that is not owned by a text-entry target.
|
||||||
|
- The rejecting input must return `input_not_applied`, never a successful
|
||||||
|
requested-value length.
|
||||||
|
|
||||||
|
### `canvas-input.html`
|
||||||
|
|
||||||
|
This is the PR3 end-to-end fixture. Its cross-origin frame contains a
|
||||||
|
visual-only three-row Canvas. A screenshot-bound click on row 3 exposes and
|
||||||
|
focuses a standard textbox; ordinary `fill` must then write and read back its
|
||||||
|
real value through the textbox's owning target. The fixture deliberately does
|
||||||
|
not provide a Canvas-specific fill action.
|
||||||
|
|
||||||
### `dynamic.html`
|
### `dynamic.html`
|
||||||
|
|
||||||
Use the normal DOM buttons and re-observe after each action:
|
Use the normal DOM buttons and re-observe after each action:
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Canvas to standard input fixture</title>
|
||||||
|
<style>
|
||||||
|
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||||
|
iframe { width: 300px; height: 220px; border: 2px solid #5d6d7e; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Canvas to standard input fixture</h1>
|
||||||
|
<p>Select row 3 in the visual-only Canvas, then fill the exposed standard textbox.</p>
|
||||||
|
<iframe
|
||||||
|
title="cross-origin Canvas editor"
|
||||||
|
src="http://localhost:4173/frame.html?label=canvas-input-grid&interactive=1&editor=1"
|
||||||
|
></iframe>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -19,6 +19,18 @@
|
|||||||
if (params.get("canvas") === "0") canvas.remove();
|
if (params.get("canvas") === "0") canvas.remove();
|
||||||
else {
|
else {
|
||||||
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
|
||||||
|
if (params.get("editor") === "1") {
|
||||||
|
const context = canvas.getContext("2d");
|
||||||
|
context.font = "14px system-ui, sans-serif";
|
||||||
|
context.textBaseline = "middle";
|
||||||
|
context.strokeStyle = "#1e8449";
|
||||||
|
context.fillStyle = "#145a32";
|
||||||
|
for (let row = 0; row < 3; row += 1) {
|
||||||
|
const top = row * 30;
|
||||||
|
context.strokeRect(0.5, top + 0.5, 219, 29);
|
||||||
|
context.fillText(`Row ${row + 1}`, 12, top + 15);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (params.get("interactive") === "1") {
|
if (params.get("interactive") === "1") {
|
||||||
const status = document.createElement("p");
|
const status = document.createElement("p");
|
||||||
status.id = "pointer-status";
|
status.id = "pointer-status";
|
||||||
@@ -39,6 +51,22 @@
|
|||||||
canvas.addEventListener("click", () => {
|
canvas.addEventListener("click", () => {
|
||||||
status.textContent += "; click received";
|
status.textContent += "; click received";
|
||||||
});
|
});
|
||||||
|
if (params.get("editor") === "1") {
|
||||||
|
canvas.addEventListener("click", (event) => {
|
||||||
|
const rect = canvas.getBoundingClientRect();
|
||||||
|
const canvasY = ((event.clientY - rect.top) / rect.height) * canvas.height;
|
||||||
|
const row = Math.min(2, Math.max(0, Math.floor(canvasY / 30)));
|
||||||
|
status.textContent += `; selected row=${row + 1}`;
|
||||||
|
document.querySelector("#canvas-cell-editor")?.remove();
|
||||||
|
if (row !== 2) return;
|
||||||
|
const editor = document.createElement("input");
|
||||||
|
editor.id = "canvas-cell-editor";
|
||||||
|
editor.setAttribute("aria-label", "selected canvas cell editor");
|
||||||
|
editor.placeholder = "Type into selected row 3";
|
||||||
|
canvas.insertAdjacentElement("afterend", editor);
|
||||||
|
editor.focus();
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (params.get("nested") === "1") {
|
if (params.get("nested") === "1") {
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<style>
|
||||||
|
body { margin: 0; padding: 12px; font: 14px system-ui, sans-serif; }
|
||||||
|
label, textarea, input, [contenteditable] { display: block; margin: 8px 0; }
|
||||||
|
input, textarea, [contenteditable] { width: 240px; padding: 6px; border: 1px solid #85929e; }
|
||||||
|
iframe { width: 280px; height: 190px; border: 2px solid #85929e; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<label>Frame input <input aria-label="frame input" /></label>
|
||||||
|
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||||
|
<div contenteditable="true" aria-label="frame editor"></div>
|
||||||
|
<label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
|
||||||
|
<label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
|
||||||
|
<label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
|
||||||
|
<output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
|
||||||
|
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||||
|
<script>
|
||||||
|
const params = new URLSearchParams(location.search);
|
||||||
|
const remountHost = document.querySelector("#remount-host");
|
||||||
|
const armRemount = (input) => {
|
||||||
|
input.addEventListener("input", () => {
|
||||||
|
const replacement = document.createElement("input");
|
||||||
|
replacement.setAttribute("aria-label", "remounting input");
|
||||||
|
replacement.value = input.value;
|
||||||
|
armRemount(replacement);
|
||||||
|
input.replaceWith(replacement);
|
||||||
|
replacement.focus();
|
||||||
|
}, { once: true });
|
||||||
|
};
|
||||||
|
armRemount(remountHost.querySelector("input"));
|
||||||
|
const trusted = document.querySelector("#trusted");
|
||||||
|
trusted.addEventListener("input", (event) => {
|
||||||
|
if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
|
||||||
|
});
|
||||||
|
const proxy = document.querySelector("#proxy");
|
||||||
|
const proxyOutput = document.querySelector("#proxy-output");
|
||||||
|
let proxyValue = "";
|
||||||
|
proxy.addEventListener("input", (event) => {
|
||||||
|
if (event.isComposing) return;
|
||||||
|
proxyValue += event.data ?? proxy.value;
|
||||||
|
proxy.value = "";
|
||||||
|
proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
|
||||||
|
});
|
||||||
|
const rejecting = document.querySelector("#rejecting");
|
||||||
|
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||||
|
if (params.get("nested") === "1") {
|
||||||
|
const iframe = document.createElement("iframe");
|
||||||
|
iframe.title = "nested input frame";
|
||||||
|
iframe.src = "input-frame.html";
|
||||||
|
document.body.append(iframe);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>Frame input routing matrix</title>
|
||||||
|
<style>
|
||||||
|
body { padding: 16px; font: 14px system-ui, sans-serif; }
|
||||||
|
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
|
||||||
|
iframe { width: 300px; height: 430px; border: 2px solid #5d6d7e; }
|
||||||
|
input { padding: 6px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Frame input routing matrix</h1>
|
||||||
|
<label>Top input <input aria-label="top input" /></label>
|
||||||
|
<div class="frames">
|
||||||
|
<iframe title="same-origin inputs" src="input-frame.html"></iframe>
|
||||||
|
<iframe title="cross-origin inputs" src="http://localhost:4173/input-frame.html"></iframe>
|
||||||
|
<iframe title="nested inputs" src="input-frame.html?nested=1"></iframe>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
|||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||||
bsk observe --session <id> → again after navigation / DOM change
|
bsk observe --session <id> → again after navigation / DOM change
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Choosing `fill`, `type`, and `press`
|
||||||
|
|
||||||
|
Choose by the intended postcondition, not by whether observation happens to
|
||||||
|
render `textbox [empty]`:
|
||||||
|
|
||||||
|
- `fill` sets a field's retained value. Use it when success means the target
|
||||||
|
value must equal the requested text; it replaces existing content by default.
|
||||||
|
- `type` inserts text at the active caret/selection in a live editing session.
|
||||||
|
Use it when the application must consume keyboard/IME text; it never clears
|
||||||
|
first and reports dispatch only.
|
||||||
|
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||||
|
Do not encode special keys inside `type --text`.
|
||||||
|
|
||||||
|
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||||
|
available, or pass `--focused` only after an immediately preceding action in
|
||||||
|
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||||
|
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||||
|
application may already have consumed the text before clearing its proxy DOM
|
||||||
|
value. Re-observe first. After `type`, verify the application output with one
|
||||||
|
observation or screenshot; `verification=dispatched` is not business success.
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
|||||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||||
```
|
```
|
||||||
|
|
||||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
|||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||||
|
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||||
|
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
|||||||
use bsk_protocol::Method;
|
use bsk_protocol::Method;
|
||||||
use bsk_protocol::tools::{
|
use bsk_protocol::tools::{
|
||||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||||
};
|
};
|
||||||
use clap::{Args, ValueEnum};
|
use clap::{Args, ValueEnum};
|
||||||
|
|
||||||
@@ -95,6 +95,27 @@ fn split_target(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn split_type_target(
|
||||||
|
positional: Option<String>,
|
||||||
|
explicit_ref: Option<String>,
|
||||||
|
explicit_selector: Option<String>,
|
||||||
|
focused: bool,
|
||||||
|
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||||
|
if focused {
|
||||||
|
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"pass a target or --focused, not both"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok((None, None));
|
||||||
|
}
|
||||||
|
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||||
|
CliError::Local(anyhow::anyhow!(
|
||||||
|
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// bsk click
|
// bsk click
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// bsk type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Args)]
|
||||||
|
#[group(id = "type_destination", required = true, multiple = false)]
|
||||||
|
pub struct TypeDestinationArgs {
|
||||||
|
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||||
|
#[arg(value_name = "TARGET")]
|
||||||
|
pub target: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long = "ref")]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long = "selector")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
|
||||||
|
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||||
|
#[arg(long)]
|
||||||
|
pub focused: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Args)]
|
||||||
|
pub struct TypeArgs {
|
||||||
|
#[command(flatten)]
|
||||||
|
pub destination: TypeDestinationArgs,
|
||||||
|
|
||||||
|
/// Text to dispatch through the live keyboard/IME editing session.
|
||||||
|
#[arg(long)]
|
||||||
|
pub text: String,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
pub session: String,
|
||||||
|
|
||||||
|
#[arg(long = "tab-id")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
|
||||||
|
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||||
|
pub timeout: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||||
|
let (ref_, selector) = split_type_target(
|
||||||
|
args.destination.target.clone(),
|
||||||
|
args.destination.ref_.clone(),
|
||||||
|
args.destination.selector.clone(),
|
||||||
|
args.destination.focused,
|
||||||
|
)?;
|
||||||
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
|
let params = TypeParams {
|
||||||
|
session_id: args.session.clone(),
|
||||||
|
text: args.text.clone(),
|
||||||
|
ref_,
|
||||||
|
selector,
|
||||||
|
focused: args.destination.focused,
|
||||||
|
tab_id: args.tab_id,
|
||||||
|
timeout_ms: Some(args.timeout),
|
||||||
|
};
|
||||||
|
let reply: TypeResult = call(
|
||||||
|
info.sock_path,
|
||||||
|
Method::ToolType,
|
||||||
|
params,
|
||||||
|
"type-1",
|
||||||
|
args.timeout,
|
||||||
|
)?;
|
||||||
|
match format {
|
||||||
|
Format::Json => {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&reply)
|
||||||
|
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Format::Human => {
|
||||||
|
let target = if args.destination.focused {
|
||||||
|
"focused".into()
|
||||||
|
} else {
|
||||||
|
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||||
|
};
|
||||||
|
println!(
|
||||||
|
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||||
|
reply.tab_id, reply.text_length
|
||||||
|
);
|
||||||
|
print_dialog_summaries(&reply.dialogs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// bsk press
|
// bsk press
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -659,6 +769,52 @@ mod tests {
|
|||||||
assert_eq!(parsed.args.session, "ohli");
|
assert_eq!(parsed.args.session, "ohli");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||||
|
use clap::Parser;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(name = "type")]
|
||||||
|
struct Wrapper {
|
||||||
|
#[command(flatten)]
|
||||||
|
args: TypeArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed =
|
||||||
|
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||||
|
.expect("type args should parse");
|
||||||
|
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||||
|
assert_eq!(parsed.args.text, "测试");
|
||||||
|
assert_eq!(parsed.args.session, "ohli");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||||
|
use clap::Parser;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(name = "type")]
|
||||||
|
struct Wrapper {
|
||||||
|
#[command(flatten)]
|
||||||
|
args: TypeArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed =
|
||||||
|
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||||
|
.expect("focused type args should parse");
|
||||||
|
assert!(parsed.args.destination.focused);
|
||||||
|
assert!(parsed.args.destination.target.is_none());
|
||||||
|
|
||||||
|
let focused = split_type_target(None, None, None, true).unwrap();
|
||||||
|
assert_eq!(focused, (None, None));
|
||||||
|
|
||||||
|
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||||
|
assert!(missing.to_string().contains("--focused"));
|
||||||
|
|
||||||
|
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||||
|
assert!(mixed.to_string().contains("not both"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
|||||||
use crate::cli::get_html::GetHtmlArgs;
|
use crate::cli::get_html::GetHtmlArgs;
|
||||||
use crate::cli::human_loop::RequestHelpArgs;
|
use crate::cli::human_loop::RequestHelpArgs;
|
||||||
use crate::cli::install_skill::InstallSkillArgs;
|
use crate::cli::install_skill::InstallSkillArgs;
|
||||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||||
use crate::cli::network::NetworkArgs;
|
use crate::cli::network::NetworkArgs;
|
||||||
use crate::cli::observe::ObserveArgs;
|
use crate::cli::observe::ObserveArgs;
|
||||||
@@ -171,6 +171,9 @@ pub enum Command {
|
|||||||
/// Fill an input / textarea / contenteditable.
|
/// Fill an input / textarea / contenteditable.
|
||||||
Fill(FillArgs),
|
Fill(FillArgs),
|
||||||
|
|
||||||
|
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||||
|
Type(TypeArgs),
|
||||||
|
|
||||||
/// Dispatch a keyboard key combo.
|
/// Dispatch a keyboard key combo.
|
||||||
Press(PressArgs),
|
Press(PressArgs),
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,8 @@ pub mod reason {
|
|||||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||||
|
pub const FOCUS_TARGET_UNRESOLVED: &str = "focus_target_unresolved";
|
||||||
|
pub const INPUT_NOT_APPLIED: &str = "input_not_applied";
|
||||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||||
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
pub const OPTION_NOT_FOUND: &str = "option_not_found";
|
||||||
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count";
|
||||||
@@ -234,6 +236,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
|||||||
),
|
),
|
||||||
exit_code: base.exit_code,
|
exit_code: base.exit_code,
|
||||||
},
|
},
|
||||||
|
(ErrorCode::CdpFailed, reason::FOCUS_TARGET_UNRESOLVED) => RenderInfo {
|
||||||
|
summary: "focused frame target could not be determined",
|
||||||
|
hint: Some(
|
||||||
|
"pass an explicit --ref or --selector so the key event can be routed safely",
|
||||||
|
),
|
||||||
|
exit_code: base.exit_code,
|
||||||
|
},
|
||||||
|
(ErrorCode::CdpFailed, reason::INPUT_NOT_APPLIED) => RenderInfo {
|
||||||
|
summary: "the page did not retain the requested input",
|
||||||
|
hint: Some(
|
||||||
|
"refresh the snapshot and retry; the page may be rejecting or resetting controlled input",
|
||||||
|
),
|
||||||
|
exit_code: base.exit_code,
|
||||||
|
},
|
||||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||||
summary: "snapshot ref does not support this operation",
|
summary: "snapshot ref does not support this operation",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
@@ -532,4 +548,20 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(info.hint.unwrap().contains("exactly one"));
|
assert!(info.hint.unwrap().contains("exactly one"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn focus_target_unresolved_overrides_cdp_copy() {
|
||||||
|
let data = serde_json::json!({ "reason": reason::FOCUS_TARGET_UNRESOLVED });
|
||||||
|
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||||
|
assert_eq!(info.summary, "focused frame target could not be determined");
|
||||||
|
assert!(info.hint.unwrap().contains("--ref"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_not_applied_overrides_cdp_copy() {
|
||||||
|
let data = serde_json::json!({ "reason": reason::INPUT_NOT_APPLIED });
|
||||||
|
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
|
||||||
|
assert_eq!(info.summary, "the page did not retain the requested input");
|
||||||
|
assert!(info.hint.unwrap().contains("rejecting"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
|||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolHover
|
| Method::ToolHover
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect
|
| Method::ToolSelect
|
||||||
| Method::ToolEvaluate
|
| Method::ToolEvaluate
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
|||||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||||
|
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
|
||||||
//! through the IPC + per-session queue + fake extension and assert
|
//! through the IPC + per-session queue + fake extension and assert
|
||||||
//! the wire shapes line up. The extension stub mirrors each method's
|
//! the wire shapes line up. The extension stub mirrors each method's
|
||||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
|||||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||||
|
WaitUntil,
|
||||||
};
|
};
|
||||||
use bsk_protocol::{
|
use bsk_protocol::{
|
||||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
|||||||
handle.shutdown().await;
|
handle.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn type_round_trips_dispatched_verification() {
|
||||||
|
let (handle, sock) = spawn_daemon().await;
|
||||||
|
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||||
|
let _ = do_handshake(&mut ws).await;
|
||||||
|
|
||||||
|
run_extension(ws, |req| {
|
||||||
|
assert_eq!(req.method, Method::ToolType);
|
||||||
|
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||||
|
assert_eq!(p.text, "测试");
|
||||||
|
assert!(!p.focused);
|
||||||
|
ResponseBody::Ok(
|
||||||
|
serde_json::to_value(TypeResult {
|
||||||
|
tab_id: 12,
|
||||||
|
used_ref: Some("e3".into()),
|
||||||
|
used_selector: None,
|
||||||
|
text_length: 2,
|
||||||
|
verification: TypeVerification::Dispatched,
|
||||||
|
dialogs: vec![],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
let session_id = ipc_session_start(&sock).await;
|
||||||
|
let result: TypeResult = ipc_tool_call(
|
||||||
|
&sock,
|
||||||
|
Method::ToolType,
|
||||||
|
TypeParams {
|
||||||
|
session_id,
|
||||||
|
text: "测试".into(),
|
||||||
|
ref_: Some("@e3".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("type ok");
|
||||||
|
assert_eq!(result.text_length, 2);
|
||||||
|
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||||
|
handle.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn press_round_trips_compound_key() {
|
async fn press_round_trips_compound_key() {
|
||||||
let (handle, sock) = spawn_daemon().await;
|
let (handle, sock) = spawn_daemon().await;
|
||||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
|||||||
| Method::ToolReload
|
| Method::ToolReload
|
||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||||
code: ErrorCode::CdpFailed,
|
code: ErrorCode::CdpFailed,
|
||||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
|||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||||
|
|
||||||
|
let type_result = ipc_tool_call::<_, Value>(
|
||||||
|
&sock,
|
||||||
|
Method::ToolType,
|
||||||
|
TypeParams {
|
||||||
|
session_id: session_id.clone(),
|
||||||
|
text: "hello".into(),
|
||||||
|
ref_: Some("@e1".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||||
|
|
||||||
let press = ipc_tool_call::<_, Value>(
|
let press = ipc_tool_call::<_, Value>(
|
||||||
&sock,
|
&sock,
|
||||||
Method::ToolPress,
|
Method::ToolPress,
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
"format": "int64"
|
"format": "int64"
|
||||||
},
|
},
|
||||||
"text": {
|
"text": {
|
||||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
|
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"truncated": {
|
"truncated": {
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "TypeParams",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"session_id",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"focused": {
|
||||||
|
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"ref": {
|
||||||
|
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"session_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"timeout_ms": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 1.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "TypeResult",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"tab_id",
|
||||||
|
"text_length",
|
||||||
|
"verification"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"dialogs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"text_length": {
|
||||||
|
"description": "UTF-16 code-unit length of text accepted for dispatch.",
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"used_ref": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"used_selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"verification": {
|
||||||
|
"$ref": "#/definitions/TypeVerification"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"JavaScriptDialogHandledAction": {
|
||||||
|
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"accepted",
|
||||||
|
"dismissed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"JavaScriptDialogInfo": {
|
||||||
|
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"handled",
|
||||||
|
"message",
|
||||||
|
"sequence",
|
||||||
|
"tab_id",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"default_prompt": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"handled": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||||
|
},
|
||||||
|
"has_browser_handler": {
|
||||||
|
"type": [
|
||||||
|
"boolean",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sequence": {
|
||||||
|
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint64",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogType"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"JavaScriptDialogType": {
|
||||||
|
"description": "Native JS dialog kind reported by CDP.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"alert",
|
||||||
|
"confirm",
|
||||||
|
"prompt",
|
||||||
|
"beforeunload"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"TypeVerification": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"dispatched"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -81,6 +81,8 @@ fn main() {
|
|||||||
dump!(HoverResult, "tool_hover_result");
|
dump!(HoverResult, "tool_hover_result");
|
||||||
dump!(FillParams, "tool_fill_params");
|
dump!(FillParams, "tool_fill_params");
|
||||||
dump!(FillResult, "tool_fill_result");
|
dump!(FillResult, "tool_fill_result");
|
||||||
|
dump!(TypeParams, "tool_type_params");
|
||||||
|
dump!(TypeResult, "tool_type_result");
|
||||||
dump!(PressParams, "tool_press_params");
|
dump!(PressParams, "tool_press_params");
|
||||||
dump!(PressResult, "tool_press_result");
|
dump!(PressResult, "tool_press_result");
|
||||||
dump!(SelectParams, "tool_select_params");
|
dump!(SelectParams, "tool_select_params");
|
||||||
|
|||||||
@@ -74,6 +74,8 @@ pub enum Method {
|
|||||||
ToolHover,
|
ToolHover,
|
||||||
#[serde(rename = "tool.fill")]
|
#[serde(rename = "tool.fill")]
|
||||||
ToolFill,
|
ToolFill,
|
||||||
|
#[serde(rename = "tool.type")]
|
||||||
|
ToolType,
|
||||||
#[serde(rename = "tool.press")]
|
#[serde(rename = "tool.press")]
|
||||||
ToolPress,
|
ToolPress,
|
||||||
#[serde(rename = "tool.select")]
|
#[serde(rename = "tool.select")]
|
||||||
@@ -154,6 +156,7 @@ impl Method {
|
|||||||
| Method::ToolReload
|
| Method::ToolReload
|
||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect
|
| Method::ToolSelect
|
||||||
| Method::ToolEvaluate
|
| Method::ToolEvaluate
|
||||||
@@ -246,6 +249,13 @@ mod tests {
|
|||||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_method_round_trips() {
|
||||||
|
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
|
||||||
|
assert_eq!(method, Method::ToolType);
|
||||||
|
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cancel_params_and_result_round_trip() {
|
fn cancel_params_and_result_round_trip() {
|
||||||
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
||||||
@@ -284,6 +294,7 @@ mod tests {
|
|||||||
assert!(Method::ToolReload.is_mutating());
|
assert!(Method::ToolReload.is_mutating());
|
||||||
assert!(Method::ToolClick.is_mutating());
|
assert!(Method::ToolClick.is_mutating());
|
||||||
assert!(Method::ToolFill.is_mutating());
|
assert!(Method::ToolFill.is_mutating());
|
||||||
|
assert!(Method::ToolType.is_mutating());
|
||||||
assert!(Method::ToolPress.is_mutating());
|
assert!(Method::ToolPress.is_mutating());
|
||||||
assert!(Method::ToolSelect.is_mutating());
|
assert!(Method::ToolSelect.is_mutating());
|
||||||
assert!(Method::ToolEvaluate.is_mutating());
|
assert!(Method::ToolEvaluate.is_mutating());
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
|
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
|
||||||
//! `tool.select`).
|
//! `tool.press`, `tool.select`).
|
||||||
//!
|
//!
|
||||||
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
||||||
//! normalised against the session's RefStore) **or** a CSS selector
|
//! normalised against the session's RefStore) **or** a CSS selector
|
||||||
@@ -188,6 +188,60 @@ pub struct FillResult {
|
|||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct TypeParams {
|
||||||
|
pub session_id: String,
|
||||||
|
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
|
||||||
|
/// does not require the focused DOM editable to retain this text.
|
||||||
|
pub text: String,
|
||||||
|
/// Target to focus before dispatching text. Exactly one targeting mode
|
||||||
|
/// must be selected: `ref` / `selector`, or `focused=true`.
|
||||||
|
#[serde(
|
||||||
|
rename = "ref",
|
||||||
|
alias = "ref_",
|
||||||
|
default,
|
||||||
|
skip_serializing_if = "Option::is_none"
|
||||||
|
)]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
/// Explicitly dispatch to the unique focused text-entry target instead
|
||||||
|
/// of focusing a ref or selector.
|
||||||
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||||
|
pub focused: bool,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 1))]
|
||||||
|
pub timeout_ms: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TypeVerification {
|
||||||
|
/// Chromium accepted the complete keyboard/IME dispatch transaction.
|
||||||
|
/// The caller must observe the application output to verify business state.
|
||||||
|
Dispatched,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct TypeResult {
|
||||||
|
pub tab_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_ref: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_selector: Option<String>,
|
||||||
|
/// UTF-16 code-unit length of text accepted for dispatch.
|
||||||
|
pub text_length: u32,
|
||||||
|
pub verification: TypeVerification,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// press
|
// press
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -351,6 +405,50 @@ mod tests {
|
|||||||
assert_eq!(v, json!("meta"));
|
assert_eq!(v, json!("meta"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_payloads_distinguish_dispatch_from_value_verification() {
|
||||||
|
let params = TypeParams {
|
||||||
|
session_id: "abcd".into(),
|
||||||
|
text: "测试".into(),
|
||||||
|
ref_: Some("@e3".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: Some(4),
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
};
|
||||||
|
let encoded = serde_json::to_value(¶ms).unwrap();
|
||||||
|
assert_eq!(encoded["ref"], json!("@e3"));
|
||||||
|
assert_eq!(encoded["text"], json!("测试"));
|
||||||
|
assert!(encoded.get("focused").is_none());
|
||||||
|
|
||||||
|
let focused = TypeParams {
|
||||||
|
session_id: "abcd".into(),
|
||||||
|
text: "hello".into(),
|
||||||
|
ref_: None,
|
||||||
|
selector: None,
|
||||||
|
focused: true,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: None,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(focused).unwrap()["focused"],
|
||||||
|
json!(true)
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = TypeResult {
|
||||||
|
tab_id: 4,
|
||||||
|
used_ref: Some("e3".into()),
|
||||||
|
used_selector: None,
|
||||||
|
text_length: 2,
|
||||||
|
verification: TypeVerification::Dispatched,
|
||||||
|
dialogs: vec![],
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(result).unwrap()["verification"],
|
||||||
|
json!("dispatched")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn press_result_round_trips() {
|
fn press_result_round_trips() {
|
||||||
let r = PressResult {
|
let r = PressResult {
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
|
|||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct SnapshotResult {
|
pub struct SnapshotResult {
|
||||||
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
||||||
/// agent can copy them into subsequent `tool.click` / `tool.fill`
|
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
|
||||||
/// selectors.
|
/// `tool.type` targets.
|
||||||
pub text: String,
|
pub text: String,
|
||||||
/// Number of `@e<N>` refs registered for this session by this
|
/// Number of `@e<N>` refs registered for this session by this
|
||||||
/// snapshot. Equivalent to the size of the new ref-store map.
|
/// snapshot. Equivalent to the size of the new ref-store map.
|
||||||
|
|||||||
Reference in new issue
Block a user