mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): add type primitive
This commit is contained in:
Ljy-0827
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fillEditable } from "../editable-input";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
interface FakeOptions {
|
||||
initialValue?: string;
|
||||
finalValue: string;
|
||||
remount?: boolean;
|
||||
}
|
||||
|
||||
function editableState(value: string, connected = true, focused = true) {
|
||||
return {
|
||||
result: {
|
||||
value: { supported: true, connected, focused, value, kind: "input" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeEditableCdp(options: FakeOptions) {
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
let inserted = false;
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||
sent.push({ method, params });
|
||||
if (method === "Input.insertText") {
|
||||
inserted = true;
|
||||
return {} as T;
|
||||
}
|
||||
if (method === "Input.dispatchKeyEvent") {
|
||||
if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
|
||||
return {} as T;
|
||||
}
|
||||
if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
|
||||
|
||||
const call = params as { objectId?: string; functionDeclaration?: string };
|
||||
const fn = call.functionDeclaration ?? "";
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } } as T;
|
||||
}
|
||||
if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
|
||||
if (fn.includes("bsk-editable-live")) {
|
||||
return {
|
||||
result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
|
||||
} as T;
|
||||
}
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
if (call.objectId === "replacement") return editableState(options.finalValue) as T;
|
||||
if (options.remount && inserted) {
|
||||
return editableState(options.finalValue, false, false) as T;
|
||||
}
|
||||
return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
|
||||
}
|
||||
throw new Error("unexpected Runtime.callFunctionOn function");
|
||||
});
|
||||
return { cdp: { send } as CdpRunner, sent };
|
||||
}
|
||||
|
||||
describe("fillEditable", () => {
|
||||
it("replaces text through the browser input path without synthetic events or value setters", async () => {
|
||||
const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "新值", replacedNode: false });
|
||||
expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
|
||||
const declarations = fake.sent
|
||||
.map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
|
||||
.join("\n");
|
||||
expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||
});
|
||||
|
||||
it("reacquires the focused editable after a framework remount", async () => {
|
||||
const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "persisted", replacedNode: true });
|
||||
});
|
||||
|
||||
it("reports a page rollback after the bounded settlement", async () => {
|
||||
const fake = makeEditableCdp({ finalValue: "" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: {
|
||||
reason: "input_not_applied",
|
||||
expected_value_length: 8,
|
||||
actual_value_length: 0,
|
||||
phase: "readback",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("clears to an empty value with a browser Backspace edit", async () => {
|
||||
const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
|
||||
|
||||
const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
|
||||
|
||||
expect(result).toMatchObject({ value: "" });
|
||||
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
handleType,
|
||||
modifiersBitfield,
|
||||
parseKeySpec,
|
||||
resolveKeyDescriptor,
|
||||
@@ -71,8 +72,21 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
function fillRuntimeReadback(value: string) {
|
||||
return (params: unknown) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value } } };
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value,
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
};
|
||||
@@ -770,13 +784,13 @@ describe("handleFill", () => {
|
||||
expect(res.used_ref).toBe("e1");
|
||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||
expect(insert?.params).toEqual({ text: "hello" });
|
||||
// clear_before defaults to true → callFunctionOn invoked once to
|
||||
// clear, once to fire input/change after typing.
|
||||
// clear_before defaults to true, so the edit transaction prepares a
|
||||
// replacement selection and performs a settled live-value readback.
|
||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("clear_before=false skips the wipe call", async () => {
|
||||
it("clear_before=false skips replacement selection", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||
@@ -789,12 +803,20 @@ describe("handleFill", () => {
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||
"Runtime.callFunctionOn": (p) => {
|
||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (fn.includes("bsk-input-readback")) {
|
||||
return { result: { value: { supported: true, value: "existingx" } } };
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "existingx",
|
||||
kind: "textarea",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
// No "this.value = ''" clearing on a no-clear path; only the
|
||||
// post-input dispatchEvent.
|
||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
||||
expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||
return { result: { type: "undefined" } };
|
||||
},
|
||||
"Input.insertText": () => ({}),
|
||||
@@ -870,9 +892,23 @@ describe("handleFill", () => {
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
return fn.includes("bsk-input-readback")
|
||||
? { result: { value: { supported: true, value: "中文输入" } } }
|
||||
: { result: { type: "undefined" } };
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "中文输入",
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
}
|
||||
throw new Error(`unexpected child call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
@@ -916,7 +952,7 @@ describe("handleFill", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
||||
it("stops before Input.insertText when abort fires after preparing replacement", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 555, { tabId: 4 });
|
||||
@@ -928,8 +964,25 @@ describe("handleFill", () => {
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.focus": () => ({}),
|
||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||
"Runtime.callFunctionOn": () => {
|
||||
abort.abort();
|
||||
"Runtime.callFunctionOn": (params) => {
|
||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||
if (fn.includes("bsk-editable-state")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
supported: true,
|
||||
connected: true,
|
||||
focused: true,
|
||||
value: "",
|
||||
kind: "input",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (fn.includes("bsk-editable-selection")) {
|
||||
abort.abort();
|
||||
return { result: { value: { selected: true } } };
|
||||
}
|
||||
return { result: { type: "undefined" } };
|
||||
},
|
||||
"Input.insertText": () => ({}),
|
||||
@@ -972,6 +1025,204 @@ describe("handleFill", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleType", () => {
|
||||
it("requires an explicit target mode instead of silently using focus", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const missing = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(missing).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: expect.stringContaining("focused=true"),
|
||||
});
|
||||
|
||||
const mixed = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", focused: true, text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(mixed).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: expect.stringContaining("not both"),
|
||||
});
|
||||
expect(fake.sent).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
|
||||
childCalls.push({ method, params });
|
||||
if (method === "DOM.describeNode") {
|
||||
return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "测试" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
used_ref: "e1",
|
||||
text_length: 2,
|
||||
verification: "dispatched",
|
||||
});
|
||||
expect(childCalls.map((call) => call.method)).toEqual([
|
||||
"DOM.describeNode",
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.focus",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.insertText",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
|
||||
});
|
||||
|
||||
it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const childCalls: string[] = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||
childCalls.push(method);
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
|
||||
};
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", focused: true, text: "A" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||
expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
|
||||
expect(childCalls).toEqual([
|
||||
"Runtime.evaluate",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses focused mode when focus is not owned by a text-entry target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({
|
||||
result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", focused: true, text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "focus_target_unresolved", candidate_count: 0 },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects explicit non-text input controls", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.describeNode": () => ({
|
||||
node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "invalid_params",
|
||||
data: { reason: "target_not_fillable" },
|
||||
});
|
||||
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a Surface ref instead of treating it as a text target", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 77, {
|
||||
tabId: 4,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
});
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const result = await handleType(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseKeySpec", () => {
|
||||
it("splits compound expressions into modifiers + base key", () => {
|
||||
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { dispatchTextInput } from "../text-input";
|
||||
|
||||
function makeCdp(onSend?: (method: string, params?: object) => void) {
|
||||
const sent: Array<{ method: string; params?: object }> = [];
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||
sent.push({ method, params });
|
||||
onSend?.(method, params);
|
||||
return {} as T;
|
||||
});
|
||||
return { cdp: { send } as CdpRunner, sent };
|
||||
}
|
||||
|
||||
describe("dispatchTextInput", () => {
|
||||
it("types ASCII through complete keydown, char, and keyup sequences", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "Ab");
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(
|
||||
fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
|
||||
).toEqual([
|
||||
"Input.dispatchKeyEvent:rawKeyDown",
|
||||
"Input.dispatchKeyEvent:char",
|
||||
"Input.dispatchKeyEvent:keyUp",
|
||||
"Input.dispatchKeyEvent:rawKeyDown",
|
||||
"Input.dispatchKeyEvent:char",
|
||||
"Input.dispatchKeyEvent:keyUp",
|
||||
]);
|
||||
expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
|
||||
expect(fake.sent[1].params).toMatchObject({ text: "A" });
|
||||
});
|
||||
|
||||
it("types non-ASCII through the committed IME insertion path", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试");
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(fake.sent).toEqual([
|
||||
{
|
||||
method: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "rawKeyDown",
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
},
|
||||
},
|
||||
{ method: "Input.insertText", params: { text: "测试" } },
|
||||
{
|
||||
method: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves text order across keyboard and IME segments", async () => {
|
||||
const fake = makeCdp();
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
|
||||
|
||||
expect(result).toBeNull();
|
||||
const meaningful = fake.sent.flatMap((call) => {
|
||||
if (call.method === "Input.insertText") {
|
||||
return [`ime:${(call.params as { text: string }).text}`];
|
||||
}
|
||||
if (
|
||||
call.method === "Input.dispatchKeyEvent" &&
|
||||
(call.params as { type?: string }).type === "char"
|
||||
) {
|
||||
return [`key:${(call.params as { text: string }).text}`];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
|
||||
});
|
||||
|
||||
it("reports cancellation observed after an accepted IME insertion", async () => {
|
||||
const abort = new AbortController();
|
||||
const fake = makeCdp((method) => {
|
||||
if (method === "Input.insertText") abort.abort();
|
||||
});
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.insertText",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||
});
|
||||
|
||||
it("releases the IME process key without committing when cancellation follows keydown", async () => {
|
||||
const abort = new AbortController();
|
||||
const fake = makeCdp((method, params) => {
|
||||
if (
|
||||
method === "Input.dispatchKeyEvent" &&
|
||||
(params as { type?: string }).type === "rawKeyDown"
|
||||
) {
|
||||
abort.abort();
|
||||
}
|
||||
});
|
||||
|
||||
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
|
||||
expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||
});
|
||||
});
|
||||
@@ -28,6 +28,7 @@ import type {
|
||||
ScreenshotParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
TypeParams,
|
||||
WaitForNavigationParams,
|
||||
} from "@/transport/types";
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
@@ -35,7 +36,14 @@ import { handleConsole } from "./console";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleClick,
|
||||
handleFill,
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
handleType,
|
||||
} from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
handleNavigateBack,
|
||||
@@ -491,6 +499,17 @@ export class ToolDispatcher {
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.type":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TypeParams,
|
||||
() =>
|
||||
handleType(
|
||||
this.sessions,
|
||||
req.params as TypeParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.press":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as PressParams,
|
||||
@@ -717,6 +736,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.fill":
|
||||
case "tool.type":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.evaluate":
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
// Browser-native editing transaction for input, textarea, and contenteditable
|
||||
// targets. The CDP target/session is selected by interaction.ts; this module
|
||||
// owns the shorter-lived editable identity inside that target.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
interface EditableState {
|
||||
supported: boolean;
|
||||
connected: boolean;
|
||||
focused: boolean;
|
||||
value: string;
|
||||
kind: "input" | "textarea" | "contenteditable" | "unsupported";
|
||||
}
|
||||
|
||||
export interface EditableFillResult {
|
||||
value: string;
|
||||
replacedNode: boolean;
|
||||
}
|
||||
|
||||
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
|
||||
}
|
||||
|
||||
async function inspectEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
objectId: string,
|
||||
): Promise<EditableState> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { value?: EditableState };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-state */
|
||||
const doc = this.ownerDocument;
|
||||
const focused = !!doc && doc.activeElement === this;
|
||||
if (this instanceof HTMLInputElement) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.value, kind: 'input' };
|
||||
}
|
||||
if (this instanceof HTMLTextAreaElement) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.value, kind: 'textarea' };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
return { supported: true, connected: this.isConnected, focused,
|
||||
value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
|
||||
}
|
||||
return { supported: false, connected: this.isConnected === true, focused,
|
||||
value: '', kind: 'unsupported' };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const state = reply.result?.value;
|
||||
if (!state || typeof state.value !== "string") {
|
||||
throw new Error("editable state probe returned no value");
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the active editable from the anchor's own document. Using the
|
||||
* anchor's execution context also works for same-process nested frames, where
|
||||
* a target-level Runtime.evaluate would otherwise see the top document.
|
||||
*/
|
||||
async function focusedEditableFromAnchor(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<string | null> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { objectId?: string; subtype?: string };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId: anchorObjectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-live */
|
||||
const active = this.ownerDocument?.activeElement;
|
||||
if (!active) return null;
|
||||
const editable = active instanceof HTMLInputElement ||
|
||||
active instanceof HTMLTextAreaElement || active.isContentEditable;
|
||||
return editable ? active : null;
|
||||
}`,
|
||||
returnByValue: false,
|
||||
});
|
||||
if (reply.result?.subtype === "null") return null;
|
||||
return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
|
||||
}
|
||||
|
||||
async function resolveLiveEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
|
||||
const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
|
||||
if (anchor.supported && anchor.connected && anchor.focused) {
|
||||
return { objectId: anchorObjectId, state: anchor, replaced: false };
|
||||
}
|
||||
|
||||
const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
|
||||
if (!focusedObjectId) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the target editable lost focus or detached before input",
|
||||
{ phase: "resolve_live_editable" },
|
||||
);
|
||||
}
|
||||
const focused = await inspectEditable(cdp, tabId, focusedObjectId);
|
||||
if (!focused.supported || !focused.connected || !focused.focused) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the replacement editable was not live and focused",
|
||||
{ phase: "resolve_live_editable" },
|
||||
);
|
||||
}
|
||||
return { objectId: focusedObjectId, state: focused, replaced: true };
|
||||
}
|
||||
|
||||
async function selectEditableContents(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
objectId: string,
|
||||
): Promise<void> {
|
||||
const reply = await cdp.send<{
|
||||
result?: { value?: { selected?: boolean } };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-selection */
|
||||
if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
|
||||
return { selected: false };
|
||||
}
|
||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||
try {
|
||||
this.setSelectionRange(0, this.value.length);
|
||||
} catch {
|
||||
if (typeof this.select !== 'function') return { selected: false };
|
||||
this.select();
|
||||
}
|
||||
return { selected: true };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
const selection = this.ownerDocument.getSelection();
|
||||
if (!selection) return { selected: false };
|
||||
const range = this.ownerDocument.createRange();
|
||||
range.selectNodeContents(this);
|
||||
selection.removeAllRanges();
|
||||
selection.addRange(range);
|
||||
return { selected: true };
|
||||
}
|
||||
return { selected: false };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (reply.result?.value?.selected !== true) {
|
||||
throw new Error("could not select the live editable contents");
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForEditableSettlement(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
): Promise<void> {
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId: anchorObjectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-editable-sync */
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, 100);
|
||||
const view = this.ownerDocument?.defaultView;
|
||||
if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
|
||||
view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
|
||||
});
|
||||
}`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
key: "Backspace",
|
||||
code: "Backspace",
|
||||
windowsVirtualKeyCode: 8,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: "Backspace",
|
||||
code: "Backspace",
|
||||
windowsVirtualKeyCode: 8,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace or append text through Chromium's editing pipeline. No value setter
|
||||
* or synthetic DOM events are used: controlled components observe the same
|
||||
* beforeinput/input path as a real browser edit. The live editable is resolved
|
||||
* again after settlement so a framework may remount it without producing a
|
||||
* false failure or a stale-object success.
|
||||
*/
|
||||
export async function fillEditable(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
anchorObjectId: string,
|
||||
value: string,
|
||||
clearBefore: boolean,
|
||||
signal?: AbortSignal,
|
||||
): Promise<EditableFillResult | RpcError> {
|
||||
try {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
|
||||
if ("code" in initial) return initial;
|
||||
let replacedNode = initial.replaced;
|
||||
|
||||
if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
|
||||
const abortedAfterSelection = cancelled(signal);
|
||||
if (abortedAfterSelection) return abortedAfterSelection;
|
||||
|
||||
if (value.length > 0) {
|
||||
await cdp.send(tabId, "Input.insertText", { text: value });
|
||||
} else if (clearBefore) {
|
||||
await deleteSelection(cdp, tabId);
|
||||
}
|
||||
|
||||
const abortedAfterInput = cancelled(signal);
|
||||
if (abortedAfterInput) return abortedAfterInput;
|
||||
await waitForEditableSettlement(cdp, tabId, initial.objectId);
|
||||
|
||||
const abortedAfterSettlement = cancelled(signal);
|
||||
if (abortedAfterSettlement) return abortedAfterSettlement;
|
||||
|
||||
let finalObjectId = initial.objectId;
|
||||
let finalState = await inspectEditable(cdp, tabId, finalObjectId);
|
||||
if (!finalState.connected || !finalState.focused) {
|
||||
const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
|
||||
if ("code" in live) return live;
|
||||
finalObjectId = live.objectId;
|
||||
finalState = live.state;
|
||||
replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
|
||||
}
|
||||
|
||||
const applied = clearBefore
|
||||
? finalState.value === value
|
||||
: value.length === 0 || finalState.value.includes(value);
|
||||
if (!finalState.supported || !applied) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the page did not retain the requested input value",
|
||||
{
|
||||
expected_value_length: value.length,
|
||||
actual_value_length: finalState.value.length,
|
||||
phase: "readback",
|
||||
editable_replaced: replacedNode,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { value: finalState.value, replacedNode };
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
{ phase: "edit_transaction" },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
|
||||
// `tool.select`.
|
||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
|
||||
// `tool.press`, and `tool.select`.
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
@@ -27,8 +27,11 @@ import type {
|
||||
RpcError,
|
||||
SelectParams,
|
||||
SelectResult,
|
||||
TypeParams,
|
||||
TypeResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { fillEditable } from "./editable-input";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
@@ -45,6 +48,7 @@ import {
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||
import { dispatchTextInput } from "./text-input";
|
||||
|
||||
export interface InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
@@ -244,18 +248,20 @@ function inputContextForNode(
|
||||
interface FocusProbe {
|
||||
hasFocus: boolean;
|
||||
activeTag: string;
|
||||
textEntry: boolean;
|
||||
}
|
||||
|
||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||
async function resolveFocusedInputContext(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
mode: "press" | "type",
|
||||
): Promise<InputDispatchContext | RpcError> {
|
||||
if (!cdp.getFrameGraph) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
"press cannot determine the focused frame target",
|
||||
`${mode} cannot determine the focused frame target`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -284,9 +290,21 @@ async function resolveFocusedInputContext(
|
||||
expression: `(() => {
|
||||
/* bsk-focus-target-probe */
|
||||
const active = document.activeElement;
|
||||
const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
|
||||
const nonTextInputTypes = new Set([
|
||||
'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
|
||||
'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
|
||||
'time', 'week'
|
||||
]);
|
||||
const role = active instanceof Element ? active.getAttribute('role') : null;
|
||||
return {
|
||||
hasFocus: document.hasFocus(),
|
||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
|
||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
|
||||
textEntry:
|
||||
(active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
|
||||
active instanceof HTMLTextAreaElement ||
|
||||
(active instanceof HTMLElement && active.isContentEditable) ||
|
||||
role === 'textbox' || role === 'searchbox'
|
||||
};
|
||||
})()`,
|
||||
returnByValue: true,
|
||||
@@ -302,16 +320,17 @@ async function resolveFocusedInputContext(
|
||||
// A top document whose active element is an iframe delegates focus to
|
||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||
// target while same-process frames continue to use the shared root target.
|
||||
const owners = focused.filter(
|
||||
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
|
||||
);
|
||||
const owners = focused.filter(({ probe }) => {
|
||||
if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
|
||||
return mode !== "type" || probe.textEntry;
|
||||
});
|
||||
if (owners.length !== 1) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"focus_target_unresolved",
|
||||
owners.length === 0
|
||||
? "press found no unique focused frame target"
|
||||
: "press found multiple focused frame targets",
|
||||
? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
|
||||
: `${mode} found multiple focused frame targets`,
|
||||
{ candidate_count: owners.length },
|
||||
);
|
||||
}
|
||||
@@ -682,6 +701,47 @@ function isFillable(node: DescribedNode): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** `type` also supports custom text-entry widgets that expose an ARIA role. */
|
||||
function isTypeTarget(node: DescribedNode): boolean {
|
||||
const tag = (node.nodeName ?? "").toUpperCase();
|
||||
const attrs = node.attributes ?? [];
|
||||
const attributes = new Map<string, string>();
|
||||
for (let i = 0; i + 1 < attrs.length; i += 2) {
|
||||
attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
|
||||
}
|
||||
if (tag === "TEXTAREA") return true;
|
||||
if (tag === "INPUT") {
|
||||
const type = attributes.get("type") ?? "text";
|
||||
return !new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"color",
|
||||
"date",
|
||||
"datetime-local",
|
||||
"file",
|
||||
"hidden",
|
||||
"image",
|
||||
"month",
|
||||
"radio",
|
||||
"range",
|
||||
"reset",
|
||||
"submit",
|
||||
"time",
|
||||
"week",
|
||||
]).has(type);
|
||||
}
|
||||
const contentEditable = attributes.get("contenteditable");
|
||||
if (
|
||||
contentEditable === "" ||
|
||||
contentEditable === "true" ||
|
||||
contentEditable === "plaintext-only"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const role = attributes.get("role");
|
||||
return role === "textbox" || role === "searchbox";
|
||||
}
|
||||
|
||||
type SelectMutationResult =
|
||||
| {
|
||||
ok: true;
|
||||
@@ -767,125 +827,114 @@ export async function handleFill(
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
|
||||
try {
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
else {
|
||||
const proto = this instanceof HTMLTextAreaElement
|
||||
? HTMLTextAreaElement.prototype
|
||||
: HTMLInputElement.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
||||
else this.value = '';
|
||||
}
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
||||
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
// Give controlled inputs one bounded browser-render turn to accept or
|
||||
// reject the dispatched value before reporting success.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-sync */
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const finish = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(finish, 100);
|
||||
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
|
||||
requestAnimationFrame(() => requestAnimationFrame(finish));
|
||||
});
|
||||
}`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
let actualValue: string;
|
||||
try {
|
||||
const readback = await nodeCdp.send<{
|
||||
result?: { value?: { supported?: boolean; value?: string } };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
/* bsk-input-readback */
|
||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||
return { supported: true, value: this.value };
|
||||
}
|
||||
if (this.isContentEditable) {
|
||||
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
|
||||
}
|
||||
return { supported: false, value: '' };
|
||||
}`,
|
||||
returnByValue: true,
|
||||
});
|
||||
const state = readback.result?.value;
|
||||
actualValue = typeof state?.value === "string" ? state.value : "";
|
||||
const applied =
|
||||
state?.supported === true &&
|
||||
(clearBefore
|
||||
? actualValue === params.value
|
||||
: params.value.length === 0 || actualValue.includes(params.value));
|
||||
if (!applied) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
"the page did not retain the requested input value",
|
||||
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"input_not_applied",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
);
|
||||
}
|
||||
const fillResult = await fillEditable(
|
||||
nodeCdp,
|
||||
target.tabId,
|
||||
objectId,
|
||||
params.value,
|
||||
clearBefore,
|
||||
deps.signal,
|
||||
);
|
||||
if (isRpcError(fillResult)) return fillResult;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
value_length: actualValue.length,
|
||||
value_length: fillResult.value.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function handleType(
|
||||
manager: SessionManager,
|
||||
params: TypeParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<TypeResult | RpcError> {
|
||||
if (!params || typeof params.text !== "string") {
|
||||
return { code: "invalid_params", message: "type requires a text string" };
|
||||
}
|
||||
if (params.focused !== undefined && typeof params.focused !== "boolean") {
|
||||
return { code: "invalid_params", message: "type focused must be a boolean" };
|
||||
}
|
||||
const hasExplicitTarget =
|
||||
(typeof params.ref === "string" && params.ref.length > 0) ||
|
||||
(typeof params.selector === "string" && params.selector.length > 0);
|
||||
if (params.focused === true && hasExplicitTarget) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "type: pass a ref/selector or focused=true, not both",
|
||||
};
|
||||
}
|
||||
if (params.focused !== true && !hasExplicitTarget) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: "type requires a ref/selector or focused=true",
|
||||
};
|
||||
}
|
||||
const ctxOrErr = lookupSession(manager, params, "type");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "type");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
let input: InputDispatchContext | RpcError;
|
||||
if (hasExplicitTarget) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
|
||||
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||
if (isRpcError(explicitInput)) return explicitInput;
|
||||
input = explicitInput;
|
||||
try {
|
||||
const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: explicitInput.backendNodeId },
|
||||
);
|
||||
if (!described.node || !isTypeTarget(described.node)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"target_not_fillable",
|
||||
`element ${described.node?.nodeName ?? "?"} not text-editable`,
|
||||
);
|
||||
}
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
explicitInput.target,
|
||||
explicitInput.backendNodeId,
|
||||
explicitInput.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||
backendNodeId: explicitInput.backendNodeId,
|
||||
});
|
||||
} catch (err) {
|
||||
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
|
||||
if (dispatchError) return dispatchError;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: input.usedRef,
|
||||
used_selector: input.usedSelector,
|
||||
text_length: params.text.length,
|
||||
verification: "dispatched",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1090,7 +1139,7 @@ export async function handlePress(
|
||||
};
|
||||
}
|
||||
} else {
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
|
||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
|
||||
if (isRpcError(input)) return input;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// Browser-native text entry for live editing sessions. Unlike fill, this
|
||||
// module does not assume that the focused editable retains a DOM value: a
|
||||
// Canvas editor may use an input/textarea only as a transient keyboard/IME
|
||||
// sink and clear it after consuming each event.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||
return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
|
||||
}
|
||||
|
||||
interface KeyDescriptor {
|
||||
key: string;
|
||||
code: string;
|
||||
windowsVirtualKeyCode: number;
|
||||
nativeVirtualKeyCode?: number;
|
||||
modifiers: number;
|
||||
}
|
||||
|
||||
function asciiKeyDescriptor(character: string): KeyDescriptor {
|
||||
if (/^[a-z]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Key${character.toUpperCase()}`,
|
||||
windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
if (/^[A-Z]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Key${character}`,
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 8,
|
||||
};
|
||||
}
|
||||
if (/^[0-9]$/.test(character)) {
|
||||
return {
|
||||
key: character,
|
||||
code: `Digit${character}`,
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
if (character === " ") {
|
||||
return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
|
||||
}
|
||||
return {
|
||||
key: character,
|
||||
code: "",
|
||||
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||
modifiers: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one complete browser key transaction and always release the key. The
|
||||
* commit callback owns the text-producing phase: a `char` event for a
|
||||
* keyboard character, or `Input.insertText` for committed IME text.
|
||||
*/
|
||||
async function dispatchKeyTransaction(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
descriptor: KeyDescriptor,
|
||||
commit: () => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
let keyDown = false;
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "rawKeyDown",
|
||||
...descriptor,
|
||||
});
|
||||
keyDown = true;
|
||||
const abortedAfterDown = cancelled(signal);
|
||||
if (!abortedAfterDown) await commit();
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
...descriptor,
|
||||
});
|
||||
keyDown = false;
|
||||
return abortedAfterDown ?? cancelled(signal);
|
||||
} catch (err) {
|
||||
if (keyDown) {
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
...descriptor,
|
||||
});
|
||||
} catch {
|
||||
// Best-effort cleanup keeps the page from observing a stuck key.
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function typeAsciiCharacter(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
character: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const descriptor = asciiKeyDescriptor(character);
|
||||
return dispatchKeyTransaction(
|
||||
cdp,
|
||||
tabId,
|
||||
descriptor,
|
||||
async () => {
|
||||
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||
type: "char",
|
||||
...descriptor,
|
||||
text: character,
|
||||
unmodifiedText: character,
|
||||
});
|
||||
},
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
async function typeImeSegment(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
text: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
// A committed IME value is still part of a keyboard transaction. The
|
||||
// Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
|
||||
// IME owns the keystroke. It lets selection-based editors enter text mode
|
||||
// before insertText commits the Unicode value, without inventing a visible
|
||||
// ASCII prefix or leaving a composition transaction open.
|
||||
const processKey: KeyDescriptor = {
|
||||
key: "Process",
|
||||
code: "",
|
||||
windowsVirtualKeyCode: 229,
|
||||
nativeVirtualKeyCode: 229,
|
||||
modifiers: 0,
|
||||
};
|
||||
return dispatchKeyTransaction(
|
||||
cdp,
|
||||
tabId,
|
||||
processKey,
|
||||
() => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
|
||||
* delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
|
||||
* IME/software-keyboard insertion path. Successful return means the browser
|
||||
* accepted the input commands, not that a DOM value retained the text.
|
||||
*/
|
||||
export async function dispatchTextInput(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
text: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RpcError | null> {
|
||||
const abortedBefore = cancelled(signal);
|
||||
if (abortedBefore) return abortedBefore;
|
||||
|
||||
const segments: Array<{ ime: boolean; text: string }> = [];
|
||||
for (const character of text) {
|
||||
const ime = character.codePointAt(0)! > 0x7f;
|
||||
const previous = segments.at(-1);
|
||||
if (previous?.ime === ime) previous.text += character;
|
||||
else segments.push({ ime, text: character });
|
||||
}
|
||||
|
||||
try {
|
||||
for (const segment of segments) {
|
||||
if (segment.ime) {
|
||||
const error = await typeImeSegment(cdp, tabId, segment.text, signal);
|
||||
if (error) return error;
|
||||
continue;
|
||||
}
|
||||
for (const character of segment.text) {
|
||||
const error = await typeAsciiCharacter(cdp, tabId, character, signal);
|
||||
if (error) return error;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -484,6 +484,26 @@ export interface FillResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface TypeParams {
|
||||
session_id: string;
|
||||
text: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
/** Explicitly dispatch to the unique focused text-entry target. */
|
||||
focused?: boolean;
|
||||
tab_id?: number;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface TypeResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
text_length: number;
|
||||
verification: "dispatched";
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface PressParams {
|
||||
session_id: string;
|
||||
key: string;
|
||||
|
||||
@@ -97,14 +97,25 @@ Run with a `900x700` Agent Window.
|
||||
|
||||
This page exercises generic frame-aware input routing. It contains a top-level
|
||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||
frame includes an input, textarea, contenteditable editor, and a controlled
|
||||
input that resets its value on the next animation frame.
|
||||
frame includes an input, textarea, contenteditable editor, an input that
|
||||
replaces its DOM node after accepted input, an input that rejects synthetic
|
||||
DOM events, a transient keyboard/IME input proxy that consumes and clears its
|
||||
DOM value, and a controlled input that resets its value on the next animation
|
||||
frame.
|
||||
|
||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||
ref's owning CDP target.
|
||||
- `press --ref` must focus and dispatch every key event through that same target.
|
||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||
ownership cannot be determined uniquely.
|
||||
- The remounting input must retain the value through its replacement node.
|
||||
- The trusted input must be filled without synthetic `input` / `change` events.
|
||||
- `type` must deliver ASCII key events and committed non-ASCII IME/software
|
||||
keyboard insertion to the input proxy. Its DOM value remains empty while
|
||||
`proxy output` retains the consumed text; the result reports
|
||||
`verification=dispatched`, not a retained DOM value.
|
||||
- `type` must reject an omitted target unless `focused=true` is explicit, and
|
||||
focused mode must reject focus that is not owned by a text-entry target.
|
||||
- The rejecting input must return `input_not_applied`, never a successful
|
||||
requested-value length.
|
||||
|
||||
|
||||
@@ -14,9 +14,38 @@
|
||||
<label>Frame input <input aria-label="frame input" /></label>
|
||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||
<div contenteditable="true" aria-label="frame editor"></div>
|
||||
<label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
|
||||
<label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
|
||||
<label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
|
||||
<output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
|
||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const remountHost = document.querySelector("#remount-host");
|
||||
const armRemount = (input) => {
|
||||
input.addEventListener("input", () => {
|
||||
const replacement = document.createElement("input");
|
||||
replacement.setAttribute("aria-label", "remounting input");
|
||||
replacement.value = input.value;
|
||||
armRemount(replacement);
|
||||
input.replaceWith(replacement);
|
||||
replacement.focus();
|
||||
}, { once: true });
|
||||
};
|
||||
armRemount(remountHost.querySelector("input"));
|
||||
const trusted = document.querySelector("#trusted");
|
||||
trusted.addEventListener("input", (event) => {
|
||||
if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
|
||||
});
|
||||
const proxy = document.querySelector("#proxy");
|
||||
const proxyOutput = document.querySelector("#proxy-output");
|
||||
let proxyValue = "";
|
||||
proxy.addEventListener("input", (event) => {
|
||||
if (event.isComposing) return;
|
||||
proxyValue += event.data ?? proxy.value;
|
||||
proxy.value = "";
|
||||
proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
|
||||
});
|
||||
const rejecting = document.querySelector("#rejecting");
|
||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||
if (params.get("nested") === "1") {
|
||||
|
||||
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
### Choosing `fill`, `type`, and `press`
|
||||
|
||||
Choose by the intended postcondition, not by whether observation happens to
|
||||
render `textbox [empty]`:
|
||||
|
||||
- `fill` sets a field's retained value. Use it when success means the target
|
||||
value must equal the requested text; it replaces existing content by default.
|
||||
- `type` inserts text at the active caret/selection in a live editing session.
|
||||
Use it when the application must consume keyboard/IME text; it never clears
|
||||
first and reports dispatch only.
|
||||
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||
Do not encode special keys inside `type --text`.
|
||||
|
||||
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||
available, or pass `--focused` only after an immediately preceding action in
|
||||
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||
application may already have consumed the text before clearing its proxy DOM
|
||||
value. Re-observe first. After `type`, verify the application output with one
|
||||
observation or screenshot; `verification=dispatched` is not business success.
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -95,6 +95,27 @@ fn split_target(
|
||||
}
|
||||
}
|
||||
|
||||
fn split_type_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
focused: bool,
|
||||
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||
if focused {
|
||||
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"pass a target or --focused, not both"
|
||||
)));
|
||||
}
|
||||
return Ok((None, None));
|
||||
}
|
||||
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||
CliError::Local(anyhow::anyhow!(
|
||||
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
#[group(id = "type_destination", required = true, multiple = false)]
|
||||
pub struct TypeDestinationArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||
#[arg(value_name = "TARGET")]
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||
#[arg(long)]
|
||||
pub focused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct TypeArgs {
|
||||
#[command(flatten)]
|
||||
pub destination: TypeDestinationArgs,
|
||||
|
||||
/// Text to dispatch through the live keyboard/IME editing session.
|
||||
#[arg(long)]
|
||||
pub text: String,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||
let (ref_, selector) = split_type_target(
|
||||
args.destination.target.clone(),
|
||||
args.destination.ref_.clone(),
|
||||
args.destination.selector.clone(),
|
||||
args.destination.focused,
|
||||
)?;
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let params = TypeParams {
|
||||
session_id: args.session.clone(),
|
||||
text: args.text.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
focused: args.destination.focused,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: TypeResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolType,
|
||||
params,
|
||||
"type-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target = if args.destination.focused {
|
||||
"focused".into()
|
||||
} else {
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||
};
|
||||
println!(
|
||||
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||
reply.tab_id, reply.text_length
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -659,6 +769,52 @@ mod tests {
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||
.expect("type args should parse");
|
||||
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||
assert_eq!(parsed.args.text, "测试");
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||
.expect("focused type args should parse");
|
||||
assert!(parsed.args.destination.focused);
|
||||
assert!(parsed.args.destination.target.is_none());
|
||||
|
||||
let focused = split_type_target(None, None, None, true).unwrap();
|
||||
assert_eq!(focused, (None, None));
|
||||
|
||||
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||
assert!(missing.to_string().contains("--focused"));
|
||||
|
||||
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||
assert!(mixed.to_string().contains("not both"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||
use clap::Parser;
|
||||
|
||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -171,6 +171,9 @@ pub enum Command {
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||
Type(TypeArgs),
|
||||
|
||||
/// Dispatch a keyboard key combo.
|
||||
Press(PressArgs),
|
||||
|
||||
|
||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
|
||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||
WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn type_round_trips_dispatched_verification() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolType);
|
||||
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||
assert_eq!(p.text, "测试");
|
||||
assert!(!p.focused);
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(TypeResult {
|
||||
tab_id: 12,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: TypeResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id,
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("type ok");
|
||||
assert_eq!(result.text_length, 2);
|
||||
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn press_round_trips_compound_key() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::CdpFailed,
|
||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
.unwrap_err();
|
||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||
|
||||
let type_result = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id: session_id.clone(),
|
||||
text: "hello".into(),
|
||||
ref_: Some("@e1".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||
|
||||
let press = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolPress,
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
|
||||
"type": "string"
|
||||
},
|
||||
"truncated": {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id",
|
||||
"text"
|
||||
],
|
||||
"properties": {
|
||||
"focused": {
|
||||
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"ref": {
|
||||
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
|
||||
"type": "string"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tab_id",
|
||||
"text_length",
|
||||
"verification"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"text_length": {
|
||||
"description": "UTF-16 code-unit length of text accepted for dispatch.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"verification": {
|
||||
"$ref": "#/definitions/TypeVerification"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"TypeVerification": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"dispatched"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,8 @@ fn main() {
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(TypeParams, "tool_type_params");
|
||||
dump!(TypeResult, "tool_type_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
dump!(PressResult, "tool_press_result");
|
||||
dump!(SelectParams, "tool_select_params");
|
||||
|
||||
@@ -74,6 +74,8 @@ pub enum Method {
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.type")]
|
||||
ToolType,
|
||||
#[serde(rename = "tool.press")]
|
||||
ToolPress,
|
||||
#[serde(rename = "tool.select")]
|
||||
@@ -154,6 +156,7 @@ impl Method {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
@@ -246,6 +249,13 @@ mod tests {
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_method_round_trips() {
|
||||
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
|
||||
assert_eq!(method, Method::ToolType);
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_params_and_result_round_trip() {
|
||||
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
||||
@@ -284,6 +294,7 @@ mod tests {
|
||||
assert!(Method::ToolReload.is_mutating());
|
||||
assert!(Method::ToolClick.is_mutating());
|
||||
assert!(Method::ToolFill.is_mutating());
|
||||
assert!(Method::ToolType.is_mutating());
|
||||
assert!(Method::ToolPress.is_mutating());
|
||||
assert!(Method::ToolSelect.is_mutating());
|
||||
assert!(Method::ToolEvaluate.is_mutating());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
|
||||
//! `tool.select`).
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
|
||||
//! `tool.press`, `tool.select`).
|
||||
//!
|
||||
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
||||
//! normalised against the session's RefStore) **or** a CSS selector
|
||||
@@ -188,6 +188,60 @@ pub struct FillResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeParams {
|
||||
pub session_id: String,
|
||||
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
|
||||
/// does not require the focused DOM editable to retain this text.
|
||||
pub text: String,
|
||||
/// Target to focus before dispatching text. Exactly one targeting mode
|
||||
/// must be selected: `ref` / `selector`, or `focused=true`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Explicitly dispatch to the unique focused text-entry target instead
|
||||
/// of focusing a ref or selector.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub focused: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TypeVerification {
|
||||
/// Chromium accepted the complete keyboard/IME dispatch transaction.
|
||||
/// The caller must observe the application output to verify business state.
|
||||
Dispatched,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// UTF-16 code-unit length of text accepted for dispatch.
|
||||
pub text_length: u32,
|
||||
pub verification: TypeVerification,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -351,6 +405,50 @@ mod tests {
|
||||
assert_eq!(v, json!("meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_payloads_distinguish_dispatch_from_value_verification() {
|
||||
let params = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: Some(4),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let encoded = serde_json::to_value(¶ms).unwrap();
|
||||
assert_eq!(encoded["ref"], json!("@e3"));
|
||||
assert_eq!(encoded["text"], json!("测试"));
|
||||
assert!(encoded.get("focused").is_none());
|
||||
|
||||
let focused = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "hello".into(),
|
||||
ref_: None,
|
||||
selector: None,
|
||||
focused: true,
|
||||
tab_id: None,
|
||||
timeout_ms: None,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(focused).unwrap()["focused"],
|
||||
json!(true)
|
||||
);
|
||||
|
||||
let result = TypeResult {
|
||||
tab_id: 4,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(result).unwrap()["verification"],
|
||||
json!("dispatched")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn press_result_round_trips() {
|
||||
let r = PressResult {
|
||||
|
||||
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SnapshotResult {
|
||||
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill`
|
||||
/// selectors.
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
|
||||
/// `tool.type` targets.
|
||||
pub text: String,
|
||||
/// Number of `@e<N>` refs registered for this session by this
|
||||
/// snapshot. Equivalent to the size of the new ref-store map.
|
||||
|
||||
Reference in New Issue
Block a user