mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): add type primitive
This commit is contained in:
1 parent
e069abaf2e
commit
a1904f8141
23 files changed
+1776
-156
No files matched your search
@@ -0,0 +1,104 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { fillEditable } from "../editable-input";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
|
||||||
|
interface FakeOptions {
|
||||||
|
initialValue?: string;
|
||||||
|
finalValue: string;
|
||||||
|
remount?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function editableState(value: string, connected = true, focused = true) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: { supported: true, connected, focused, value, kind: "input" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeEditableCdp(options: FakeOptions) {
|
||||||
|
const sent: Array<{ method: string; params?: object }> = [];
|
||||||
|
let inserted = false;
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||||
|
sent.push({ method, params });
|
||||||
|
if (method === "Input.insertText") {
|
||||||
|
inserted = true;
|
||||||
|
return {} as T;
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchKeyEvent") {
|
||||||
|
if ((params as { type?: string }).type === "rawKeyDown") inserted = true;
|
||||||
|
return {} as T;
|
||||||
|
}
|
||||||
|
if (method !== "Runtime.callFunctionOn") throw new Error(`unexpected ${method}`);
|
||||||
|
|
||||||
|
const call = params as { objectId?: string; functionDeclaration?: string };
|
||||||
|
const fn = call.functionDeclaration ?? "";
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } } as T;
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-sync")) return { result: { value: undefined } } as T;
|
||||||
|
if (fn.includes("bsk-editable-live")) {
|
||||||
|
return {
|
||||||
|
result: options.remount && inserted ? { objectId: "replacement" } : { objectId: "anchor" },
|
||||||
|
} as T;
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
if (call.objectId === "replacement") return editableState(options.finalValue) as T;
|
||||||
|
if (options.remount && inserted) {
|
||||||
|
return editableState(options.finalValue, false, false) as T;
|
||||||
|
}
|
||||||
|
return editableState(inserted ? options.finalValue : (options.initialValue ?? "")) as T;
|
||||||
|
}
|
||||||
|
throw new Error("unexpected Runtime.callFunctionOn function");
|
||||||
|
});
|
||||||
|
return { cdp: { send } as CdpRunner, sent };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("fillEditable", () => {
|
||||||
|
it("replaces text through the browser input path without synthetic events or value setters", async () => {
|
||||||
|
const fake = makeEditableCdp({ initialValue: "old", finalValue: "新值" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "新值", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "新值", replacedNode: false });
|
||||||
|
expect(fake.sent).toContainEqual({ method: "Input.insertText", params: { text: "新值" } });
|
||||||
|
const declarations = fake.sent
|
||||||
|
.map((call) => (call.params as { functionDeclaration?: string })?.functionDeclaration ?? "")
|
||||||
|
.join("\n");
|
||||||
|
expect(declarations).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reacquires the focused editable after a framework remount", async () => {
|
||||||
|
const fake = makeEditableCdp({ finalValue: "persisted", remount: true });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "persisted", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "persisted", replacedNode: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a page rollback after the bounded settlement", async () => {
|
||||||
|
const fake = makeEditableCdp({ finalValue: "" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "rejected", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: {
|
||||||
|
reason: "input_not_applied",
|
||||||
|
expected_value_length: 8,
|
||||||
|
actual_value_length: 0,
|
||||||
|
phase: "readback",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears to an empty value with a browser Backspace edit", async () => {
|
||||||
|
const fake = makeEditableCdp({ initialValue: "old", finalValue: "" });
|
||||||
|
|
||||||
|
const result = await fillEditable(fake.cdp, 4, "anchor", "", true);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ value: "" });
|
||||||
|
expect(fake.sent.some((call) => call.method === "Input.insertText")).toBe(false);
|
||||||
|
expect(fake.sent.filter((call) => call.method === "Input.dispatchKeyEvent")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
handleHover,
|
handleHover,
|
||||||
handlePress,
|
handlePress,
|
||||||
handleSelect,
|
handleSelect,
|
||||||
|
handleType,
|
||||||
modifiersBitfield,
|
modifiersBitfield,
|
||||||
parseKeySpec,
|
parseKeySpec,
|
||||||
resolveKeyDescriptor,
|
resolveKeyDescriptor,
|
||||||
@@ -71,8 +72,21 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
|||||||
function fillRuntimeReadback(value: string) {
|
function fillRuntimeReadback(value: string) {
|
||||||
return (params: unknown) => {
|
return (params: unknown) => {
|
||||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
if (fn.includes("bsk-input-readback")) {
|
if (fn.includes("bsk-editable-state")) {
|
||||||
return { result: { value: { supported: true, value } } };
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value,
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
}
|
}
|
||||||
return { result: { type: "undefined" } };
|
return { result: { type: "undefined" } };
|
||||||
};
|
};
|
||||||
@@ -770,13 +784,13 @@ describe("handleFill", () => {
|
|||||||
expect(res.used_ref).toBe("e1");
|
expect(res.used_ref).toBe("e1");
|
||||||
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
const insert = fake.sent.find((c) => c.method === "Input.insertText");
|
||||||
expect(insert?.params).toEqual({ text: "hello" });
|
expect(insert?.params).toEqual({ text: "hello" });
|
||||||
// clear_before defaults to true → callFunctionOn invoked once to
|
// clear_before defaults to true, so the edit transaction prepares a
|
||||||
// clear, once to fire input/change after typing.
|
// replacement selection and performs a settled live-value readback.
|
||||||
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
const callFns = fake.sent.filter((c) => c.method === "Runtime.callFunctionOn");
|
||||||
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
expect(callFns.length).toBeGreaterThanOrEqual(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clear_before=false skips the wipe call", async () => {
|
it("clear_before=false skips replacement selection", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e1", 1, { tabId: 4 });
|
ctx.refStore.set("e1", 1, { tabId: 4 });
|
||||||
@@ -789,12 +803,20 @@ describe("handleFill", () => {
|
|||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-2" } }),
|
||||||
"Runtime.callFunctionOn": (p) => {
|
"Runtime.callFunctionOn": (p) => {
|
||||||
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
const fn = (p as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||||
if (fn.includes("bsk-input-readback")) {
|
if (fn.includes("bsk-editable-state")) {
|
||||||
return { result: { value: { supported: true, value: "existingx" } } };
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "existingx",
|
||||||
|
kind: "textarea",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
}
|
}
|
||||||
// No "this.value = ''" clearing on a no-clear path; only the
|
expect(fn).not.toMatch(/dispatchEvent|\.value\s*=/);
|
||||||
// post-input dispatchEvent.
|
|
||||||
expect(fn).not.toMatch(/this\.value\s*=\s*''/);
|
|
||||||
return { result: { type: "undefined" } };
|
return { result: { type: "undefined" } };
|
||||||
},
|
},
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
@@ -870,9 +892,23 @@ describe("handleFill", () => {
|
|||||||
}
|
}
|
||||||
if (method === "Runtime.callFunctionOn") {
|
if (method === "Runtime.callFunctionOn") {
|
||||||
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
return fn.includes("bsk-input-readback")
|
if (fn.includes("bsk-editable-state")) {
|
||||||
? { result: { value: { supported: true, value: "中文输入" } } }
|
return {
|
||||||
: { result: { type: "undefined" } };
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "中文输入",
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
|
}
|
||||||
|
return { result: { type: "undefined" } };
|
||||||
}
|
}
|
||||||
throw new Error(`unexpected child call ${method}`);
|
throw new Error(`unexpected child call ${method}`);
|
||||||
}) as CdpRunner["sendToTarget"];
|
}) as CdpRunner["sendToTarget"];
|
||||||
@@ -916,7 +952,7 @@ describe("handleFill", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("stops before Input.insertText when abort fires after clearing", async () => {
|
it("stops before Input.insertText when abort fires after preparing replacement", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
ctx.refStore.set("e1", 555, { tabId: 4 });
|
ctx.refStore.set("e1", 555, { tabId: 4 });
|
||||||
@@ -928,8 +964,25 @@ describe("handleFill", () => {
|
|||||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
"DOM.focus": () => ({}),
|
"DOM.focus": () => ({}),
|
||||||
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
"DOM.resolveNode": () => ({ object: { objectId: "obj-1" } }),
|
||||||
"Runtime.callFunctionOn": () => {
|
"Runtime.callFunctionOn": (params) => {
|
||||||
abort.abort();
|
const fn = String((params as { functionDeclaration?: string }).functionDeclaration ?? "");
|
||||||
|
if (fn.includes("bsk-editable-state")) {
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
value: {
|
||||||
|
supported: true,
|
||||||
|
connected: true,
|
||||||
|
focused: true,
|
||||||
|
value: "",
|
||||||
|
kind: "input",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (fn.includes("bsk-editable-selection")) {
|
||||||
|
abort.abort();
|
||||||
|
return { result: { value: { selected: true } } };
|
||||||
|
}
|
||||||
return { result: { type: "undefined" } };
|
return { result: { type: "undefined" } };
|
||||||
},
|
},
|
||||||
"Input.insertText": () => ({}),
|
"Input.insertText": () => ({}),
|
||||||
@@ -972,6 +1025,204 @@ describe("handleFill", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleType", () => {
|
||||||
|
it("requires an explicit target mode instead of silently using focus", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({});
|
||||||
|
|
||||||
|
const missing = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
expect(missing).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: expect.stringContaining("focused=true"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const mixed = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", focused: true, text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
expect(mixed).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
message: expect.stringContaining("not both"),
|
||||||
|
});
|
||||||
|
expect(fake.sent).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes committed IME text through the explicit OOPIF target without DOM readback", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, {
|
||||||
|
tabId: 4,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}) });
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 12,
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: Array<{ method: string; params?: object }> = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method, params) => {
|
||||||
|
childCalls.push({ method, params });
|
||||||
|
if (method === "DOM.describeNode") {
|
||||||
|
return { node: { backendNodeId: 77, nodeName: "TEXTAREA", attributes: [] } };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "测试" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
tab_id: 4,
|
||||||
|
used_ref: "e1",
|
||||||
|
text_length: 2,
|
||||||
|
verification: "dispatched",
|
||||||
|
});
|
||||||
|
expect(childCalls.map((call) => call.method)).toEqual([
|
||||||
|
"DOM.describeNode",
|
||||||
|
"DOM.scrollIntoViewIfNeeded",
|
||||||
|
"DOM.focus",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.insertText",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
expect(childCalls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses the focused OOPIF only when focused mode resolves a text-entry target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "iframe", textEntry: false } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 4 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
target: { tabId: 4, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const childCalls: string[] = [];
|
||||||
|
fake.cdp.sendToTarget = vi.fn(async (_target, method) => {
|
||||||
|
childCalls.push(method);
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
return {
|
||||||
|
result: { value: { hasFocus: true, activeTag: "textarea", textEntry: true } },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", focused: true, text: "A" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in result) throw new Error(JSON.stringify(result));
|
||||||
|
expect(result).toMatchObject({ text_length: 1, verification: "dispatched" });
|
||||||
|
expect(childCalls).toEqual([
|
||||||
|
"Runtime.evaluate",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses focused mode when focus is not owned by a text-entry target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"Runtime.evaluate": () => ({
|
||||||
|
result: { value: { hasFocus: true, activeTag: "button", textEntry: false } },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", focused: true, text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "cdp_failed",
|
||||||
|
data: { reason: "focus_target_unresolved", candidate_count: 0 },
|
||||||
|
});
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects explicit non-text input controls", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, { tabId: 4 });
|
||||||
|
const fake = makeFakeCdp({
|
||||||
|
"DOM.describeNode": () => ({
|
||||||
|
node: { backendNodeId: 77, nodeName: "INPUT", attributes: ["type", "checkbox"] },
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "invalid_params",
|
||||||
|
data: { reason: "target_not_fillable" },
|
||||||
|
});
|
||||||
|
expect(fake.sent.some((call) => call.method.startsWith("Input."))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a Surface ref instead of treating it as a text target", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e1", 77, {
|
||||||
|
tabId: 4,
|
||||||
|
kind: "surface",
|
||||||
|
capabilities: ["screenshot"],
|
||||||
|
});
|
||||||
|
const fake = makeFakeCdp({});
|
||||||
|
|
||||||
|
const result = await handleType(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e1", text: "x" },
|
||||||
|
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
code: "permission_denied",
|
||||||
|
data: { reason: "ref_capability_denied", required_capability: "interact" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("parseKeySpec", () => {
|
describe("parseKeySpec", () => {
|
||||||
it("splits compound expressions into modifiers + base key", () => {
|
it("splits compound expressions into modifiers + base key", () => {
|
||||||
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
expect(parseKeySpec("Ctrl+A")).toEqual({ key: "A", modifiers: ["ctrl"] });
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { dispatchTextInput } from "../text-input";
|
||||||
|
|
||||||
|
function makeCdp(onSend?: (method: string, params?: object) => void) {
|
||||||
|
const sent: Array<{ method: string; params?: object }> = [];
|
||||||
|
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object): Promise<T> => {
|
||||||
|
sent.push({ method, params });
|
||||||
|
onSend?.(method, params);
|
||||||
|
return {} as T;
|
||||||
|
});
|
||||||
|
return { cdp: { send } as CdpRunner, sent };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("dispatchTextInput", () => {
|
||||||
|
it("types ASCII through complete keydown, char, and keyup sequences", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "Ab");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(
|
||||||
|
fake.sent.map((call) => `${call.method}:${(call.params as { type?: string }).type}`),
|
||||||
|
).toEqual([
|
||||||
|
"Input.dispatchKeyEvent:rawKeyDown",
|
||||||
|
"Input.dispatchKeyEvent:char",
|
||||||
|
"Input.dispatchKeyEvent:keyUp",
|
||||||
|
"Input.dispatchKeyEvent:rawKeyDown",
|
||||||
|
"Input.dispatchKeyEvent:char",
|
||||||
|
"Input.dispatchKeyEvent:keyUp",
|
||||||
|
]);
|
||||||
|
expect(fake.sent[0].params).toMatchObject({ key: "A", code: "KeyA", modifiers: 8 });
|
||||||
|
expect(fake.sent[1].params).toMatchObject({ text: "A" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("types non-ASCII through the committed IME insertion path", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(fake.sent).toEqual([
|
||||||
|
{
|
||||||
|
method: "Input.dispatchKeyEvent",
|
||||||
|
params: {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ method: "Input.insertText", params: { text: "测试" } },
|
||||||
|
{
|
||||||
|
method: "Input.dispatchKeyEvent",
|
||||||
|
params: {
|
||||||
|
type: "keyUp",
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves text order across keyboard and IME segments", async () => {
|
||||||
|
const fake = makeCdp();
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "A测试1");
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
const meaningful = fake.sent.flatMap((call) => {
|
||||||
|
if (call.method === "Input.insertText") {
|
||||||
|
return [`ime:${(call.params as { text: string }).text}`];
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
call.method === "Input.dispatchKeyEvent" &&
|
||||||
|
(call.params as { type?: string }).type === "char"
|
||||||
|
) {
|
||||||
|
return [`key:${(call.params as { text: string }).text}`];
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
expect(meaningful).toEqual(["key:A", "ime:测试", "key:1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports cancellation observed after an accepted IME insertion", async () => {
|
||||||
|
const abort = new AbortController();
|
||||||
|
const fake = makeCdp((method) => {
|
||||||
|
if (method === "Input.insertText") abort.abort();
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.insertText",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("releases the IME process key without committing when cancellation follows keydown", async () => {
|
||||||
|
const abort = new AbortController();
|
||||||
|
const fake = makeCdp((method, params) => {
|
||||||
|
if (
|
||||||
|
method === "Input.dispatchKeyEvent" &&
|
||||||
|
(params as { type?: string }).type === "rawKeyDown"
|
||||||
|
) {
|
||||||
|
abort.abort();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await dispatchTextInput(fake.cdp, 4, "测试", abort.signal);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ code: "cancelled" });
|
||||||
|
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
"Input.dispatchKeyEvent",
|
||||||
|
]);
|
||||||
|
expect(fake.sent[0].params).toMatchObject({ type: "rawKeyDown", key: "Process" });
|
||||||
|
expect(fake.sent[1].params).toMatchObject({ type: "keyUp", key: "Process" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -28,6 +28,7 @@ import type {
|
|||||||
ScreenshotParams,
|
ScreenshotParams,
|
||||||
SelectParams,
|
SelectParams,
|
||||||
SnapshotParams,
|
SnapshotParams,
|
||||||
|
TypeParams,
|
||||||
WaitForNavigationParams,
|
WaitForNavigationParams,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { isRequestFrame } from "@/transport/types";
|
import { isRequestFrame } from "@/transport/types";
|
||||||
@@ -35,7 +36,14 @@ import { handleConsole } from "./console";
|
|||||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||||
import { handleEvaluate } from "./evaluate";
|
import { handleEvaluate } from "./evaluate";
|
||||||
import { handleRequestHelp } from "./human-loop";
|
import { handleRequestHelp } from "./human-loop";
|
||||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
import {
|
||||||
|
handleClick,
|
||||||
|
handleFill,
|
||||||
|
handleHover,
|
||||||
|
handlePress,
|
||||||
|
handleSelect,
|
||||||
|
handleType,
|
||||||
|
} from "./interaction";
|
||||||
import {
|
import {
|
||||||
handleNavigate,
|
handleNavigate,
|
||||||
handleNavigateBack,
|
handleNavigateBack,
|
||||||
@@ -491,6 +499,17 @@ export class ToolDispatcher {
|
|||||||
),
|
),
|
||||||
signal,
|
signal,
|
||||||
);
|
);
|
||||||
|
case "tool.type":
|
||||||
|
return this.withHoverReleaseForRequest(
|
||||||
|
req.params as TypeParams,
|
||||||
|
() =>
|
||||||
|
handleType(
|
||||||
|
this.sessions,
|
||||||
|
req.params as TypeParams,
|
||||||
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||||
|
),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
return this.withHoverReleaseForRequest(
|
return this.withHoverReleaseForRequest(
|
||||||
req.params as PressParams,
|
req.params as PressParams,
|
||||||
@@ -717,6 +736,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
|||||||
case "tool.click":
|
case "tool.click":
|
||||||
case "tool.hover":
|
case "tool.hover":
|
||||||
case "tool.fill":
|
case "tool.fill":
|
||||||
|
case "tool.type":
|
||||||
case "tool.press":
|
case "tool.press":
|
||||||
case "tool.select":
|
case "tool.select":
|
||||||
case "tool.evaluate":
|
case "tool.evaluate":
|
||||||
|
|||||||
@@ -0,0 +1,282 @@
|
|||||||
|
// Browser-native editing transaction for input, textarea, and contenteditable
|
||||||
|
// targets. The CDP target/session is selected by interaction.ts; this module
|
||||||
|
// owns the shorter-lived editable identity inside that target.
|
||||||
|
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { rpcError } from "./errors";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
|
||||||
|
interface EditableState {
|
||||||
|
supported: boolean;
|
||||||
|
connected: boolean;
|
||||||
|
focused: boolean;
|
||||||
|
value: string;
|
||||||
|
kind: "input" | "textarea" | "contenteditable" | "unsupported";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EditableFillResult {
|
||||||
|
value: string;
|
||||||
|
replacedNode: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||||
|
return signal?.aborted ? { code: "cancelled", message: "fill aborted" } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function inspectEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<EditableState> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { value?: EditableState };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-state */
|
||||||
|
const doc = this.ownerDocument;
|
||||||
|
const focused = !!doc && doc.activeElement === this;
|
||||||
|
if (this instanceof HTMLInputElement) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.value, kind: 'input' };
|
||||||
|
}
|
||||||
|
if (this instanceof HTMLTextAreaElement) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.value, kind: 'textarea' };
|
||||||
|
}
|
||||||
|
if (this.isContentEditable) {
|
||||||
|
return { supported: true, connected: this.isConnected, focused,
|
||||||
|
value: this.innerText ?? this.textContent ?? '', kind: 'contenteditable' };
|
||||||
|
}
|
||||||
|
return { supported: false, connected: this.isConnected === true, focused,
|
||||||
|
value: '', kind: 'unsupported' };
|
||||||
|
}`,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const state = reply.result?.value;
|
||||||
|
if (!state || typeof state.value !== "string") {
|
||||||
|
throw new Error("editable state probe returned no value");
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the active editable from the anchor's own document. Using the
|
||||||
|
* anchor's execution context also works for same-process nested frames, where
|
||||||
|
* a target-level Runtime.evaluate would otherwise see the top document.
|
||||||
|
*/
|
||||||
|
async function focusedEditableFromAnchor(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<string | null> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { objectId?: string; subtype?: string };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId: anchorObjectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-live */
|
||||||
|
const active = this.ownerDocument?.activeElement;
|
||||||
|
if (!active) return null;
|
||||||
|
const editable = active instanceof HTMLInputElement ||
|
||||||
|
active instanceof HTMLTextAreaElement || active.isContentEditable;
|
||||||
|
return editable ? active : null;
|
||||||
|
}`,
|
||||||
|
returnByValue: false,
|
||||||
|
});
|
||||||
|
if (reply.result?.subtype === "null") return null;
|
||||||
|
return typeof reply.result?.objectId === "string" ? reply.result.objectId : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveLiveEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<{ objectId: string; state: EditableState; replaced: boolean } | RpcError> {
|
||||||
|
const anchor = await inspectEditable(cdp, tabId, anchorObjectId);
|
||||||
|
if (anchor.supported && anchor.connected && anchor.focused) {
|
||||||
|
return { objectId: anchorObjectId, state: anchor, replaced: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const focusedObjectId = await focusedEditableFromAnchor(cdp, tabId, anchorObjectId);
|
||||||
|
if (!focusedObjectId) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the target editable lost focus or detached before input",
|
||||||
|
{ phase: "resolve_live_editable" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const focused = await inspectEditable(cdp, tabId, focusedObjectId);
|
||||||
|
if (!focused.supported || !focused.connected || !focused.focused) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the replacement editable was not live and focused",
|
||||||
|
{ phase: "resolve_live_editable" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { objectId: focusedObjectId, state: focused, replaced: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function selectEditableContents(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
objectId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const reply = await cdp.send<{
|
||||||
|
result?: { value?: { selected?: boolean } };
|
||||||
|
}>(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-selection */
|
||||||
|
if (!this.isConnected || this.ownerDocument?.activeElement !== this) {
|
||||||
|
return { selected: false };
|
||||||
|
}
|
||||||
|
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
||||||
|
try {
|
||||||
|
this.setSelectionRange(0, this.value.length);
|
||||||
|
} catch {
|
||||||
|
if (typeof this.select !== 'function') return { selected: false };
|
||||||
|
this.select();
|
||||||
|
}
|
||||||
|
return { selected: true };
|
||||||
|
}
|
||||||
|
if (this.isContentEditable) {
|
||||||
|
const selection = this.ownerDocument.getSelection();
|
||||||
|
if (!selection) return { selected: false };
|
||||||
|
const range = this.ownerDocument.createRange();
|
||||||
|
range.selectNodeContents(this);
|
||||||
|
selection.removeAllRanges();
|
||||||
|
selection.addRange(range);
|
||||||
|
return { selected: true };
|
||||||
|
}
|
||||||
|
return { selected: false };
|
||||||
|
}`,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
if (reply.result?.value?.selected !== true) {
|
||||||
|
throw new Error("could not select the live editable contents");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForEditableSettlement(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||||
|
objectId: anchorObjectId,
|
||||||
|
functionDeclaration: `function() {
|
||||||
|
/* bsk-editable-sync */
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
let settled = false;
|
||||||
|
const finish = () => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const timer = setTimeout(finish, 100);
|
||||||
|
const view = this.ownerDocument?.defaultView;
|
||||||
|
if (!view || typeof view.requestAnimationFrame !== 'function') { finish(); return; }
|
||||||
|
view.requestAnimationFrame(() => view.requestAnimationFrame(finish));
|
||||||
|
});
|
||||||
|
}`,
|
||||||
|
awaitPromise: true,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteSelection(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
key: "Backspace",
|
||||||
|
code: "Backspace",
|
||||||
|
windowsVirtualKeyCode: 8,
|
||||||
|
});
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
key: "Backspace",
|
||||||
|
code: "Backspace",
|
||||||
|
windowsVirtualKeyCode: 8,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replace or append text through Chromium's editing pipeline. No value setter
|
||||||
|
* or synthetic DOM events are used: controlled components observe the same
|
||||||
|
* beforeinput/input path as a real browser edit. The live editable is resolved
|
||||||
|
* again after settlement so a framework may remount it without producing a
|
||||||
|
* false failure or a stale-object success.
|
||||||
|
*/
|
||||||
|
export async function fillEditable(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
anchorObjectId: string,
|
||||||
|
value: string,
|
||||||
|
clearBefore: boolean,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<EditableFillResult | RpcError> {
|
||||||
|
try {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
const initial = await resolveLiveEditable(cdp, tabId, anchorObjectId);
|
||||||
|
if ("code" in initial) return initial;
|
||||||
|
let replacedNode = initial.replaced;
|
||||||
|
|
||||||
|
if (clearBefore) await selectEditableContents(cdp, tabId, initial.objectId);
|
||||||
|
const abortedAfterSelection = cancelled(signal);
|
||||||
|
if (abortedAfterSelection) return abortedAfterSelection;
|
||||||
|
|
||||||
|
if (value.length > 0) {
|
||||||
|
await cdp.send(tabId, "Input.insertText", { text: value });
|
||||||
|
} else if (clearBefore) {
|
||||||
|
await deleteSelection(cdp, tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const abortedAfterInput = cancelled(signal);
|
||||||
|
if (abortedAfterInput) return abortedAfterInput;
|
||||||
|
await waitForEditableSettlement(cdp, tabId, initial.objectId);
|
||||||
|
|
||||||
|
const abortedAfterSettlement = cancelled(signal);
|
||||||
|
if (abortedAfterSettlement) return abortedAfterSettlement;
|
||||||
|
|
||||||
|
let finalObjectId = initial.objectId;
|
||||||
|
let finalState = await inspectEditable(cdp, tabId, finalObjectId);
|
||||||
|
if (!finalState.connected || !finalState.focused) {
|
||||||
|
const live = await resolveLiveEditable(cdp, tabId, initial.objectId);
|
||||||
|
if ("code" in live) return live;
|
||||||
|
finalObjectId = live.objectId;
|
||||||
|
finalState = live.state;
|
||||||
|
replacedNode ||= live.replaced || finalObjectId !== initial.objectId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const applied = clearBefore
|
||||||
|
? finalState.value === value
|
||||||
|
: value.length === 0 || finalState.value.includes(value);
|
||||||
|
if (!finalState.supported || !applied) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
"the page did not retain the requested input value",
|
||||||
|
{
|
||||||
|
expected_value_length: value.length,
|
||||||
|
actual_value_length: finalState.value.length,
|
||||||
|
phase: "readback",
|
||||||
|
editable_replaced: replacedNode,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { value: finalState.value, replacedNode };
|
||||||
|
} catch (err) {
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"input_not_applied",
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
{ phase: "edit_transaction" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
|
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.type`,
|
||||||
// `tool.select`.
|
// `tool.press`, and `tool.select`.
|
||||||
//
|
//
|
||||||
// All interaction tools:
|
// All interaction tools:
|
||||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||||
@@ -27,8 +27,11 @@ import type {
|
|||||||
RpcError,
|
RpcError,
|
||||||
SelectParams,
|
SelectParams,
|
||||||
SelectResult,
|
SelectResult,
|
||||||
|
TypeParams,
|
||||||
|
TypeResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
|
import { fillEditable } from "./editable-input";
|
||||||
import { backendNodeToObject } from "./element-geometry";
|
import { backendNodeToObject } from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||||
@@ -45,6 +48,7 @@ import {
|
|||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import { resolveSurfacePointerTarget } from "./surface-target";
|
import { resolveSurfacePointerTarget } from "./surface-target";
|
||||||
|
import { dispatchTextInput } from "./text-input";
|
||||||
|
|
||||||
export interface InteractionDeps {
|
export interface InteractionDeps {
|
||||||
cdp: CdpRunner;
|
cdp: CdpRunner;
|
||||||
@@ -244,18 +248,20 @@ function inputContextForNode(
|
|||||||
interface FocusProbe {
|
interface FocusProbe {
|
||||||
hasFocus: boolean;
|
hasFocus: boolean;
|
||||||
activeTag: string;
|
activeTag: string;
|
||||||
|
textEntry: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
/** Resolve keyboard input to the CDP target that currently owns focus. */
|
||||||
async function resolveFocusedInputContext(
|
async function resolveFocusedInputContext(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
|
mode: "press" | "type",
|
||||||
): Promise<InputDispatchContext | RpcError> {
|
): Promise<InputDispatchContext | RpcError> {
|
||||||
if (!cdp.getFrameGraph) {
|
if (!cdp.getFrameGraph) {
|
||||||
return rpcError(
|
return rpcError(
|
||||||
"cdp_failed",
|
"cdp_failed",
|
||||||
"focus_target_unresolved",
|
"focus_target_unresolved",
|
||||||
"press cannot determine the focused frame target",
|
`${mode} cannot determine the focused frame target`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -284,9 +290,21 @@ async function resolveFocusedInputContext(
|
|||||||
expression: `(() => {
|
expression: `(() => {
|
||||||
/* bsk-focus-target-probe */
|
/* bsk-focus-target-probe */
|
||||||
const active = document.activeElement;
|
const active = document.activeElement;
|
||||||
|
const inputType = active instanceof HTMLInputElement ? active.type.toLowerCase() : '';
|
||||||
|
const nonTextInputTypes = new Set([
|
||||||
|
'button', 'checkbox', 'color', 'date', 'datetime-local', 'file',
|
||||||
|
'hidden', 'image', 'month', 'radio', 'range', 'reset', 'submit',
|
||||||
|
'time', 'week'
|
||||||
|
]);
|
||||||
|
const role = active instanceof Element ? active.getAttribute('role') : null;
|
||||||
return {
|
return {
|
||||||
hasFocus: document.hasFocus(),
|
hasFocus: document.hasFocus(),
|
||||||
activeTag: active instanceof Element ? active.tagName.toLowerCase() : ''
|
activeTag: active instanceof Element ? active.tagName.toLowerCase() : '',
|
||||||
|
textEntry:
|
||||||
|
(active instanceof HTMLInputElement && !nonTextInputTypes.has(inputType)) ||
|
||||||
|
active instanceof HTMLTextAreaElement ||
|
||||||
|
(active instanceof HTMLElement && active.isContentEditable) ||
|
||||||
|
role === 'textbox' || role === 'searchbox'
|
||||||
};
|
};
|
||||||
})()`,
|
})()`,
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
@@ -302,16 +320,17 @@ async function resolveFocusedInputContext(
|
|||||||
// A top document whose active element is an iframe delegates focus to
|
// A top document whose active element is an iframe delegates focus to
|
||||||
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
// that frame. Removing those delegating targets leaves the deepest OOPIF
|
||||||
// target while same-process frames continue to use the shared root target.
|
// target while same-process frames continue to use the shared root target.
|
||||||
const owners = focused.filter(
|
const owners = focused.filter(({ probe }) => {
|
||||||
({ probe }) => probe.activeTag !== "iframe" && probe.activeTag !== "frame",
|
if (probe.activeTag === "iframe" || probe.activeTag === "frame") return false;
|
||||||
);
|
return mode !== "type" || probe.textEntry;
|
||||||
|
});
|
||||||
if (owners.length !== 1) {
|
if (owners.length !== 1) {
|
||||||
return rpcError(
|
return rpcError(
|
||||||
"cdp_failed",
|
"cdp_failed",
|
||||||
"focus_target_unresolved",
|
"focus_target_unresolved",
|
||||||
owners.length === 0
|
owners.length === 0
|
||||||
? "press found no unique focused frame target"
|
? `${mode} found no unique focused${mode === "type" ? " text-entry" : " frame"} target`
|
||||||
: "press found multiple focused frame targets",
|
: `${mode} found multiple focused frame targets`,
|
||||||
{ candidate_count: owners.length },
|
{ candidate_count: owners.length },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -682,6 +701,47 @@ function isFillable(node: DescribedNode): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** `type` also supports custom text-entry widgets that expose an ARIA role. */
|
||||||
|
function isTypeTarget(node: DescribedNode): boolean {
|
||||||
|
const tag = (node.nodeName ?? "").toUpperCase();
|
||||||
|
const attrs = node.attributes ?? [];
|
||||||
|
const attributes = new Map<string, string>();
|
||||||
|
for (let i = 0; i + 1 < attrs.length; i += 2) {
|
||||||
|
attributes.set(attrs[i].toLowerCase(), attrs[i + 1].toLowerCase());
|
||||||
|
}
|
||||||
|
if (tag === "TEXTAREA") return true;
|
||||||
|
if (tag === "INPUT") {
|
||||||
|
const type = attributes.get("type") ?? "text";
|
||||||
|
return !new Set([
|
||||||
|
"button",
|
||||||
|
"checkbox",
|
||||||
|
"color",
|
||||||
|
"date",
|
||||||
|
"datetime-local",
|
||||||
|
"file",
|
||||||
|
"hidden",
|
||||||
|
"image",
|
||||||
|
"month",
|
||||||
|
"radio",
|
||||||
|
"range",
|
||||||
|
"reset",
|
||||||
|
"submit",
|
||||||
|
"time",
|
||||||
|
"week",
|
||||||
|
]).has(type);
|
||||||
|
}
|
||||||
|
const contentEditable = attributes.get("contenteditable");
|
||||||
|
if (
|
||||||
|
contentEditable === "" ||
|
||||||
|
contentEditable === "true" ||
|
||||||
|
contentEditable === "plaintext-only"
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const role = attributes.get("role");
|
||||||
|
return role === "textbox" || role === "searchbox";
|
||||||
|
}
|
||||||
|
|
||||||
type SelectMutationResult =
|
type SelectMutationResult =
|
||||||
| {
|
| {
|
||||||
ok: true;
|
ok: true;
|
||||||
@@ -767,125 +827,114 @@ export async function handleFill(
|
|||||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||||
const objectId = objectIdOrErr;
|
const objectId = objectIdOrErr;
|
||||||
const clearBefore = params.clear_before ?? true;
|
const clearBefore = params.clear_before ?? true;
|
||||||
|
const fillResult = await fillEditable(
|
||||||
try {
|
nodeCdp,
|
||||||
if (clearBefore) {
|
target.tabId,
|
||||||
// Clear input/textarea value or wipe contenteditable innerText,
|
objectId,
|
||||||
// then fire `input` so frameworks observe the empty state.
|
params.value,
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
clearBefore,
|
||||||
objectId,
|
deps.signal,
|
||||||
functionDeclaration: `function() {
|
);
|
||||||
if (this.isContentEditable) { this.textContent = ''; }
|
if (isRpcError(fillResult)) return fillResult;
|
||||||
else {
|
|
||||||
const proto = this instanceof HTMLTextAreaElement
|
|
||||||
? HTMLTextAreaElement.prototype
|
|
||||||
: HTMLInputElement.prototype;
|
|
||||||
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value');
|
|
||||||
if (descriptor && descriptor.set) descriptor.set.call(this, '');
|
|
||||||
else this.value = '';
|
|
||||||
}
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
// CDP `Input.insertText` handles IME / multi-byte input out of the
|
|
||||||
// box, much more reliably than per-key `dispatchKeyEvent`.
|
|
||||||
await nodeCdp.send(target.tabId, "Input.insertText", { text: params.value });
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
||||||
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
// Give controlled inputs one bounded browser-render turn to accept or
|
|
||||||
// reject the dispatched value before reporting success.
|
|
||||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
/* bsk-input-sync */
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
let settled = false;
|
|
||||||
const finish = () => {
|
|
||||||
if (settled) return;
|
|
||||||
settled = true;
|
|
||||||
clearTimeout(timer);
|
|
||||||
resolve();
|
|
||||||
};
|
|
||||||
const timer = setTimeout(finish, 100);
|
|
||||||
if (typeof requestAnimationFrame !== 'function') { finish(); return; }
|
|
||||||
requestAnimationFrame(() => requestAnimationFrame(finish));
|
|
||||||
});
|
|
||||||
}`,
|
|
||||||
awaitPromise: true,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
return {
|
|
||||||
code: "cdp_failed",
|
|
||||||
message: err instanceof Error ? err.message : String(err),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (throwIfAborted(deps.signal)) {
|
|
||||||
return { code: "cancelled", message: "fill aborted" };
|
|
||||||
}
|
|
||||||
|
|
||||||
let actualValue: string;
|
|
||||||
try {
|
|
||||||
const readback = await nodeCdp.send<{
|
|
||||||
result?: { value?: { supported?: boolean; value?: string } };
|
|
||||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
|
||||||
objectId,
|
|
||||||
functionDeclaration: `function() {
|
|
||||||
/* bsk-input-readback */
|
|
||||||
if (this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement) {
|
|
||||||
return { supported: true, value: this.value };
|
|
||||||
}
|
|
||||||
if (this.isContentEditable) {
|
|
||||||
return { supported: true, value: this.innerText ?? this.textContent ?? '' };
|
|
||||||
}
|
|
||||||
return { supported: false, value: '' };
|
|
||||||
}`,
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
const state = readback.result?.value;
|
|
||||||
actualValue = typeof state?.value === "string" ? state.value : "";
|
|
||||||
const applied =
|
|
||||||
state?.supported === true &&
|
|
||||||
(clearBefore
|
|
||||||
? actualValue === params.value
|
|
||||||
: params.value.length === 0 || actualValue.includes(params.value));
|
|
||||||
if (!applied) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"input_not_applied",
|
|
||||||
"the page did not retain the requested input value",
|
|
||||||
{ expected_value_length: params.value.length, actual_value_length: actualValue.length },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
return rpcError(
|
|
||||||
"cdp_failed",
|
|
||||||
"input_not_applied",
|
|
||||||
err instanceof Error ? err.message : String(err),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
used_ref: input.usedRef,
|
used_ref: input.usedRef,
|
||||||
used_selector: input.usedSelector,
|
used_selector: input.usedSelector,
|
||||||
value_length: actualValue.length,
|
value_length: fillResult.value.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tool.type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export async function handleType(
|
||||||
|
manager: SessionManager,
|
||||||
|
params: TypeParams,
|
||||||
|
deps: InteractionDeps = getDefaultDeps(),
|
||||||
|
): Promise<TypeResult | RpcError> {
|
||||||
|
if (!params || typeof params.text !== "string") {
|
||||||
|
return { code: "invalid_params", message: "type requires a text string" };
|
||||||
|
}
|
||||||
|
if (params.focused !== undefined && typeof params.focused !== "boolean") {
|
||||||
|
return { code: "invalid_params", message: "type focused must be a boolean" };
|
||||||
|
}
|
||||||
|
const hasExplicitTarget =
|
||||||
|
(typeof params.ref === "string" && params.ref.length > 0) ||
|
||||||
|
(typeof params.selector === "string" && params.selector.length > 0);
|
||||||
|
if (params.focused === true && hasExplicitTarget) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "type: pass a ref/selector or focused=true, not both",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (params.focused !== true && !hasExplicitTarget) {
|
||||||
|
return {
|
||||||
|
code: "invalid_params",
|
||||||
|
message: "type requires a ref/selector or focused=true",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const ctxOrErr = lookupSession(manager, params, "type");
|
||||||
|
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||||
|
const ctx = ctxOrErr;
|
||||||
|
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||||
|
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||||
|
if (isRpcError(target)) return target;
|
||||||
|
const denied = enforceAgentWindow(ctx, target, "type");
|
||||||
|
if (denied) return denied;
|
||||||
|
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||||
|
|
||||||
|
let input: InputDispatchContext | RpcError;
|
||||||
|
if (hasExplicitTarget) {
|
||||||
|
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "type");
|
||||||
|
const explicitInput = inputContextForNode(deps.cdp, node);
|
||||||
|
if (isRpcError(explicitInput)) return explicitInput;
|
||||||
|
input = explicitInput;
|
||||||
|
try {
|
||||||
|
const described = await explicitInput.cdp.send<{ node?: DescribedNode }>(
|
||||||
|
target.tabId,
|
||||||
|
"DOM.describeNode",
|
||||||
|
{ backendNodeId: explicitInput.backendNodeId },
|
||||||
|
);
|
||||||
|
if (!described.node || !isTypeTarget(described.node)) {
|
||||||
|
return rpcError(
|
||||||
|
"invalid_params",
|
||||||
|
"target_not_fillable",
|
||||||
|
`element ${described.node?.nodeName ?? "?"} not text-editable`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const scrollErr = await scrollElementAndFramesIntoView(
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
explicitInput.target,
|
||||||
|
explicitInput.backendNodeId,
|
||||||
|
explicitInput.frameId,
|
||||||
|
);
|
||||||
|
if (scrollErr) return scrollErr;
|
||||||
|
if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "type aborted" };
|
||||||
|
await explicitInput.cdp.send(target.tabId, "DOM.focus", {
|
||||||
|
backendNodeId: explicitInput.backendNodeId,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "type");
|
||||||
|
if (isRpcError(input)) return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
|
const dispatchError = await dispatchTextInput(input.cdp, target.tabId, params.text, deps.signal);
|
||||||
|
if (dispatchError) return dispatchError;
|
||||||
|
|
||||||
|
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||||
|
tab_id: target.tabId,
|
||||||
|
used_ref: input.usedRef,
|
||||||
|
used_selector: input.usedSelector,
|
||||||
|
text_length: params.text.length,
|
||||||
|
verification: "dispatched",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1090,7 +1139,7 @@ export async function handlePress(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
input = await resolveFocusedInputContext(deps.cdp, target.tabId);
|
input = await resolveFocusedInputContext(deps.cdp, target.tabId, "press");
|
||||||
if (isRpcError(input)) return input;
|
if (isRpcError(input)) return input;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
// Browser-native text entry for live editing sessions. Unlike fill, this
|
||||||
|
// module does not assume that the focused editable retains a DOM value: a
|
||||||
|
// Canvas editor may use an input/textarea only as a transient keyboard/IME
|
||||||
|
// sink and clear it after consuming each event.
|
||||||
|
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
|
||||||
|
function cancelled(signal: AbortSignal | undefined): RpcError | null {
|
||||||
|
return signal?.aborted ? { code: "cancelled", message: "type aborted" } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface KeyDescriptor {
|
||||||
|
key: string;
|
||||||
|
code: string;
|
||||||
|
windowsVirtualKeyCode: number;
|
||||||
|
nativeVirtualKeyCode?: number;
|
||||||
|
modifiers: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function asciiKeyDescriptor(character: string): KeyDescriptor {
|
||||||
|
if (/^[a-z]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Key${character.toUpperCase()}`,
|
||||||
|
windowsVirtualKeyCode: character.toUpperCase().charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (/^[A-Z]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Key${character}`,
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 8,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (/^[0-9]$/.test(character)) {
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: `Digit${character}`,
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (character === " ") {
|
||||||
|
return { key: " ", code: "Space", windowsVirtualKeyCode: 32, modifiers: 0 };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
key: character,
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: character.charCodeAt(0),
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run one complete browser key transaction and always release the key. The
|
||||||
|
* commit callback owns the text-producing phase: a `char` event for a
|
||||||
|
* keyboard character, or `Input.insertText` for committed IME text.
|
||||||
|
*/
|
||||||
|
async function dispatchKeyTransaction(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
descriptor: KeyDescriptor,
|
||||||
|
commit: () => Promise<void>,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
let keyDown = false;
|
||||||
|
try {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "rawKeyDown",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
keyDown = true;
|
||||||
|
const abortedAfterDown = cancelled(signal);
|
||||||
|
if (!abortedAfterDown) await commit();
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
keyDown = false;
|
||||||
|
return abortedAfterDown ?? cancelled(signal);
|
||||||
|
} catch (err) {
|
||||||
|
if (keyDown) {
|
||||||
|
try {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "keyUp",
|
||||||
|
...descriptor,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup keeps the page from observing a stuck key.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function typeAsciiCharacter(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
character: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const descriptor = asciiKeyDescriptor(character);
|
||||||
|
return dispatchKeyTransaction(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
descriptor,
|
||||||
|
async () => {
|
||||||
|
await cdp.send(tabId, "Input.dispatchKeyEvent", {
|
||||||
|
type: "char",
|
||||||
|
...descriptor,
|
||||||
|
text: character,
|
||||||
|
unmodifiedText: character,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function typeImeSegment(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
text: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
// A committed IME value is still part of a keyboard transaction. The
|
||||||
|
// Process key (VK_PROCESSKEY / 229) is the platform-neutral signal that an
|
||||||
|
// IME owns the keystroke. It lets selection-based editors enter text mode
|
||||||
|
// before insertText commits the Unicode value, without inventing a visible
|
||||||
|
// ASCII prefix or leaving a composition transaction open.
|
||||||
|
const processKey: KeyDescriptor = {
|
||||||
|
key: "Process",
|
||||||
|
code: "",
|
||||||
|
windowsVirtualKeyCode: 229,
|
||||||
|
nativeVirtualKeyCode: 229,
|
||||||
|
modifiers: 0,
|
||||||
|
};
|
||||||
|
return dispatchKeyTransaction(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
processKey,
|
||||||
|
() => cdp.send(tabId, "Input.insertText", { text }).then(() => undefined),
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch text through the browser's keyboard/IME pipelines. ASCII runs are
|
||||||
|
* delivered as physical-key-shaped events; non-ASCII runs use CDP's committed
|
||||||
|
* IME/software-keyboard insertion path. Successful return means the browser
|
||||||
|
* accepted the input commands, not that a DOM value retained the text.
|
||||||
|
*/
|
||||||
|
export async function dispatchTextInput(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
text: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<RpcError | null> {
|
||||||
|
const abortedBefore = cancelled(signal);
|
||||||
|
if (abortedBefore) return abortedBefore;
|
||||||
|
|
||||||
|
const segments: Array<{ ime: boolean; text: string }> = [];
|
||||||
|
for (const character of text) {
|
||||||
|
const ime = character.codePointAt(0)! > 0x7f;
|
||||||
|
const previous = segments.at(-1);
|
||||||
|
if (previous?.ime === ime) previous.text += character;
|
||||||
|
else segments.push({ ime, text: character });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const segment of segments) {
|
||||||
|
if (segment.ime) {
|
||||||
|
const error = await typeImeSegment(cdp, tabId, segment.text, signal);
|
||||||
|
if (error) return error;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (const character of segment.text) {
|
||||||
|
const error = await typeAsciiCharacter(cdp, tabId, character, signal);
|
||||||
|
if (error) return error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
code: "cdp_failed",
|
||||||
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -484,6 +484,26 @@ export interface FillResult {
|
|||||||
dialogs?: JavaScriptDialogInfo[];
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TypeParams {
|
||||||
|
session_id: string;
|
||||||
|
text: string;
|
||||||
|
ref?: string;
|
||||||
|
selector?: string;
|
||||||
|
/** Explicitly dispatch to the unique focused text-entry target. */
|
||||||
|
focused?: boolean;
|
||||||
|
tab_id?: number;
|
||||||
|
timeout_ms?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TypeResult {
|
||||||
|
tab_id: number;
|
||||||
|
used_ref?: string;
|
||||||
|
used_selector?: string;
|
||||||
|
text_length: number;
|
||||||
|
verification: "dispatched";
|
||||||
|
dialogs?: JavaScriptDialogInfo[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface PressParams {
|
export interface PressParams {
|
||||||
session_id: string;
|
session_id: string;
|
||||||
key: string;
|
key: string;
|
||||||
|
|||||||
@@ -97,14 +97,25 @@ Run with a `900x700` Agent Window.
|
|||||||
|
|
||||||
This page exercises generic frame-aware input routing. It contains a top-level
|
This page exercises generic frame-aware input routing. It contains a top-level
|
||||||
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
input plus same-origin, cross-origin (OOPIF), and nested frame inputs. Each
|
||||||
frame includes an input, textarea, contenteditable editor, and a controlled
|
frame includes an input, textarea, contenteditable editor, an input that
|
||||||
input that resets its value on the next animation frame.
|
replaces its DOM node after accepted input, an input that rejects synthetic
|
||||||
|
DOM events, a transient keyboard/IME input proxy that consumes and clears its
|
||||||
|
DOM value, and a controlled input that resets its value on the next animation
|
||||||
|
frame.
|
||||||
|
|
||||||
- `fill` must route focus, text insertion, DOM events, and readback through the
|
- `fill` must route focus, text insertion, DOM events, and readback through the
|
||||||
ref's owning CDP target.
|
ref's owning CDP target.
|
||||||
- `press --ref` must focus and dispatch every key event through that same target.
|
- `press --ref` must focus and dispatch every key event through that same target.
|
||||||
- Untargeted `press` must follow the actual focused target or refuse when focus
|
- Untargeted `press` must follow the actual focused target or refuse when focus
|
||||||
ownership cannot be determined uniquely.
|
ownership cannot be determined uniquely.
|
||||||
|
- The remounting input must retain the value through its replacement node.
|
||||||
|
- The trusted input must be filled without synthetic `input` / `change` events.
|
||||||
|
- `type` must deliver ASCII key events and committed non-ASCII IME/software
|
||||||
|
keyboard insertion to the input proxy. Its DOM value remains empty while
|
||||||
|
`proxy output` retains the consumed text; the result reports
|
||||||
|
`verification=dispatched`, not a retained DOM value.
|
||||||
|
- `type` must reject an omitted target unless `focused=true` is explicit, and
|
||||||
|
focused mode must reject focus that is not owned by a text-entry target.
|
||||||
- The rejecting input must return `input_not_applied`, never a successful
|
- The rejecting input must return `input_not_applied`, never a successful
|
||||||
requested-value length.
|
requested-value length.
|
||||||
|
|
||||||
|
|||||||
@@ -14,9 +14,38 @@
|
|||||||
<label>Frame input <input aria-label="frame input" /></label>
|
<label>Frame input <input aria-label="frame input" /></label>
|
||||||
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
<label>Frame textarea <textarea aria-label="frame textarea"></textarea></label>
|
||||||
<div contenteditable="true" aria-label="frame editor"></div>
|
<div contenteditable="true" aria-label="frame editor"></div>
|
||||||
|
<label>Remounting input <span id="remount-host"><input aria-label="remounting input" /></span></label>
|
||||||
|
<label>Trusted input <input id="trusted" aria-label="trusted input" /></label>
|
||||||
|
<label>Input proxy <input id="proxy" aria-label="input proxy" /></label>
|
||||||
|
<output id="proxy-output" aria-label="proxy output">proxy output: empty</output>
|
||||||
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
<label>Rejecting input <input id="rejecting" aria-label="rejecting input" /></label>
|
||||||
<script>
|
<script>
|
||||||
const params = new URLSearchParams(location.search);
|
const params = new URLSearchParams(location.search);
|
||||||
|
const remountHost = document.querySelector("#remount-host");
|
||||||
|
const armRemount = (input) => {
|
||||||
|
input.addEventListener("input", () => {
|
||||||
|
const replacement = document.createElement("input");
|
||||||
|
replacement.setAttribute("aria-label", "remounting input");
|
||||||
|
replacement.value = input.value;
|
||||||
|
armRemount(replacement);
|
||||||
|
input.replaceWith(replacement);
|
||||||
|
replacement.focus();
|
||||||
|
}, { once: true });
|
||||||
|
};
|
||||||
|
armRemount(remountHost.querySelector("input"));
|
||||||
|
const trusted = document.querySelector("#trusted");
|
||||||
|
trusted.addEventListener("input", (event) => {
|
||||||
|
if (!event.isTrusted) requestAnimationFrame(() => { trusted.value = ""; });
|
||||||
|
});
|
||||||
|
const proxy = document.querySelector("#proxy");
|
||||||
|
const proxyOutput = document.querySelector("#proxy-output");
|
||||||
|
let proxyValue = "";
|
||||||
|
proxy.addEventListener("input", (event) => {
|
||||||
|
if (event.isComposing) return;
|
||||||
|
proxyValue += event.data ?? proxy.value;
|
||||||
|
proxy.value = "";
|
||||||
|
proxyOutput.textContent = `proxy output: ${proxyValue || "empty"}`;
|
||||||
|
});
|
||||||
const rejecting = document.querySelector("#rejecting");
|
const rejecting = document.querySelector("#rejecting");
|
||||||
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
rejecting.addEventListener("input", () => requestAnimationFrame(() => { rejecting.value = ""; }));
|
||||||
if (params.get("nested") === "1") {
|
if (params.get("nested") === "1") {
|
||||||
|
|||||||
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
|||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||||
bsk observe --session <id> → again after navigation / DOM change
|
bsk observe --session <id> → again after navigation / DOM change
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Choosing `fill`, `type`, and `press`
|
||||||
|
|
||||||
|
Choose by the intended postcondition, not by whether observation happens to
|
||||||
|
render `textbox [empty]`:
|
||||||
|
|
||||||
|
- `fill` sets a field's retained value. Use it when success means the target
|
||||||
|
value must equal the requested text; it replaces existing content by default.
|
||||||
|
- `type` inserts text at the active caret/selection in a live editing session.
|
||||||
|
Use it when the application must consume keyboard/IME text; it never clears
|
||||||
|
first and reports dispatch only.
|
||||||
|
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||||
|
Do not encode special keys inside `type --text`.
|
||||||
|
|
||||||
|
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||||
|
available, or pass `--focused` only after an immediately preceding action in
|
||||||
|
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||||
|
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||||
|
application may already have consumed the text before clearing its proxy DOM
|
||||||
|
value. Re-observe first. After `type`, verify the application output with one
|
||||||
|
observation or screenshot; `verification=dispatched` is not business success.
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
|||||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||||
```
|
```
|
||||||
|
|
||||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
|||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||||
|
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||||
|
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
|||||||
use bsk_protocol::Method;
|
use bsk_protocol::Method;
|
||||||
use bsk_protocol::tools::{
|
use bsk_protocol::tools::{
|
||||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||||
};
|
};
|
||||||
use clap::{Args, ValueEnum};
|
use clap::{Args, ValueEnum};
|
||||||
|
|
||||||
@@ -95,6 +95,27 @@ fn split_target(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn split_type_target(
|
||||||
|
positional: Option<String>,
|
||||||
|
explicit_ref: Option<String>,
|
||||||
|
explicit_selector: Option<String>,
|
||||||
|
focused: bool,
|
||||||
|
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||||
|
if focused {
|
||||||
|
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"pass a target or --focused, not both"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok((None, None));
|
||||||
|
}
|
||||||
|
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||||
|
CliError::Local(anyhow::anyhow!(
|
||||||
|
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// bsk click
|
// bsk click
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// bsk type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Args)]
|
||||||
|
#[group(id = "type_destination", required = true, multiple = false)]
|
||||||
|
pub struct TypeDestinationArgs {
|
||||||
|
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||||
|
#[arg(value_name = "TARGET")]
|
||||||
|
pub target: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long = "ref")]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long = "selector")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
|
||||||
|
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||||
|
#[arg(long)]
|
||||||
|
pub focused: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Args)]
|
||||||
|
pub struct TypeArgs {
|
||||||
|
#[command(flatten)]
|
||||||
|
pub destination: TypeDestinationArgs,
|
||||||
|
|
||||||
|
/// Text to dispatch through the live keyboard/IME editing session.
|
||||||
|
#[arg(long)]
|
||||||
|
pub text: String,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
pub session: String,
|
||||||
|
|
||||||
|
#[arg(long = "tab-id")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
|
||||||
|
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||||
|
pub timeout: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||||
|
let (ref_, selector) = split_type_target(
|
||||||
|
args.destination.target.clone(),
|
||||||
|
args.destination.ref_.clone(),
|
||||||
|
args.destination.selector.clone(),
|
||||||
|
args.destination.focused,
|
||||||
|
)?;
|
||||||
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
|
let params = TypeParams {
|
||||||
|
session_id: args.session.clone(),
|
||||||
|
text: args.text.clone(),
|
||||||
|
ref_,
|
||||||
|
selector,
|
||||||
|
focused: args.destination.focused,
|
||||||
|
tab_id: args.tab_id,
|
||||||
|
timeout_ms: Some(args.timeout),
|
||||||
|
};
|
||||||
|
let reply: TypeResult = call(
|
||||||
|
info.sock_path,
|
||||||
|
Method::ToolType,
|
||||||
|
params,
|
||||||
|
"type-1",
|
||||||
|
args.timeout,
|
||||||
|
)?;
|
||||||
|
match format {
|
||||||
|
Format::Json => {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&reply)
|
||||||
|
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Format::Human => {
|
||||||
|
let target = if args.destination.focused {
|
||||||
|
"focused".into()
|
||||||
|
} else {
|
||||||
|
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||||
|
};
|
||||||
|
println!(
|
||||||
|
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||||
|
reply.tab_id, reply.text_length
|
||||||
|
);
|
||||||
|
print_dialog_summaries(&reply.dialogs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// bsk press
|
// bsk press
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -659,6 +769,52 @@ mod tests {
|
|||||||
assert_eq!(parsed.args.session, "ohli");
|
assert_eq!(parsed.args.session, "ohli");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||||
|
use clap::Parser;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(name = "type")]
|
||||||
|
struct Wrapper {
|
||||||
|
#[command(flatten)]
|
||||||
|
args: TypeArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed =
|
||||||
|
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||||
|
.expect("type args should parse");
|
||||||
|
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||||
|
assert_eq!(parsed.args.text, "测试");
|
||||||
|
assert_eq!(parsed.args.session, "ohli");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||||
|
use clap::Parser;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(name = "type")]
|
||||||
|
struct Wrapper {
|
||||||
|
#[command(flatten)]
|
||||||
|
args: TypeArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed =
|
||||||
|
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||||
|
.expect("focused type args should parse");
|
||||||
|
assert!(parsed.args.destination.focused);
|
||||||
|
assert!(parsed.args.destination.target.is_none());
|
||||||
|
|
||||||
|
let focused = split_type_target(None, None, None, true).unwrap();
|
||||||
|
assert_eq!(focused, (None, None));
|
||||||
|
|
||||||
|
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||||
|
assert!(missing.to_string().contains("--focused"));
|
||||||
|
|
||||||
|
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||||
|
assert!(mixed.to_string().contains("not both"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
|||||||
use crate::cli::get_html::GetHtmlArgs;
|
use crate::cli::get_html::GetHtmlArgs;
|
||||||
use crate::cli::human_loop::RequestHelpArgs;
|
use crate::cli::human_loop::RequestHelpArgs;
|
||||||
use crate::cli::install_skill::InstallSkillArgs;
|
use crate::cli::install_skill::InstallSkillArgs;
|
||||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||||
use crate::cli::network::NetworkArgs;
|
use crate::cli::network::NetworkArgs;
|
||||||
use crate::cli::observe::ObserveArgs;
|
use crate::cli::observe::ObserveArgs;
|
||||||
@@ -171,6 +171,9 @@ pub enum Command {
|
|||||||
/// Fill an input / textarea / contenteditable.
|
/// Fill an input / textarea / contenteditable.
|
||||||
Fill(FillArgs),
|
Fill(FillArgs),
|
||||||
|
|
||||||
|
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||||
|
Type(TypeArgs),
|
||||||
|
|
||||||
/// Dispatch a keyboard key combo.
|
/// Dispatch a keyboard key combo.
|
||||||
Press(PressArgs),
|
Press(PressArgs),
|
||||||
|
|
||||||
|
|||||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
|||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolHover
|
| Method::ToolHover
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect
|
| Method::ToolSelect
|
||||||
| Method::ToolEvaluate
|
| Method::ToolEvaluate
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
|||||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||||
|
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
|
||||||
//! through the IPC + per-session queue + fake extension and assert
|
//! through the IPC + per-session queue + fake extension and assert
|
||||||
//! the wire shapes line up. The extension stub mirrors each method's
|
//! the wire shapes line up. The extension stub mirrors each method's
|
||||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
|||||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||||
|
WaitUntil,
|
||||||
};
|
};
|
||||||
use bsk_protocol::{
|
use bsk_protocol::{
|
||||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
|||||||
handle.shutdown().await;
|
handle.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn type_round_trips_dispatched_verification() {
|
||||||
|
let (handle, sock) = spawn_daemon().await;
|
||||||
|
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||||
|
let _ = do_handshake(&mut ws).await;
|
||||||
|
|
||||||
|
run_extension(ws, |req| {
|
||||||
|
assert_eq!(req.method, Method::ToolType);
|
||||||
|
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||||
|
assert_eq!(p.text, "测试");
|
||||||
|
assert!(!p.focused);
|
||||||
|
ResponseBody::Ok(
|
||||||
|
serde_json::to_value(TypeResult {
|
||||||
|
tab_id: 12,
|
||||||
|
used_ref: Some("e3".into()),
|
||||||
|
used_selector: None,
|
||||||
|
text_length: 2,
|
||||||
|
verification: TypeVerification::Dispatched,
|
||||||
|
dialogs: vec![],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
let session_id = ipc_session_start(&sock).await;
|
||||||
|
let result: TypeResult = ipc_tool_call(
|
||||||
|
&sock,
|
||||||
|
Method::ToolType,
|
||||||
|
TypeParams {
|
||||||
|
session_id,
|
||||||
|
text: "测试".into(),
|
||||||
|
ref_: Some("@e3".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("type ok");
|
||||||
|
assert_eq!(result.text_length, 2);
|
||||||
|
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||||
|
handle.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn press_round_trips_compound_key() {
|
async fn press_round_trips_compound_key() {
|
||||||
let (handle, sock) = spawn_daemon().await;
|
let (handle, sock) = spawn_daemon().await;
|
||||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
|||||||
| Method::ToolReload
|
| Method::ToolReload
|
||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||||
code: ErrorCode::CdpFailed,
|
code: ErrorCode::CdpFailed,
|
||||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
|||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||||
|
|
||||||
|
let type_result = ipc_tool_call::<_, Value>(
|
||||||
|
&sock,
|
||||||
|
Method::ToolType,
|
||||||
|
TypeParams {
|
||||||
|
session_id: session_id.clone(),
|
||||||
|
text: "hello".into(),
|
||||||
|
ref_: Some("@e1".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||||
|
|
||||||
let press = ipc_tool_call::<_, Value>(
|
let press = ipc_tool_call::<_, Value>(
|
||||||
&sock,
|
&sock,
|
||||||
Method::ToolPress,
|
Method::ToolPress,
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
"format": "int64"
|
"format": "int64"
|
||||||
},
|
},
|
||||||
"text": {
|
"text": {
|
||||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
|
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"truncated": {
|
"truncated": {
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "TypeParams",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"session_id",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"focused": {
|
||||||
|
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"ref": {
|
||||||
|
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"session_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"timeout_ms": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 1.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "TypeResult",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"tab_id",
|
||||||
|
"text_length",
|
||||||
|
"verification"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"dialogs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"text_length": {
|
||||||
|
"description": "UTF-16 code-unit length of text accepted for dispatch.",
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"used_ref": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"used_selector": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"verification": {
|
||||||
|
"$ref": "#/definitions/TypeVerification"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"definitions": {
|
||||||
|
"JavaScriptDialogHandledAction": {
|
||||||
|
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"accepted",
|
||||||
|
"dismissed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"JavaScriptDialogInfo": {
|
||||||
|
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"handled",
|
||||||
|
"message",
|
||||||
|
"sequence",
|
||||||
|
"tab_id",
|
||||||
|
"type"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"default_prompt": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"handled": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||||
|
},
|
||||||
|
"has_browser_handler": {
|
||||||
|
"type": [
|
||||||
|
"boolean",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sequence": {
|
||||||
|
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint64",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"tab_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"type": {
|
||||||
|
"$ref": "#/definitions/JavaScriptDialogType"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"JavaScriptDialogType": {
|
||||||
|
"description": "Native JS dialog kind reported by CDP.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"alert",
|
||||||
|
"confirm",
|
||||||
|
"prompt",
|
||||||
|
"beforeunload"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"TypeVerification": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"dispatched"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -81,6 +81,8 @@ fn main() {
|
|||||||
dump!(HoverResult, "tool_hover_result");
|
dump!(HoverResult, "tool_hover_result");
|
||||||
dump!(FillParams, "tool_fill_params");
|
dump!(FillParams, "tool_fill_params");
|
||||||
dump!(FillResult, "tool_fill_result");
|
dump!(FillResult, "tool_fill_result");
|
||||||
|
dump!(TypeParams, "tool_type_params");
|
||||||
|
dump!(TypeResult, "tool_type_result");
|
||||||
dump!(PressParams, "tool_press_params");
|
dump!(PressParams, "tool_press_params");
|
||||||
dump!(PressResult, "tool_press_result");
|
dump!(PressResult, "tool_press_result");
|
||||||
dump!(SelectParams, "tool_select_params");
|
dump!(SelectParams, "tool_select_params");
|
||||||
|
|||||||
@@ -74,6 +74,8 @@ pub enum Method {
|
|||||||
ToolHover,
|
ToolHover,
|
||||||
#[serde(rename = "tool.fill")]
|
#[serde(rename = "tool.fill")]
|
||||||
ToolFill,
|
ToolFill,
|
||||||
|
#[serde(rename = "tool.type")]
|
||||||
|
ToolType,
|
||||||
#[serde(rename = "tool.press")]
|
#[serde(rename = "tool.press")]
|
||||||
ToolPress,
|
ToolPress,
|
||||||
#[serde(rename = "tool.select")]
|
#[serde(rename = "tool.select")]
|
||||||
@@ -154,6 +156,7 @@ impl Method {
|
|||||||
| Method::ToolReload
|
| Method::ToolReload
|
||||||
| Method::ToolClick
|
| Method::ToolClick
|
||||||
| Method::ToolFill
|
| Method::ToolFill
|
||||||
|
| Method::ToolType
|
||||||
| Method::ToolPress
|
| Method::ToolPress
|
||||||
| Method::ToolSelect
|
| Method::ToolSelect
|
||||||
| Method::ToolEvaluate
|
| Method::ToolEvaluate
|
||||||
@@ -246,6 +249,13 @@ mod tests {
|
|||||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_method_round_trips() {
|
||||||
|
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
|
||||||
|
assert_eq!(method, Method::ToolType);
|
||||||
|
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cancel_params_and_result_round_trip() {
|
fn cancel_params_and_result_round_trip() {
|
||||||
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
||||||
@@ -284,6 +294,7 @@ mod tests {
|
|||||||
assert!(Method::ToolReload.is_mutating());
|
assert!(Method::ToolReload.is_mutating());
|
||||||
assert!(Method::ToolClick.is_mutating());
|
assert!(Method::ToolClick.is_mutating());
|
||||||
assert!(Method::ToolFill.is_mutating());
|
assert!(Method::ToolFill.is_mutating());
|
||||||
|
assert!(Method::ToolType.is_mutating());
|
||||||
assert!(Method::ToolPress.is_mutating());
|
assert!(Method::ToolPress.is_mutating());
|
||||||
assert!(Method::ToolSelect.is_mutating());
|
assert!(Method::ToolSelect.is_mutating());
|
||||||
assert!(Method::ToolEvaluate.is_mutating());
|
assert!(Method::ToolEvaluate.is_mutating());
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
|
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
|
||||||
//! `tool.select`).
|
//! `tool.press`, `tool.select`).
|
||||||
//!
|
//!
|
||||||
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
||||||
//! normalised against the session's RefStore) **or** a CSS selector
|
//! normalised against the session's RefStore) **or** a CSS selector
|
||||||
@@ -188,6 +188,60 @@ pub struct FillResult {
|
|||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// type
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct TypeParams {
|
||||||
|
pub session_id: String,
|
||||||
|
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
|
||||||
|
/// does not require the focused DOM editable to retain this text.
|
||||||
|
pub text: String,
|
||||||
|
/// Target to focus before dispatching text. Exactly one targeting mode
|
||||||
|
/// must be selected: `ref` / `selector`, or `focused=true`.
|
||||||
|
#[serde(
|
||||||
|
rename = "ref",
|
||||||
|
alias = "ref_",
|
||||||
|
default,
|
||||||
|
skip_serializing_if = "Option::is_none"
|
||||||
|
)]
|
||||||
|
pub ref_: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub selector: Option<String>,
|
||||||
|
/// Explicitly dispatch to the unique focused text-entry target instead
|
||||||
|
/// of focusing a ref or selector.
|
||||||
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||||
|
pub focused: bool,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tab_id: Option<i64>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[schemars(range(min = 1))]
|
||||||
|
pub timeout_ms: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum TypeVerification {
|
||||||
|
/// Chromium accepted the complete keyboard/IME dispatch transaction.
|
||||||
|
/// The caller must observe the application output to verify business state.
|
||||||
|
Dispatched,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct TypeResult {
|
||||||
|
pub tab_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_ref: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub used_selector: Option<String>,
|
||||||
|
/// UTF-16 code-unit length of text accepted for dispatch.
|
||||||
|
pub text_length: u32,
|
||||||
|
pub verification: TypeVerification,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// press
|
// press
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -351,6 +405,50 @@ mod tests {
|
|||||||
assert_eq!(v, json!("meta"));
|
assert_eq!(v, json!("meta"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_payloads_distinguish_dispatch_from_value_verification() {
|
||||||
|
let params = TypeParams {
|
||||||
|
session_id: "abcd".into(),
|
||||||
|
text: "测试".into(),
|
||||||
|
ref_: Some("@e3".into()),
|
||||||
|
selector: None,
|
||||||
|
focused: false,
|
||||||
|
tab_id: Some(4),
|
||||||
|
timeout_ms: Some(5_000),
|
||||||
|
};
|
||||||
|
let encoded = serde_json::to_value(¶ms).unwrap();
|
||||||
|
assert_eq!(encoded["ref"], json!("@e3"));
|
||||||
|
assert_eq!(encoded["text"], json!("测试"));
|
||||||
|
assert!(encoded.get("focused").is_none());
|
||||||
|
|
||||||
|
let focused = TypeParams {
|
||||||
|
session_id: "abcd".into(),
|
||||||
|
text: "hello".into(),
|
||||||
|
ref_: None,
|
||||||
|
selector: None,
|
||||||
|
focused: true,
|
||||||
|
tab_id: None,
|
||||||
|
timeout_ms: None,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(focused).unwrap()["focused"],
|
||||||
|
json!(true)
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = TypeResult {
|
||||||
|
tab_id: 4,
|
||||||
|
used_ref: Some("e3".into()),
|
||||||
|
used_selector: None,
|
||||||
|
text_length: 2,
|
||||||
|
verification: TypeVerification::Dispatched,
|
||||||
|
dialogs: vec![],
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(result).unwrap()["verification"],
|
||||||
|
json!("dispatched")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn press_result_round_trips() {
|
fn press_result_round_trips() {
|
||||||
let r = PressResult {
|
let r = PressResult {
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
|
|||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct SnapshotResult {
|
pub struct SnapshotResult {
|
||||||
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
||||||
/// agent can copy them into subsequent `tool.click` / `tool.fill`
|
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
|
||||||
/// selectors.
|
/// `tool.type` targets.
|
||||||
pub text: String,
|
pub text: String,
|
||||||
/// Number of `@e<N>` refs registered for this session by this
|
/// Number of `@e<N>` refs registered for this session by this
|
||||||
/// snapshot. Equivalent to the size of the new ref-store map.
|
/// snapshot. Equivalent to the size of the new ref-store map.
|
||||||
|
|||||||
Reference in new issue
Block a user