mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
merge: resolve SKILL.md conflicts with main
Keep the expanded command-reference tables from feat/upload-download (including the new File Transfer section) over main's compact two-line format. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
drakezhang
@@ -7,6 +7,10 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-1",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
);
|
||||
|
||||
expect(screen.getByText("允许借用标签页？")).toBeTruthy();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-fallback",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
|
||||
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
|
||||
expect(message.requestId.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("proactively focuses the user window before sending the overlay message", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await pending;
|
||||
|
||||
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
|
||||
});
|
||||
|
||||
it("still proceeds when proactive window focus fails", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
windows.update.mockRejectedValueOnce(new Error("window gone"));
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(await pending).toBe(true);
|
||||
});
|
||||
|
||||
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
|
||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||
// AX captures would leave the two halves of one observation describing the
|
||||
// page on either side of that change.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const methodOrder: string[] = [];
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
methodOrder.push(method);
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
// Report the button as a `:hover` rule target so a real hover fires.
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||
}
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", probe_hover: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||
expect(firstHover).toBeGreaterThan(-1);
|
||||
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||
});
|
||||
|
||||
it("runs conditional surface probing only when observe opts in", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const withoutProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if ("code" in withoutProbe) {
|
||||
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||
}
|
||||
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
const withProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||
deps,
|
||||
);
|
||||
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface BorrowResponseMessage {
|
||||
allowed: boolean;
|
||||
}
|
||||
|
||||
export const CONFIRMATION_TIMEOUT_MS = 5000;
|
||||
export const CONFIRMATION_TIMEOUT_MS = 60_000;
|
||||
const EXIT_ANIMATION_MS = 150;
|
||||
/** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */
|
||||
export const PROGRESS_TRANSITION_MS = 1000;
|
||||
@@ -480,6 +480,13 @@ export async function requestBorrowConfirmation(
|
||||
settle(false);
|
||||
}, BACKGROUND_TIMEOUT_MS);
|
||||
|
||||
// Bring the user window to the front so the overlay is visible even
|
||||
// when the Agent Window has stolen focus. This is fire-and-forget: if
|
||||
// it fails the overlay / OS notification still give the user a path.
|
||||
void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => {
|
||||
console.debug("[bsk borrow] proactive focus of user window failed", err);
|
||||
});
|
||||
|
||||
// Surface the OS notification *before* messaging any candidate so the
|
||||
// user has a parallel signal even if every content script is missing.
|
||||
// The Allow / Deny buttons let the user authorize explicitly when the
|
||||
|
||||
@@ -387,7 +387,11 @@ export class ToolDispatcher {
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
// Active hover probing is opt-in. A held hover latch still
|
||||
// suppresses it, because probing would move the cursor off
|
||||
// the element the caller is deliberately holding.
|
||||
conditionalSurfaceProbe:
|
||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
|
||||
@@ -49,11 +49,14 @@ import {
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
|
||||
export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
supplementalNames: options.supplementalNames,
|
||||
});
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured);
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
|
||||
}
|
||||
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
|
||||
return {
|
||||
...scene,
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
@@ -1010,17 +1018,69 @@ async function captureForVom(
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
/**
|
||||
* Whether hovering surfaced content absent from the static snapshot. Cached
|
||||
* tooltip names count, so this may over-report on repeat observations — it
|
||||
* errs towards telling the caller the snapshot is less fresh than it looks.
|
||||
*/
|
||||
revealedContent: boolean;
|
||||
surfaceProbes: CapturedSurfaceProbe[];
|
||||
tooltipNames: Map<string, string>;
|
||||
}
|
||||
|
||||
const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
performed: false,
|
||||
revealedContent: false,
|
||||
surfaceProbes: [],
|
||||
tooltipNames: new Map(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs both active-hover chains back to back.
|
||||
*
|
||||
* Ordering matters: this happens after DOM *and* accessibility capture so a
|
||||
* hover that opens a menu cannot leave one half of the observation describing
|
||||
* the page before the change and the other half after it. Sharing one overlay
|
||||
* bypass span also means the agent overlay toggles once per observation
|
||||
* instead of once per chain.
|
||||
*/
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<HoverProbeOutcome> {
|
||||
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
|
||||
|
||||
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
|
||||
signal: options.signal,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return {
|
||||
performed: true,
|
||||
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
|
||||
surfaceProbes,
|
||||
tooltipNames,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
});
|
||||
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
||||
const tooltipNames = options.conditionalSurfaceProbe
|
||||
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
|
||||
signal: options.signal,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
})
|
||||
: new Map<string, string>();
|
||||
const hoverProbes = await runHoverProbes(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
normalizedDocuments,
|
||||
staticSemantics,
|
||||
options,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene = projectSemanticGraph(
|
||||
normalizeSemanticStructure(
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(
|
||||
scene,
|
||||
normalizedDocuments,
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
frameDocuments: normalizedDocuments,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
surfaceProbes: hoverProbes.surfaceProbes,
|
||||
hoverProbe: {
|
||||
performed: hoverProbes.performed,
|
||||
revealedContent: hoverProbes.revealedContent,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && observation.hoverProbe?.performed
|
||||
? {
|
||||
hover_probe: {
|
||||
performed: true,
|
||||
revealed_content: observation.hoverProbe.revealedContent,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
@@ -1179,5 +1258,13 @@ export async function handleObserve(
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
return handleVomObservation(
|
||||
manager,
|
||||
params,
|
||||
"observe",
|
||||
"transient_input",
|
||||
params.probe_hover === true,
|
||||
deps,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
|
||||
|
||||
afterEach(() => {
|
||||
resetOverlayBypassDepth();
|
||||
});
|
||||
|
||||
describe("ProbeBudget", () => {
|
||||
it("refuses a candidate that cannot finish inside the budget", () => {
|
||||
const budget = new ProbeBudget(500);
|
||||
|
||||
expect(budget.canAfford(400)).toBe(true);
|
||||
expect(budget.canAfford(600)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
|
||||
const budget = new ProbeBudget(60);
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
|
||||
// A top-of-loop `elapsed > total` check would still admit this candidate
|
||||
// and then overshoot by its full cost.
|
||||
expect(budget.canAfford(50)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withOverlayBypass", () => {
|
||||
it("toggles the overlay once around nested probe phases", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("restores the overlay when the probe throws", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
withOverlayBypass(bypass, 4, async () => {
|
||||
throw new Error("probe exploded");
|
||||
}),
|
||||
).rejects.toThrow("probe exploded");
|
||||
|
||||
expect(bypass).toHaveBeenLastCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("reports a failed restore instead of swallowing it", async () => {
|
||||
const onRestoreFailure = vi.fn();
|
||||
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
|
||||
|
||||
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not pin the overlay off for the tab when a restore fails", async () => {
|
||||
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
|
||||
|
||||
const bypass = vi.fn(async () => {});
|
||||
await withOverlayBypass(bypass, 4, async () => undefined);
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("runs the probe untouched when no bypass is wired up", async () => {
|
||||
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [
|
||||
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
|
||||
];
|
||||
const graph = semantics(documents);
|
||||
|
||||
const first = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(first.size).toBe(0);
|
||||
const hoversAfterFirst = vi
|
||||
.mocked(cdp.send)
|
||||
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
|
||||
expect(hoversAfterFirst).toBeGreaterThan(0);
|
||||
|
||||
const second = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(second.size).toBe(0);
|
||||
expect(
|
||||
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
|
||||
.length,
|
||||
).toBe(hoversAfterFirst);
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
/**
|
||||
* Ceiling for the whole hover-surface phase.
|
||||
*
|
||||
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
|
||||
* so a candidate could start with 1ms left and still run its two settle
|
||||
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
|
||||
* enforced by an up-front affordability check, so the same number of candidates
|
||||
* get probed under a bound that no longer lies.
|
||||
*/
|
||||
const MAX_HOVER_PROBE_MS = 2_600;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
@@ -614,13 +620,35 @@ function confidenceForHover(
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
/**
|
||||
* One candidate costs two settle windows (baseline + post-hover) plus a few
|
||||
* CDP round trips. Used to decide whether the next candidate still fits the
|
||||
* budget before paying for it.
|
||||
*/
|
||||
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
|
||||
|
||||
export interface HoverSurfaceProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hovers a bounded set of likely menu triggers and reports the sub-items each
|
||||
* one reveals.
|
||||
*
|
||||
* Runs after DOM *and* accessibility capture. Hovering can open menus and
|
||||
* change layout, so probing between the two captures would leave the DOM half
|
||||
* of an observation describing the page before the change and the AX half
|
||||
* describing it after.
|
||||
*
|
||||
* The caller owns the overlay bypass span (see `withOverlayBypass`).
|
||||
*/
|
||||
export async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
options: HoverSurfaceProbeOptions = {},
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: [],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wall-clock budget for a probe phase.
|
||||
*
|
||||
* Callers ask whether the *next* candidate still fits before starting it,
|
||||
* rather than checking elapsed time at the top of the loop. A top-of-loop
|
||||
* check lets a candidate start with 1ms of budget left and then run to
|
||||
* completion, overshooting by a full settle window.
|
||||
*/
|
||||
export class ProbeBudget {
|
||||
private readonly startedAt = Date.now();
|
||||
|
||||
constructor(private readonly totalMs: number) {}
|
||||
|
||||
get elapsedMs(): number {
|
||||
return Date.now() - this.startedAt;
|
||||
}
|
||||
|
||||
/** True when a candidate costing `estimatedMs` still fits in the budget. */
|
||||
canAfford(estimatedMs: number): boolean {
|
||||
return this.elapsedMs + estimatedMs <= this.totalMs;
|
||||
}
|
||||
}
|
||||
|
||||
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
|
||||
|
||||
export interface OverlayBypassScopeOptions {
|
||||
/**
|
||||
* Called when the overlay could not be restored. Defaults to `console.error`
|
||||
* so a stuck-bypassed overlay is never silent; tests inject a spy.
|
||||
*/
|
||||
onRestoreFailure?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
const overlayBypassDepth = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
|
||||
*
|
||||
* Reference counted per tab so nested probe phases share a single bypass span
|
||||
* instead of toggling the overlay once per phase. Restore failures are
|
||||
* reported rather than swallowed, and the depth is released even when restore
|
||||
* throws so a failure cannot pin the overlay off for the tab's lifetime.
|
||||
*/
|
||||
export async function withOverlayBypass<T>(
|
||||
bypass: OverlayBypass | undefined,
|
||||
tabId: number,
|
||||
body: () => Promise<T>,
|
||||
options: OverlayBypassScopeOptions = {},
|
||||
): Promise<T> {
|
||||
if (!bypass) return body();
|
||||
|
||||
const depth = overlayBypassDepth.get(tabId) ?? 0;
|
||||
overlayBypassDepth.set(tabId, depth + 1);
|
||||
if (depth === 0) {
|
||||
try {
|
||||
await bypass(tabId, true);
|
||||
} catch (error) {
|
||||
// Failing to hide the overlay only risks probing our own UI, so continue.
|
||||
console.debug("[bsk hover] overlay bypass could not be enabled", error);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await body();
|
||||
} finally {
|
||||
const current = overlayBypassDepth.get(tabId) ?? 1;
|
||||
if (current <= 1) {
|
||||
overlayBypassDepth.delete(tabId);
|
||||
try {
|
||||
await bypass(tabId, false);
|
||||
} catch (error) {
|
||||
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
|
||||
}
|
||||
} else {
|
||||
overlayBypassDepth.set(tabId, current - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRestoreFailure(error: unknown): void {
|
||||
console.error("[bsk hover] agent overlay could not be restored after probing", error);
|
||||
}
|
||||
|
||||
/** Test seam: drops any bypass depth left behind by a crashed probe. */
|
||||
export function resetOverlayBypassDepth(): void {
|
||||
overlayBypassDepth.clear();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||
|
||||
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
|
||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||
const TOOLTIP_SETTLE_MS = 250;
|
||||
const CACHE_TTL_MS = 5 * 60_000;
|
||||
/**
|
||||
* Misses expire sooner than hits: a control that gains a tooltip once the app
|
||||
* finishes booting should still be discoverable, but not at the cost of
|
||||
* re-probing every unlabelled control on every observation.
|
||||
*/
|
||||
const MISS_CACHE_TTL_MS = 60_000;
|
||||
const MAX_CACHE_ENTRIES = 500;
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||
|
||||
interface TooltipCacheEntry {
|
||||
name: string;
|
||||
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
|
||||
name: string | null;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
||||
|
||||
export interface TooltipNameProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
|
||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
function cachedName<T extends FrameOwnedAxNode>(
|
||||
/** Returns the cache entry, or `undefined` when this control was never probed. */
|
||||
function cachedEntry<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
): string | undefined {
|
||||
): TooltipCacheEntry | undefined {
|
||||
const key = cacheKey(document, backendNodeId);
|
||||
const entry = tooltipCache.get(key);
|
||||
if (!entry) return undefined;
|
||||
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
|
||||
tooltipCache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return entry.name;
|
||||
return entry;
|
||||
}
|
||||
|
||||
function storeName<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
name: string,
|
||||
name: string | null,
|
||||
): void {
|
||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||
const oldest = tooltipCache.keys().next().value;
|
||||
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||
name,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
const names = new Map<string, string>();
|
||||
const pending: TooltipCandidate<T>[] = [];
|
||||
for (const candidate of candidates(documents, graph)) {
|
||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
||||
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
|
||||
if (cached) {
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
||||
// A cached miss is still a decision: skip the control instead of re-probing.
|
||||
if (cached.name !== null) {
|
||||
names.set(
|
||||
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
|
||||
cached.name,
|
||||
);
|
||||
}
|
||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
if (pending.length === 0) return names;
|
||||
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
|
||||
const contexts = new Map<string, number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of pending) {
|
||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
||||
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
let contextId = contexts.get(candidate.document.frameId);
|
||||
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||
const name = cleanTooltipDiff(before, after);
|
||||
storeName(candidate.document, candidate.backendNodeId, name ?? null);
|
||||
if (!name) continue;
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||
storeName(candidate.document, candidate.backendNodeId, name);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
} finally {
|
||||
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/**
|
||||
* Whether this observation actively hovered the page, and whether that
|
||||
* revealed content the static tree does not contain. Lets callers judge how
|
||||
* far the page may have drifted from the returned snapshot.
|
||||
*/
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}
|
||||
|
||||
export interface HoverProbeReport {
|
||||
performed: boolean;
|
||||
revealedContent: boolean;
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -330,9 +330,14 @@ export interface SnapshotResult {
|
||||
|
||||
export interface ObserveParams extends SnapshotParams {
|
||||
debug_surfaces?: boolean;
|
||||
probe_hover?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult extends SnapshotResult {
|
||||
hover_probe?: {
|
||||
performed: boolean;
|
||||
revealed_content: boolean;
|
||||
};
|
||||
debug?: {
|
||||
surface_probes?: Array<{
|
||||
trigger_backend_node_id: number;
|
||||
|
||||
+103
-160
@@ -1,160 +1,125 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
@@ -303,48 +268,26 @@ bsk record start --browser <instance-id-or-label> \
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## Recover without wandering
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
@@ -33,6 +33,12 @@ pub struct ObserveArgs {
|
||||
/// Include conditional surface probe diagnostics in JSON output.
|
||||
#[arg(long = "debug-surfaces")]
|
||||
pub debug_surfaces: bool,
|
||||
|
||||
/// Actively hover page controls to reveal hover-only menus and tooltips.
|
||||
/// Costs seconds of wall clock and touches the live page, so it is off
|
||||
/// unless a static observation looks like it is missing hover content.
|
||||
#[arg(long = "probe-hover")]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
debug_surfaces: args.debug_surfaces,
|
||||
probe_hover: args.probe_hover,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
|
||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
tab_id: 13,
|
||||
truncated: false,
|
||||
dialogs: vec![],
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -494,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
max_depth: None,
|
||||
max_tokens: None,
|
||||
debug_surfaces: false,
|
||||
probe_hover: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ObserveParams",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
@@ -30,6 +30,11 @@
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"probe_hover": {
|
||||
"description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -24,6 +24,17 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"hover_probe": {
|
||||
"description": "Present only when `probe_hover` was requested and probing actually ran.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HoverProbeReport"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ref_count": {
|
||||
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
||||
"type": "integer",
|
||||
@@ -46,6 +57,24 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HoverProbeReport": {
|
||||
"description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"performed",
|
||||
"revealed_content"
|
||||
],
|
||||
"properties": {
|
||||
"performed": {
|
||||
"description": "The real cursor was moved over page content during this observation.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"revealed_content": {
|
||||
"description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.",
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
|
||||
@@ -58,8 +58,9 @@ pub struct SnapshotResult {
|
||||
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
||||
///
|
||||
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
||||
/// such as hover-surface discovery. It must not submit input, click,
|
||||
/// navigate, or otherwise commit page state.
|
||||
/// such as hover-surface discovery, but only when `probe_hover` asks for
|
||||
/// them. It must not submit input, click, navigate, or otherwise commit
|
||||
/// page state.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveParams {
|
||||
pub session_id: String,
|
||||
@@ -77,6 +78,16 @@ pub struct ObserveParams {
|
||||
/// Include implementation diagnostics for conditional surface probes.
|
||||
#[serde(default)]
|
||||
pub debug_surfaces: bool,
|
||||
/// Opt in to active hover probing: the extension moves the real cursor
|
||||
/// over a bounded set of controls to discover hover-only menus and
|
||||
/// tooltips.
|
||||
///
|
||||
/// Off by default. Probing costs seconds of wall clock, mutates the live
|
||||
/// page, and pays off only on pages whose content is genuinely hidden
|
||||
/// behind hover. Request it when a static observation looks like it is
|
||||
/// missing hover-revealed structure.
|
||||
#[serde(default)]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -102,6 +113,17 @@ pub struct ObserveDebug {
|
||||
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||
}
|
||||
|
||||
/// Reports what active hover probing did during an observation, so the agent
|
||||
/// can judge how far the live page may have drifted from the returned text.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverProbeReport {
|
||||
/// The real cursor was moved over page content during this observation.
|
||||
pub performed: bool,
|
||||
/// Hovering surfaced content the static tree does not contain, so the page
|
||||
/// reacts to the cursor and may not have fully reverted.
|
||||
pub revealed_content: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveResult {
|
||||
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
||||
@@ -118,6 +140,9 @@ pub struct ObserveResult {
|
||||
pub truncated: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
/// Present only when `probe_hover` was requested and probing actually ran.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub hover_probe: Option<HoverProbeReport>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub debug: Option<ObserveDebug>,
|
||||
}
|
||||
@@ -308,6 +333,7 @@ mod tests {
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
@@ -315,4 +341,30 @@ mod tests {
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_params_default_to_no_hover_probing() {
|
||||
let params: ObserveParams =
|
||||
serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap();
|
||||
assert!(!params.probe_hover);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_result_round_trips_with_hover_probe_report() {
|
||||
let r = ObserveResult {
|
||||
text: "@vom 1\n".into(),
|
||||
ref_count: 0,
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: Some(HoverProbeReport {
|
||||
performed: true,
|
||||
revealed_content: true,
|
||||
}),
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,142 +5,99 @@ description: Browser automation through six injected domain tools.
|
||||
|
||||
# browser-skill for DeepSeek Harness
|
||||
|
||||
Drive the user's real Chromium browser through the six structured domain tools provided by this
|
||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
||||
Agent Window. Every call must include the tool's `action` field; other fields depend on that action.
|
||||
Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is
|
||||
isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed.
|
||||
|
||||
## Tool availability
|
||||
Loading this skill reveals six tools for the rest of the conversation:
|
||||
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. All browser work
|
||||
must use the injected tools directly. Do not invoke another process to control the browser. Direct
|
||||
tool calls preserve session ownership, the live observation window, cancellation, attachments, and
|
||||
cleanup.
|
||||
- `browser_session` owns session lifecycle.
|
||||
- `browser_page` handles navigation and lifecycle waits.
|
||||
- `browser_inspect` reads semantic, visual, console, and network state.
|
||||
- `browser_interact` performs normal page interactions.
|
||||
- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing.
|
||||
- `browser_assist` handles human help, window size, and device emulation.
|
||||
|
||||
The six tools are:
|
||||
|
||||
- `browser_session`: start, stop, or list plugin-owned sessions.
|
||||
- `browser_page`: navigate, back, forward, reload, or wait for navigation.
|
||||
- `browser_inspect`: observe, snapshot, html, screenshot, console, or network.
|
||||
- `browser_interact`: click, hover, fill, select, or press.
|
||||
- `browser_tabs`: list, create, select, close, borrow, or return tabs.
|
||||
- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user.
|
||||
Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and
|
||||
parameters; do not guess fields. All browser work must use the injected tools directly so session
|
||||
ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke
|
||||
another process to control the browser.
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every browser task follows this lifecycle:
|
||||
|
||||
1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`.
|
||||
2. Use that session for every later call. Pass `session` explicitly when more than one exists.
|
||||
3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task.
|
||||
Treat cleanup like a finally path unless the user explicitly asks to keep the session open.
|
||||
|
||||
Conceptual sequence:
|
||||
Every task owns a bounded plugin session:
|
||||
|
||||
```text
|
||||
browser_session({ action: "start", url: "https://example.com" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_interact({ action: "click", session: sessionId, target: "@e4" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_session({ action: "start", ... })
|
||||
... use the returned sessionId for browser work ...
|
||||
browser_session({ action: "stop", session: sessionId })
|
||||
```
|
||||
|
||||
The plugin owns only sessions it starts. Never guess or reuse a session id from another program.
|
||||
When `session` is omitted, most actions use the current owned session, but explicit ids are safer.
|
||||
Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another
|
||||
program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep
|
||||
the session open. Stopping also returns borrowed tabs.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Browser work is bounded. Define an observable success condition, take the shortest path, and stop
|
||||
as soon as success is visible. Do not refresh or click after success. If an action fails twice
|
||||
without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh
|
||||
observation and then ask the user.
|
||||
- Derive a concrete success condition from the user's request.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
## Observe and interact
|
||||
## Observe, act, observe
|
||||
|
||||
Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles,
|
||||
states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful.
|
||||
Prefer a fresh ref over a raw CSS selector for interactions.
|
||||
Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states,
|
||||
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||
also become stale after large DOM changes, so observe again before the next interaction.
|
||||
|
||||
Refs invalidate after navigation and may become stale after large DOM changes. Observe again after
|
||||
navigation or a meaningful DOM update before using another ref. If an observation identifies a
|
||||
hover-triggered surface, hover the trigger, observe again, then interact with the revealed control.
|
||||
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
|
||||
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||
Do not click the trigger itself unless the user wants the trigger's action.
|
||||
|
||||
Use `browser_interact` as follows:
|
||||
Escalate reading only as needed:
|
||||
|
||||
- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`.
|
||||
- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`.
|
||||
- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing.
|
||||
- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value
|
||||
attributes, not necessarily the visible labels.
|
||||
- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`.
|
||||
1. `observe` for normal understanding and interaction refs.
|
||||
2. `snapshot` when a stricter static accessibility tree is more useful.
|
||||
3. `html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
4. `screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
## Navigate and wait
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Use `browser_page` with these actions:
|
||||
Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative
|
||||
waits when no navigation is expected. After any page change, discard old refs and observe again.
|
||||
|
||||
- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`.
|
||||
- `back` and `forward` move one history entry and optionally accept lifecycle wait fields.
|
||||
- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary.
|
||||
- `wait` waits for an expected navigation after an interaction that did not already wait.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
Avoid speculative waits when no navigation is expected. After any page change, discard old refs and
|
||||
observe again.
|
||||
Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs.
|
||||
Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never
|
||||
invent a tab id or keep a personal tab borrowed across unrelated work.
|
||||
|
||||
## Reading priority
|
||||
## Ask the human when needed
|
||||
|
||||
Escalate page reading only as needed:
|
||||
Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or
|
||||
another step the user must complete. Give a precise prompt and highlight fresh targets when concrete
|
||||
controls are involved. Use completion criteria only for a clear stable success signal.
|
||||
|
||||
1. `browser_inspect` with `action: "observe"` for normal understanding and refs.
|
||||
2. The same tool with `action: "snapshot"` for a stricter static tree.
|
||||
3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and
|
||||
use only a fresh ref when scoping a subtree.
|
||||
4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref`
|
||||
crops to an element; omit it for the visible tab.
|
||||
Resume only after the user continues or the criteria complete. Treat cancellation as rejection and
|
||||
timeout as a blocker rather than retrying. Observe again after control returns before reasoning about
|
||||
the new state or using refs.
|
||||
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls.
|
||||
The same tool can resize the Agent Window or emulate a device when the task requires visual or
|
||||
responsive testing. Emulation is scoped to one tab.
|
||||
|
||||
## Tabs and the Agent Window boundary
|
||||
## Debug and recover without wandering
|
||||
|
||||
Use `browser_tabs` with these actions:
|
||||
Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics.
|
||||
Continue from returned sequence cursors instead of rereading the same buffer.
|
||||
|
||||
- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing.
|
||||
- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`.
|
||||
- `select` focuses an Agent Window tab and requires a returned `tabId`.
|
||||
- `close` closes an Agent Window tab and requires a returned `tabId`.
|
||||
- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`.
|
||||
- `return` restores a borrowed tab and requires that borrowed `tabId`.
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab: list tabs instead of guessing.
|
||||
- Unknown session: list owned sessions or start one; never try foreign ids.
|
||||
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
|
||||
- Unrecoverable failure: report the blocker and stop the owned session.
|
||||
|
||||
Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback
|
||||
cleanup. Never invent a tab id.
|
||||
|
||||
## Human help and display controls
|
||||
|
||||
Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation,
|
||||
consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or
|
||||
selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text
|
||||
conditions. After the user continues or criteria complete, observe again before reasoning about the
|
||||
new state.
|
||||
|
||||
Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use
|
||||
`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per tab.
|
||||
|
||||
## Read-only diagnostics
|
||||
|
||||
Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and
|
||||
`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and
|
||||
`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries.
|
||||
Console additionally accepts `includeStack`.
|
||||
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tool names or route around that limitation.
|
||||
|
||||
## Error handling and final checklist
|
||||
|
||||
On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On
|
||||
an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide
|
||||
whether one longer purposeful wait is justified, and never immediately repeat a failed history
|
||||
action without first checking page state.
|
||||
|
||||
Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are
|
||||
returned when practical, the owned session is stopped, and no extra page action occurs after
|
||||
success.
|
||||
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tools or route around those limits.
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
RiArrowDownSLine,
|
||||
RiCheckLine,
|
||||
RiCloseCircleLine,
|
||||
RiCloseLine,
|
||||
RiErrorWarningLine,
|
||||
RiPictureInPicture2Line,
|
||||
RiPushpinFill,
|
||||
@@ -28,6 +29,7 @@ const asIcon = (component: unknown): IconComponent => component as IconComponent
|
||||
const IconStop = asIcon(RiStopCircleLine);
|
||||
const IconPip = asIcon(RiPictureInPicture2Line);
|
||||
const IconDown = asIcon(RiArrowDownSLine);
|
||||
const IconClose = asIcon(RiCloseLine);
|
||||
const IconWarn = asIcon(RiErrorWarningLine);
|
||||
const IconPin = asIcon(RiPushpinFill);
|
||||
const IconCloseSession = asIcon(RiCloseCircleLine);
|
||||
@@ -320,6 +322,7 @@ export function OverlayBody(props: {
|
||||
now: number;
|
||||
onPopOut?: (() => void) | undefined;
|
||||
onCollapse?: (() => void) | undefined;
|
||||
onClosePip?: (() => void) | undefined;
|
||||
inPip: boolean;
|
||||
onHeaderPointerDown?: (event: ReactPointerEvent<HTMLDivElement>) => void;
|
||||
}) {
|
||||
@@ -333,6 +336,7 @@ export function OverlayBody(props: {
|
||||
now,
|
||||
onPopOut,
|
||||
onCollapse,
|
||||
onClosePip,
|
||||
inPip,
|
||||
onHeaderPointerDown,
|
||||
} = props;
|
||||
@@ -389,6 +393,16 @@ export function OverlayBody(props: {
|
||||
<IconDown size={14} />
|
||||
</button>
|
||||
) : null}
|
||||
{onClosePip !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css["icon-button"]}
|
||||
aria-label="Close mini window"
|
||||
onClick={onClosePip}
|
||||
>
|
||||
<IconClose size={14} />
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.stage}>
|
||||
{displayUrl !== undefined ? (
|
||||
@@ -572,6 +586,7 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
|
||||
: undefined
|
||||
}
|
||||
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
|
||||
onClosePip={pipWindow !== null ? () => pipWindow.close() : undefined}
|
||||
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -149,6 +149,7 @@ export function ObservationSidebarTab({
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined}
|
||||
onClosePip={pipWindow !== null ? () => pipWindow.close() : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ interface LlmLike {
|
||||
* @param data - first bytes of the capture.
|
||||
* @returns the detected media type, or undefined when unrecognized.
|
||||
*/
|
||||
function sniffImageMediaType(data: Uint8Array): string | undefined {
|
||||
export function sniffImageMediaType(data: Uint8Array): string | undefined {
|
||||
if (data.length >= 4) {
|
||||
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
|
||||
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||
|
||||
@@ -17,6 +17,7 @@ import { readFile, unlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import { sniffImageMediaType } from "./image";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
import { BskError, type BskRunner, parseBskJson, runWithSessionBusyRetry } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
@@ -467,7 +468,7 @@ export class ObservationService {
|
||||
const seq = (this.seqs.get(sessionId) ?? 0) + 1;
|
||||
this.seqs.set(sessionId, seq);
|
||||
const id = `obs-${sessionId}-${seq}`;
|
||||
this.frames.set(id, { data, mediaType: "image/png" });
|
||||
this.frames.set(id, { data, mediaType: sniffImageMediaType(data) ?? "image/png" });
|
||||
this.hashes.set(sessionId, hash);
|
||||
const ring = this.rings.get(sessionId) ?? [];
|
||||
ring.push(id);
|
||||
|
||||
@@ -298,6 +298,33 @@ describe("ObservationOverlay", () => {
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows a close button inside the PiP window that closes it", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const pipDoc = document.implementation.createHTMLDocument("pip");
|
||||
const close = vi.fn();
|
||||
const listeners = new Map<string, (() => void)[]>();
|
||||
const requestWindow = vi.fn(async () => {
|
||||
return {
|
||||
document: pipDoc,
|
||||
addEventListener: (name: string, fn: () => void) => {
|
||||
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
|
||||
},
|
||||
close,
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const popout = await screen.findByRole("button", { name: /Pop out/ });
|
||||
fireEvent.click(popout);
|
||||
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
|
||||
const closeBtn = pipDoc.body.querySelector<HTMLButtonElement>(
|
||||
'button[aria-label="Close mini window"]',
|
||||
);
|
||||
expect(closeBtn).not.toBeNull();
|
||||
closeBtn!.click();
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("renders the strip for two sessions and pins focus on click", async () => {
|
||||
const older: SessionObservation = {
|
||||
sessionId: "s1",
|
||||
|
||||
@@ -335,6 +335,18 @@ describe("thumbnail cadence", () => {
|
||||
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1");
|
||||
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
|
||||
});
|
||||
|
||||
it("detects JPEG thumbnail bytes instead of hardcoding image/png", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({
|
||||
screenshotBytes: () => new Uint8Array([0xff, 0xd8, 0xff, 0xdb, 0x00, 0x01]),
|
||||
});
|
||||
const { service } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1");
|
||||
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/jpeg");
|
||||
});
|
||||
});
|
||||
|
||||
describe("observation traffic isolation", () => {
|
||||
|
||||
@@ -47,7 +47,10 @@ describe("registerBskSkill", () => {
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||
expect(content.length).toBeGreaterThan(6_000);
|
||||
// Keep the lazily injected instructions inside a bounded prompt budget,
|
||||
// while the lower bound catches accidental truncation of the guidance.
|
||||
expect(content.length).toBeGreaterThan(3_000);
|
||||
expect(content.length).toBeLessThan(6_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// Source frontmatter is registration metadata and must not leak into the body.
|
||||
expect(content.startsWith("---")).toBe(false);
|
||||
|
||||
@@ -298,6 +298,34 @@ describe("renderVom single-layer page", () => {
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("renders through deeply nested transparent wrappers without exhausting the call stack", () => {
|
||||
const wrapperCount = 10_000;
|
||||
const nodes = [node({ id: 1, role: "RootWebArea", name: "Doc" })];
|
||||
for (let index = 0; index < wrapperCount; index += 1) {
|
||||
nodes.push(
|
||||
node({
|
||||
id: index + 2,
|
||||
parentId: index + 1,
|
||||
role: "generic",
|
||||
}),
|
||||
);
|
||||
}
|
||||
nodes.push(
|
||||
node({
|
||||
id: wrapperCount + 2,
|
||||
parentId: wrapperCount + 1,
|
||||
role: "button",
|
||||
name: "Deep action",
|
||||
tag: "button",
|
||||
}),
|
||||
);
|
||||
|
||||
const out = renderVom(scene(nodes));
|
||||
|
||||
expect(out.text).toContain('@e1 button "Deep action"');
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: wrapperCount + 2 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
@@ -785,6 +813,55 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("北京市小客车指标调控管理信息系统]");
|
||||
});
|
||||
|
||||
it("derives duplicate-label context from indexed DOM ancestry without semantic proximity", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc", contextScopeId: "root" }),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "View",
|
||||
tag: "button",
|
||||
domParentId: 2,
|
||||
domAncestorIds: [2, 1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "View",
|
||||
tag: "button",
|
||||
domParentId: 4,
|
||||
domAncestorIds: [4, 1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "group",
|
||||
name: "Project Alpha",
|
||||
domParentId: 1,
|
||||
domAncestorIds: [1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 1,
|
||||
role: "group",
|
||||
name: "Project Beta",
|
||||
domParentId: 1,
|
||||
domAncestorIds: [1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]');
|
||||
expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]');
|
||||
});
|
||||
|
||||
it("does not add low-value root context to duplicated brand links", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
@@ -833,6 +910,34 @@ describe("renderVom single-layer page", () => {
|
||||
});
|
||||
|
||||
describe("renderVom double-layer page", () => {
|
||||
it("collects blocking-layer descendants even when children precede parents in source order", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "button",
|
||||
name: "Close",
|
||||
tag: "button",
|
||||
paintOrder: 0,
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "Modal",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Close"');
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("renders a blocking modal first and folds the base page into an occluded summary", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
+134
-66
@@ -421,6 +421,41 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
||||
return children;
|
||||
}
|
||||
|
||||
const CONTEXT_LABEL_ROLES = new Set([
|
||||
"heading",
|
||||
"article",
|
||||
"section",
|
||||
"group",
|
||||
"region",
|
||||
"row",
|
||||
"form",
|
||||
]);
|
||||
|
||||
interface DomContextIndex {
|
||||
nonBoundaryByScopeParent: Map<string | undefined, Map<number, VomNode[]>>;
|
||||
sourceOrderById: Map<number, number>;
|
||||
}
|
||||
|
||||
function buildDomContextIndex(nodes: VomNode[]): DomContextIndex {
|
||||
const nonBoundaryByScopeParent = new Map<string | undefined, Map<number, VomNode[]>>();
|
||||
const sourceOrderById = new Map<number, number>();
|
||||
for (const [sourceOrder, node] of nodes.entries()) {
|
||||
sourceOrderById.set(node.id, sourceOrder);
|
||||
if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node)) || isContextBoundary(node)) continue;
|
||||
const parentId = node.domParentId;
|
||||
if (parentId === undefined || parentId === null) continue;
|
||||
let byParent = nonBoundaryByScopeParent.get(node.contextScopeId);
|
||||
if (!byParent) {
|
||||
byParent = new Map();
|
||||
nonBoundaryByScopeParent.set(node.contextScopeId, byParent);
|
||||
}
|
||||
const siblings = byParent.get(parentId) ?? [];
|
||||
siblings.push(node);
|
||||
byParent.set(parentId, siblings);
|
||||
}
|
||||
return { nonBoundaryByScopeParent, sourceOrderById };
|
||||
}
|
||||
|
||||
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const node of nodes) {
|
||||
@@ -440,13 +475,7 @@ function needsHandleContext(nodeName: string, duplicateRefNames: Set<string>): b
|
||||
}
|
||||
|
||||
function contextLabel(node: VomNode, targetName: string): string | undefined {
|
||||
if (
|
||||
!["heading", "article", "section", "group", "region", "row", "form"].includes(
|
||||
normalizedRole(node),
|
||||
)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node))) return undefined;
|
||||
return weakContextText(
|
||||
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
|
||||
targetName,
|
||||
@@ -607,19 +636,45 @@ function collectDomContext(
|
||||
): ContextCandidate[] {
|
||||
const ancestorIds = node.domAncestorIds ?? [];
|
||||
if (ancestorIds.length === 0) return [];
|
||||
const ancestors = new Set(ancestorIds);
|
||||
const contexts: ContextCandidate[] = [];
|
||||
const ancestorRank = new Map<number, number>();
|
||||
for (const [rank, ancestorId] of ancestorIds.entries()) {
|
||||
if (!ancestorRank.has(ancestorId)) ancestorRank.set(ancestorId, rank);
|
||||
}
|
||||
|
||||
for (const candidate of state.nodesById.values()) {
|
||||
const candidates = new Map<number, VomNode>();
|
||||
const nonBoundaryByParent = state.domContextIndex.nonBoundaryByScopeParent.get(
|
||||
node.contextScopeId,
|
||||
);
|
||||
for (const ancestorId of ancestorRank.keys()) {
|
||||
const boundary = state.nodesById.get(ancestorId);
|
||||
const boundaryParentId = boundary?.domParentId;
|
||||
if (
|
||||
boundary &&
|
||||
sharesContextScope(node, boundary) &&
|
||||
isContextBoundary(boundary) &&
|
||||
boundaryParentId !== undefined &&
|
||||
boundaryParentId !== null &&
|
||||
ancestorRank.has(boundaryParentId)
|
||||
) {
|
||||
candidates.set(boundary.id, boundary);
|
||||
}
|
||||
for (const candidate of nonBoundaryByParent?.get(ancestorId) ?? []) {
|
||||
candidates.set(candidate.id, candidate);
|
||||
}
|
||||
}
|
||||
|
||||
const sourceOrderById = state.domContextIndex.sourceOrderById;
|
||||
const orderedCandidates = [...candidates.values()].sort(
|
||||
(a, b) => (sourceOrderById.get(a.id) ?? 0) - (sourceOrderById.get(b.id) ?? 0),
|
||||
);
|
||||
for (const candidate of orderedCandidates) {
|
||||
if (candidate.id === node.id) continue;
|
||||
if (!sharesContextScope(node, candidate)) continue;
|
||||
if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue;
|
||||
const parentId = candidate.domParentId;
|
||||
if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue;
|
||||
if (parentId === undefined || parentId === null) continue;
|
||||
const label = contextLabel(candidate, nodeName);
|
||||
if (!label) continue;
|
||||
const rank = ancestorIds.indexOf(parentId);
|
||||
contexts.push({ text: label, source: "dom", order: rank < 0 ? contexts.length : rank });
|
||||
contexts.push({ text: label, source: "dom", order: ancestorRank.get(parentId) ?? 0 });
|
||||
}
|
||||
|
||||
return contexts;
|
||||
@@ -734,6 +789,7 @@ interface RenderState {
|
||||
children: Map<number | null, VomNode[]>;
|
||||
parentMap: Map<number, number | null>;
|
||||
nodesById: Map<number, VomNode>;
|
||||
domContextIndex: DomContextIndex;
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
@@ -768,20 +824,25 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
|
||||
}
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
parentId: number | null,
|
||||
function pushRenderChildren(
|
||||
stack: Array<{ node: VomNode; depth: number }>,
|
||||
children: readonly VomNode[],
|
||||
depth: number,
|
||||
state: RenderState,
|
||||
): void {
|
||||
if (state.stopped) return;
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], depth });
|
||||
}
|
||||
}
|
||||
|
||||
const nodes = children.get(parentId) ?? [];
|
||||
for (const node of nodes) {
|
||||
if (state.stopped) return;
|
||||
function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
|
||||
const stack: Array<{ node: VomNode; depth: number }> = [];
|
||||
pushRenderChildren(stack, children.get(null) ?? [], 1);
|
||||
|
||||
while (stack.length > 0 && !state.stopped) {
|
||||
const { node, depth } = stack.pop() as { node: VomNode; depth: number };
|
||||
|
||||
if (!shouldRender(node)) {
|
||||
renderTree(children, node.id, depth, state);
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -798,7 +859,7 @@ function renderTree(
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
break;
|
||||
}
|
||||
|
||||
state.lines.push(line);
|
||||
@@ -823,7 +884,7 @@ function renderTree(
|
||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||
continue;
|
||||
}
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -848,12 +909,13 @@ function renderNodes(
|
||||
children,
|
||||
parentMap: buildParentMap(nodes),
|
||||
nodesById: new Map(nodes.map((node) => [node.id, node])),
|
||||
domContextIndex: buildDomContextIndex(nodes),
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
};
|
||||
|
||||
renderTree(children, null, 1, state);
|
||||
renderTree(children, state);
|
||||
|
||||
return state;
|
||||
}
|
||||
@@ -922,23 +984,33 @@ function paintLineageByFrame(
|
||||
return lineage;
|
||||
}
|
||||
|
||||
function cachedPaintLineageByFrame(
|
||||
node: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
cache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): Map<string | undefined, VomNode> {
|
||||
const cached = cache.get(node.id);
|
||||
if (cached) return cached;
|
||||
const lineage = paintLineageByFrame(node, parentMap, nodesById);
|
||||
cache.set(node.id, lineage);
|
||||
return lineage;
|
||||
}
|
||||
|
||||
/** Resolve both nodes to the nearest document where their paint order is comparable. */
|
||||
function comparablePaintNodes(
|
||||
target: VomNode,
|
||||
blocker: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
lineageCache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): { target: VomNode; blocker: VomNode } | null {
|
||||
if (target.frameId === blocker.frameId) return { target, blocker };
|
||||
const targetLineage = paintLineageByFrame(target, parentMap, nodesById);
|
||||
let blockerInFrame: VomNode | undefined = blocker;
|
||||
let guard = 0;
|
||||
while (blockerInFrame && guard <= parentMap.size) {
|
||||
const targetInFrame = targetLineage.get(blockerInFrame.frameId);
|
||||
const targetLineage = cachedPaintLineageByFrame(target, parentMap, nodesById, lineageCache);
|
||||
const blockerLineage = cachedPaintLineageByFrame(blocker, parentMap, nodesById, lineageCache);
|
||||
for (const [frameId, blockerInFrame] of blockerLineage) {
|
||||
const targetInFrame = targetLineage.get(frameId);
|
||||
if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame };
|
||||
const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null;
|
||||
blockerInFrame = parentId === null ? undefined : nodesById.get(parentId);
|
||||
guard += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -949,12 +1021,13 @@ function isBlockedByRegion(
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
viewportCoverage: number,
|
||||
lineageCache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): boolean {
|
||||
if (target.id === blocker.id) return false;
|
||||
if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
|
||||
if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
|
||||
if (!target.rect || !blocker.rect) return false;
|
||||
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById);
|
||||
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById, lineageCache);
|
||||
if (!paintNodes) return false;
|
||||
|
||||
const points = interactionPoints(target.rect);
|
||||
@@ -979,17 +1052,22 @@ function isBlockedByRegion(
|
||||
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
|
||||
}
|
||||
|
||||
function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<number> {
|
||||
function collectDescendants(nodes: VomNode[], roots: Set<number>): Set<number> {
|
||||
const children = new Map<number, number[]>();
|
||||
for (const node of nodes) {
|
||||
if (node.parentId === null) continue;
|
||||
const siblings = children.get(node.parentId) ?? [];
|
||||
siblings.push(node.id);
|
||||
children.set(node.parentId, siblings);
|
||||
}
|
||||
|
||||
const included = new Set(roots);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
const queue = [...roots];
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
for (const childId of children.get(queue[index]) ?? []) {
|
||||
if (included.has(childId)) continue;
|
||||
included.add(childId);
|
||||
queue.push(childId);
|
||||
}
|
||||
}
|
||||
return included;
|
||||
@@ -998,6 +1076,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<numb
|
||||
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
const parentMap = buildParentMap(nodes);
|
||||
const nodesById = new Map(nodes.map((node) => [node.id, node]));
|
||||
const lineageCache = new Map<number, Map<string | undefined, VomNode>>();
|
||||
const candidates = nodes
|
||||
.filter(isPositionedRegionCandidate)
|
||||
.map((node) => {
|
||||
@@ -1014,40 +1093,29 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
for (const target of nodes) {
|
||||
if (!isVomReferenceNode(target)) continue;
|
||||
const blocked = candidates.some((candidate) =>
|
||||
isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage),
|
||||
isBlockedByRegion(
|
||||
target,
|
||||
candidate.node,
|
||||
parentMap,
|
||||
nodesById,
|
||||
candidate.viewportCoverage,
|
||||
lineageCache,
|
||||
),
|
||||
);
|
||||
if (blocked) blockedRoots.add(target.id);
|
||||
}
|
||||
|
||||
if (blockedRoots.size === 0) return nodes;
|
||||
const blocked = collectDescendantsOfIds(nodes, blockedRoots);
|
||||
const blocked = collectDescendants(nodes, blockedRoots);
|
||||
return nodes.filter((node) => !blocked.has(node.id));
|
||||
}
|
||||
|
||||
function collectDescendantsOfMembers(nodes: VomNode[], members: Set<number>): Set<number> {
|
||||
const included = new Set(members);
|
||||
let changed = true;
|
||||
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return included;
|
||||
}
|
||||
|
||||
function countRenderable(nodes: VomNode[]): number {
|
||||
return nodes.filter(shouldRender).length;
|
||||
}
|
||||
|
||||
function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult {
|
||||
const included = collectDescendantsOfMembers(scene.nodes, layer.members);
|
||||
const included = collectDescendants(scene.nodes, layer.members);
|
||||
const visibleNodes = scene.nodes.filter((node) => included.has(node.id));
|
||||
const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id)));
|
||||
const header = [
|
||||
|
||||
+103
-160
@@ -1,160 +1,125 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
@@ -303,48 +268,26 @@ bsk record start --browser <instance-id-or-label> \
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## Recover without wandering
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
Reference in New Issue
Block a user