figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extension): support OOPIF refs with unified frame geometry

Browse Source
This commit is contained in:
Ljy-0827
2026-08-18 19:28:16 +08:00
committed by Zhang GH
parent 22019cdf07
commit 42403e55f2
32 changed files with 2418 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/element-geometry.test.ts
+19
View File
@@ -19,6 +19,25 @@ describe("quadBoundingRect", () => {
});
describe("nodeBoundingRect", () => {
it("unions every visible content fragment", async () => {
const send = async () => ({
quads: [
[10, 20, 60, 20, 60, 40, 10, 40],
[15, 50, 90, 50, 90, 80, 15, 80],
],
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
};
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
x: 10,
y: 20,
width: 80,
height: 60,
});
});
it("falls back to visible descendant bounds for zero-size containers", async () => {
const calls: Array<{ method: string; params?: object }> = [];
const send = async (_tabId: number, method: string, params?: object) => {
apps/extension/src/tools/__tests__/frame-geometry.test.ts
+222
View File
@@ -0,0 +1,222 @@
import { describe, expect, it, vi } from "vitest";
import { type Quad, resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
import {
clipPolygon,
polygonBounds,
projectAndClipRegion,
projectUnitPoint,
rectPolygon,
} from "../geometry";
import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => {
it("maps through the iframe content box instead of its border box", () => {
const contentQuad: Quad = [
{ x: 204, y: 306 },
{ x: 604, y: 306 },
{ x: 604, y: 506 },
{ x: 204, y: 506 },
];
const projected = projectAndClipRegion(
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
{ width: 1000, height: 800 },
);
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
});
it("uses a projective mapping for perspective-transformed iframe quads", () => {
const quad: Quad = [
{ x: 0, y: 0 },
{ x: 200, y: 20 },
{ x: 180, y: 140 },
{ x: 20, y: 100 },
];
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
});
it("clips against the iframe polygon rather than its bounding box", () => {
const frameQuad: Quad = [
{ x: 50, y: 0 },
{ x: 100, y: 50 },
{ x: 50, y: 100 },
{ x: 0, y: 50 },
];
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
});
it("does not reapply same-target frame transforms around an OOPIF", async () => {
const send = vi.fn(async (_tabId, method, params) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
return backendNodeId === 20
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
}
throw new Error(`unexpected root command ${method}`);
});
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
};
const projection = await resolveFrameProjection(
cdp,
{
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same-process-parent",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
{
frameId: "oopif",
parentFrameId: "same-process-parent",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "oopif-session" },
},
],
},
"oopif",
);
expect(projection?.edges).toEqual([
{
sourceViewport: { width: 200, height: 100 },
destinationQuad: [
{ x: 110, y: 70 },
{ x: 310, y: 70 },
{ x: 310, y: 170 },
{ x: 110, y: 170 },
],
destinationClips: [
[
{ x: 100, y: 50 },
{ x: 400, y: 50 },
{ x: 400, y: 250 },
{ x: 100, y: 250 },
],
],
},
]);
});
it("clips live OOPIF geometry at every frame boundary", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("clips live geometry to same-target iframe ancestors", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["send"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4 },
},
],
})),
};
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4 },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
});
});
it("fails closed when an OOPIF frame graph is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(),
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
}
throw new Error(`unexpected ${method}`);
}) as CdpRunner["sendToTarget"],
};
await expect(
resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child-session" },
frameId: "child",
backendNodeId: 101,
}),
).resolves.toMatchObject({ code: "cdp_failed" });
});
});
apps/extension/src/tools/__tests__/human-loop.test.ts
+110 -21
View File
@@ -1,5 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { RefStore } from "@/session-manager/ref-store";
import type { RequestHelpParams } from "@/transport/types";
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
}
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
const refStore = new RefStore();
const mgr = {
get: (id: string) =>
id === sessionId
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
: null,
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
} as unknown as SessionManager;
return mgr;
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
ac.abort();
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
expect(cdpSend).toHaveBeenCalledWith(
5,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
);
expect(cdpSend).toHaveBeenCalledWith(
6,
"DOM.querySelectorAll",
expect.objectContaining({ selector: "[data-bsk-help]" }),
expect(cdpSend).not.toHaveBeenCalledWith(
expect.anything(),
"DOM.removeAttribute",
expect.anything(),
);
});
it("tags ref targets via CDP and reports them matched", async () => {
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 5 });
const mgr = {
get: (id: string) =>
id === "abcd"
? {
sessionId: "abcd",
agentWindowId: 99,
refStore: { resolve: () => 42 },
refStore,
borrowedTabs: new Map(),
}
: null,
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
} as unknown as SessionManager;
const deps = baseDeps({
cdp: {
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
send: vi.fn(async (_tabId, method) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected ${method}`);
}),
} as unknown as RequestHelpDeps["cdp"],
});
const res = await handleRequestHelp(
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
deps,
);
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
expect(sentMsg.selectors).toEqual([]);
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
expect(res).toMatchObject({
outcome: "continued",
tab_id: 5,
resolved_targets: [{ matched: true, ref: "@e1" }],
resolved_targets: [
{
matched: true,
ref: "@e1",
},
],
});
});
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
const mgr = {
get: (id: string) =>
id === "abcd"
? {
sessionId: "abcd",
agentWindowId: 99,
refStore: {
resolveEntry: () => ({
backendNodeId: 42,
tabId: 5,
frameId: "child",
cdpSessionId: "child-session",
generation: 1,
}),
},
borrowedTabs: new Map(),
}
: null,
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
} as unknown as SessionManager;
const send = vi.fn(async (_tabId, method) => {
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
throw new Error(`unexpected root ${method}`);
});
const sendToTarget = vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child ${method}`);
});
const deps = baseDeps({
cdp: {
send,
sendToTarget,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 5 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 5, sessionId: "child-session" },
},
],
})),
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
});
const res = await handleRequestHelp(
mgr,
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
deps,
);
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 5, sessionId: "child-session" },
"DOM.getContentQuads",
{ backendNodeId: 42 },
);
});
it("reports ref target unmatched when ref does not resolve", async () => {
const deps = baseDeps();
const res = await handleRequestHelp(
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
});
it("reports ref target unmatched when ref is for another tab", async () => {
const refStore = new RefStore();
refStore.set("e1", 42, { tabId: 4 });
const mgr = {
get: (id: string) =>
id === "abcd"
? {
sessionId: "abcd",
agentWindowId: 99,
refStore: {
resolve: (ref: string, opts: { tabId?: number }) =>
ref === "e1" && opts.tabId === 4 ? 42 : null,
},
refStore,
borrowedTabs: new Map(),
}
: null,
apps/extension/src/tools/__tests__/interaction.test.ts
+60
View File
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
const sendImpl = async (tabId: number, method: string, params?: object) => {
sent.push({ tabId, method, params });
const h = handlers[method];
if (!h && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!h) throw new Error(`unexpected CDP call ${method}`);
return h(params);
};
@@ -164,6 +167,63 @@ describe("handleClick", () => {
});
});
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"Input.dispatchMouseEvent": () => ({}),
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getBoxModel": () => ({
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
expect(res).toMatchObject({ x: 324, y: 386 });
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
const order: string[] = [];
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
apps/extension/src/tools/__tests__/observation.test.ts
+37
View File
@@ -63,6 +63,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
sent.push({ method, params });
const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
});
@@ -586,6 +589,7 @@ describe("buildVomScene", () => {
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: mainNodes,
axNodes: [
axNode("root", 1, "RootWebArea"),
@@ -598,6 +602,7 @@ describe("buildVomScene", () => {
contextScopeId: "first",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 7, sessionId: "session-first" },
domNodes: firstNodes,
axNodes: [
axNode("root", 100, "RootWebArea"),
@@ -609,6 +614,7 @@ describe("buildVomScene", () => {
contextScopeId: "second",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 7, sessionId: "session-second" },
domNodes: secondNodes,
axNodes: [
axNode("root", 200, "RootWebArea"),
@@ -621,6 +627,7 @@ describe("buildVomScene", () => {
contextScopeId: "nested",
parentFrameId: "second",
ownerBackendNodeId: 210,
target: { tabId: 7, sessionId: "session-nested" },
domNodes: nestedNodes,
axNodes: [
axNode("root", 300, "RootWebArea"),
@@ -673,6 +680,7 @@ describe("buildVomScene", () => {
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: [],
axNodes: [],
},
@@ -680,6 +688,7 @@ describe("buildVomScene", () => {
frameId: "child",
contextScopeId: "child",
parentFrameId: "main",
target: { tabId: 7 },
domNodes: [childNode],
axNodes: [
{
@@ -740,6 +749,7 @@ describe("buildVomScene", () => {
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: mainNodes,
axNodes: [
{
@@ -761,6 +771,7 @@ describe("buildVomScene", () => {
contextScopeId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 7, sessionId: "child-session" },
domNodes: [],
axNodes: [
{
@@ -3407,6 +3418,32 @@ describe("handleGetHtml", () => {
expect(deps.send).toHaveBeenCalledTimes(1);
});
it("reads a frame ref through its child CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e7", 4242, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const deps = makeDeps({});
const sendToTarget = vi.fn(async (target, method, params) => {
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
expect(method).toBe("DOM.getOuterHTML");
expect(params).toEqual({ backendNodeId: 4242 });
return { outerHTML: "<button>frame</button>" };
});
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
CdpRunner["sendToTarget"]
>;
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.html).toBe("<button>frame</button>");
expect(deps.send).not.toHaveBeenCalled();
});
it("returns not_found when a ref belongs to another tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
apps/extension/src/tools/__tests__/snapshot-ref.test.ts
+19
View File
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
});
describe("resolveSnapshotRef", () => {
it("preserves the frame and child session needed to route iframe refs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const expected = {
backendNodeId: 1234,
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
});
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");