mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(recorder): add iframe support
This commit is contained in:
1 parent
c08fed5de3
commit
615e866608
22 files changed
+1267
-309
No files matched your search
@@ -0,0 +1,121 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_FRAME_PORT } from "../recording/frame-bridge";
|
||||
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
|
||||
|
||||
class ListenerSet<T extends (...args: never[]) => unknown> {
|
||||
readonly listeners = new Set<T>();
|
||||
addListener = (listener: T) => this.listeners.add(listener);
|
||||
removeListener = (listener: T) => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
function fakePort(sender: chrome.runtime.MessageSender) {
|
||||
const messages: unknown[] = [];
|
||||
const onMessage = new ListenerSet<(message: unknown) => void>();
|
||||
const onDisconnect = new ListenerSet<() => void>();
|
||||
return {
|
||||
port: {
|
||||
name: RECORD_FRAME_PORT,
|
||||
sender,
|
||||
onMessage,
|
||||
onDisconnect,
|
||||
postMessage: vi.fn((message: unknown) => messages.push(message)),
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as chrome.runtime.Port,
|
||||
messages,
|
||||
receive(message: unknown) {
|
||||
for (const listener of onMessage.listeners) listener(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("RecordFrameCoordinator", () => {
|
||||
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
|
||||
const onMessage = new ListenerSet<
|
||||
(
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => boolean
|
||||
>();
|
||||
|
||||
beforeEach(() => {
|
||||
onConnect.listeners.clear();
|
||||
onMessage.listeners.clear();
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { onConnect, onMessage },
|
||||
});
|
||||
});
|
||||
|
||||
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
|
||||
const sendToDocument = vi.fn(async () => ({ ok: true }));
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [
|
||||
{ frameId: 0, documentId: "top-document" },
|
||||
{ frameId: 7, documentId: "child-document" },
|
||||
],
|
||||
sendToDocument,
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10);
|
||||
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
|
||||
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 7,
|
||||
documentId: "child-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
|
||||
tabId: 3,
|
||||
documentId: "child-document",
|
||||
browserFrameId: 7,
|
||||
producerId: "producer-1",
|
||||
});
|
||||
|
||||
const stopping = coordinator.stop("rec-1");
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
|
||||
const stop = frame.messages.find(
|
||||
(message): message is { type: "stop"; commandId: string } =>
|
||||
typeof message === "object" &&
|
||||
message !== null &&
|
||||
(message as { type?: string }).type === "stop",
|
||||
);
|
||||
expect(stop).toBeDefined();
|
||||
frame.receive({
|
||||
type: "stopped",
|
||||
requestId: "rec-1",
|
||||
commandId: stop!.commandId,
|
||||
ok: true,
|
||||
});
|
||||
await expect(stopping).resolves.toBe(true);
|
||||
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a producer used from another Document", async () => {
|
||||
const coordinator = new RecordFrameCoordinator({
|
||||
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
|
||||
sendToDocument: async () => ({ ok: true }),
|
||||
});
|
||||
coordinator.attach();
|
||||
coordinator.begin("rec-1", 10);
|
||||
await coordinator.armTab("rec-1", 3);
|
||||
const sender = {
|
||||
tab: { id: 3 },
|
||||
frameId: 0,
|
||||
documentId: "top-document",
|
||||
} as chrome.runtime.MessageSender;
|
||||
const frame = fakePort(sender);
|
||||
for (const listener of onConnect.listeners) listener(frame.port);
|
||||
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
|
||||
|
||||
expect(
|
||||
coordinator.sourceFor("rec-1", "producer-1", {
|
||||
...sender,
|
||||
documentId: "other-document",
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
|
||||
import { markRecordingDocument } from "../recording/document-marker";
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
|
||||
describe("recording document marker", () => {
|
||||
it("restores the Document attribute exactly", () => {
|
||||
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
|
||||
const marker = markRecordingDocument("producer-1");
|
||||
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("producer-1");
|
||||
marker.restore();
|
||||
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
|
||||
});
|
||||
|
||||
it("indexes the CDP scope from the safe frame identity", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("producer-1")).toEqual({
|
||||
frameId: "child",
|
||||
target: { tabId: 3, sessionId: "oopif-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a recording identity maps to multiple Documents", () => {
|
||||
const index = new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "left",
|
||||
target: { tabId: 3, sessionId: "left-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
target: { tabId: 3, sessionId: "right-session" },
|
||||
recordingDocumentId: "duplicate",
|
||||
},
|
||||
],
|
||||
matchNodes: [],
|
||||
refs: [],
|
||||
});
|
||||
|
||||
expect(index.documentScope("duplicate")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,7 @@ vi.mock("../recording/observation-capture", async (importOriginal) => {
|
||||
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
import { RecordingObservationRuntime } from "../recording/recording-runtime";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
function observation(url = "https://example.com/") {
|
||||
return {
|
||||
@@ -149,4 +150,53 @@ describe("RecordingObservationRuntime", () => {
|
||||
});
|
||||
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
|
||||
});
|
||||
|
||||
it("binds an iframe draft to its marked CDP Document scope", async () => {
|
||||
captureRecordingObservation.mockResolvedValueOnce({
|
||||
...observation(),
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
frames: [
|
||||
{
|
||||
frameId: "child",
|
||||
target: { tabId: 7, sessionId: "oopif-session" },
|
||||
recordingDocumentId: "producer-1",
|
||||
},
|
||||
],
|
||||
matchNodes: [
|
||||
{
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
rect: { x: 410, y: 20, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
}),
|
||||
});
|
||||
const recording = runtime();
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "click" as const,
|
||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
await recording.captureInitial(7);
|
||||
await recording.processDraft(7, drafts, 0, "producer-1");
|
||||
recording.cancel();
|
||||
|
||||
const draft = drafts[0];
|
||||
expect(draft?.op).toBe("click");
|
||||
if (!draft || draft.op !== "click") throw new Error("expected click draft");
|
||||
expect(draft.targetHint).toMatchObject({
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
});
|
||||
expect(draft.matchedTarget?.ref).toBe("e1");
|
||||
});
|
||||
});
|
||||
@@ -69,6 +69,23 @@ describe("matchObservationTarget", () => {
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("matches iframe capture geometry in the child viewport coordinate space", () => {
|
||||
const child = node(42, "child", 410);
|
||||
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[child],
|
||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: "child",
|
||||
geometrySpace: "local",
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("restricts a missing frame hint to the root frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
@@ -81,6 +98,21 @@ describe("matchObservationTarget", () => {
|
||||
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("fails closed when the source Document has no VOM frame mapping", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root")],
|
||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||
),
|
||||
hint: {
|
||||
frameId: null,
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
fallback: { tag: "button", role: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("returns unmatched for ambiguous geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
|
||||
Reference in new issue
Block a user