mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(recorder): bug fix
This commit is contained in:
1 parent
edb686d0a2
commit
39701e0ca5
4 files changed
+129
-61
No files matched your search
@@ -39,6 +39,7 @@ import {
|
|||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
attachRecordQueryListener,
|
attachRecordQueryListener,
|
||||||
attachRecordStepListener,
|
attachRecordStepListener,
|
||||||
|
type RecordRuntimeDeps,
|
||||||
} from "@/tools/record";
|
} from "@/tools/record";
|
||||||
import { detectBrowserMeta } from "@/transport/handshake";
|
import { detectBrowserMeta } from "@/transport/handshake";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
@@ -146,10 +147,33 @@ export default defineBackground(() => {
|
|||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const recordDeps = {
|
||||||
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
|
frameCoordinator: recordFrameCoordinator,
|
||||||
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
|
try {
|
||||||
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
type: OVERLAY_AUTOMATION_BYPASS,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Content script may be unavailable on restricted pages.
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} satisfies RecordRuntimeDeps;
|
||||||
|
recordFrameCoordinator.attach();
|
||||||
|
attachRecordStepListener(recordDeps);
|
||||||
|
attachRecordFinishListener(recordDeps);
|
||||||
|
attachRecordQueryListener(recordDeps);
|
||||||
|
|
||||||
const dispatcher = new ToolDispatcher({
|
const dispatcher = new ToolDispatcher({
|
||||||
transport,
|
transport,
|
||||||
sessions,
|
sessions,
|
||||||
cdp,
|
cdp,
|
||||||
|
recording: recordDeps,
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
onBrowserControlResumed,
|
onBrowserControlResumed,
|
||||||
approveBorrow: (ctx) =>
|
approveBorrow: (ctx) =>
|
||||||
@@ -171,30 +195,6 @@ export default defineBackground(() => {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
const recordDeps = {
|
|
||||||
tabsApi: chrome.tabs,
|
|
||||||
cdp,
|
|
||||||
frameCoordinator: recordFrameCoordinator,
|
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
|
||||||
try {
|
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
// Message listeners stay up (cheap; fire only for record message types).
|
|
||||||
// Tab / webNavigation observation attaches lazily while a recording is
|
|
||||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
|
||||||
attachRecordStepListener(recordDeps);
|
|
||||||
attachRecordFinishListener(recordDeps);
|
|
||||||
attachRecordQueryListener(recordDeps);
|
|
||||||
recordFrameCoordinator.attach();
|
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
attachBorrowNotificationClickHandler({
|
attachBorrowNotificationClickHandler({
|
||||||
onClicked: chrome.notifications.onClicked,
|
onClicked: chrome.notifications.onClicked,
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ import type {
|
|||||||
RequestFrame,
|
RequestFrame,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { ToolDispatcher } from "../dispatcher";
|
import { ToolDispatcher } from "../dispatcher";
|
||||||
|
import {
|
||||||
|
resetBrowserObservationForTests,
|
||||||
|
setBrowserObservationAttachForTests,
|
||||||
|
} from "../record";
|
||||||
|
|
||||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||||
|
|
||||||
@@ -44,9 +48,72 @@ function makeRequest(method: string, params: unknown): RequestFrame {
|
|||||||
|
|
||||||
describe("ToolDispatcher", () => {
|
describe("ToolDispatcher", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
resetBrowserObservationForTests();
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses the configured recording runtime for start and stop", async () => {
|
||||||
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const tab = {
|
||||||
|
id: 7,
|
||||||
|
windowId: 4242,
|
||||||
|
active: true,
|
||||||
|
status: "complete",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
} as chrome.tabs.Tab;
|
||||||
|
const frameCoordinator = {
|
||||||
|
begin: vi.fn(),
|
||||||
|
armTab: vi.fn(async () => true),
|
||||||
|
sourceFor: vi.fn(() => null),
|
||||||
|
stop: vi.fn(async () => true),
|
||||||
|
cancel: vi.fn(),
|
||||||
|
};
|
||||||
|
const sendToTab = vi.fn(async () => ({ ok: true }));
|
||||||
|
setBrowserObservationAttachForTests(
|
||||||
|
() => () => {},
|
||||||
|
() => () => {},
|
||||||
|
);
|
||||||
|
const dispatcher = new ToolDispatcher({
|
||||||
|
transport,
|
||||||
|
sessions,
|
||||||
|
recording: {
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(async () => tab),
|
||||||
|
query: vi.fn(async () => [tab]),
|
||||||
|
},
|
||||||
|
frameCoordinator,
|
||||||
|
sendToTab,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
dispatcher.start();
|
||||||
|
|
||||||
|
deliver(
|
||||||
|
makeRequest("tool.record_start", {
|
||||||
|
session_id: "aa11",
|
||||||
|
url: "https://example.com/start",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||||
|
|
||||||
|
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
|
||||||
|
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
|
||||||
|
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
|
||||||
|
|
||||||
|
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
|
||||||
|
await vi.waitFor(() => expect(sent).toHaveLength(2));
|
||||||
|
|
||||||
|
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
|
||||||
|
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
|
||||||
|
});
|
||||||
|
|
||||||
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
|
||||||
const { transport, sent, deliver } = fakeTransport();
|
const { transport, sent, deliver } = fakeTransport();
|
||||||
const sessions = new SessionManager({
|
const sessions = new SessionManager({
|
||||||
|
|||||||
@@ -51,7 +51,12 @@ import {
|
|||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
handleSnapshot,
|
handleSnapshot,
|
||||||
} from "./observation";
|
} from "./observation";
|
||||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
import {
|
||||||
|
handleRecordAwait,
|
||||||
|
handleRecordStart,
|
||||||
|
handleRecordStop,
|
||||||
|
type RecordRuntimeDeps,
|
||||||
|
} from "./record";
|
||||||
import {
|
import {
|
||||||
handleSessionStart,
|
handleSessionStart,
|
||||||
handleSessionStop,
|
handleSessionStop,
|
||||||
@@ -99,6 +104,7 @@ export interface DispatcherDeps {
|
|||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
cdp?: DispatcherCdpRunner;
|
cdp?: DispatcherCdpRunner;
|
||||||
|
recording?: RecordRuntimeDeps;
|
||||||
/**
|
/**
|
||||||
* Invoked whenever a dispatched RPC may have changed the live
|
* Invoked whenever a dispatched RPC may have changed the live
|
||||||
* session set (currently `tool.session_start` and
|
* session set (currently `tool.session_start` and
|
||||||
@@ -134,6 +140,7 @@ export class ToolDispatcher {
|
|||||||
private readonly transport: Transport;
|
private readonly transport: Transport;
|
||||||
private readonly sessions: SessionManager;
|
private readonly sessions: SessionManager;
|
||||||
private readonly cdp?: DispatcherCdpRunner;
|
private readonly cdp?: DispatcherCdpRunner;
|
||||||
|
private readonly recording?: RecordRuntimeDeps;
|
||||||
private readonly onSessionsChanged?: () => void;
|
private readonly onSessionsChanged?: () => void;
|
||||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
@@ -153,6 +160,7 @@ export class ToolDispatcher {
|
|||||||
this.transport = deps.transport;
|
this.transport = deps.transport;
|
||||||
this.sessions = deps.sessions;
|
this.sessions = deps.sessions;
|
||||||
this.cdp = deps.cdp;
|
this.cdp = deps.cdp;
|
||||||
|
this.recording = deps.recording;
|
||||||
this.onSessionsChanged = deps.onSessionsChanged;
|
this.onSessionsChanged = deps.onSessionsChanged;
|
||||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||||
this.approveBorrow = deps.approveBorrow;
|
this.approveBorrow = deps.approveBorrow;
|
||||||
@@ -531,44 +539,26 @@ export class ToolDispatcher {
|
|||||||
signal,
|
signal,
|
||||||
});
|
});
|
||||||
case "tool.record_start":
|
case "tool.record_start":
|
||||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
signal,
|
||||||
try {
|
})
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
: recordingRuntimeUnavailable();
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
case "tool.record_stop":
|
case "tool.record_stop":
|
||||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
bypassOverlay: async (tabId, enabled) => {
|
signal,
|
||||||
try {
|
})
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
: recordingRuntimeUnavailable();
|
||||||
type: OVERLAY_AUTOMATION_BYPASS,
|
|
||||||
enabled,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content script may be unavailable on restricted pages.
|
|
||||||
}
|
|
||||||
},
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
case "tool.record_await":
|
case "tool.record_await":
|
||||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
return this.recording
|
||||||
tabsApi: chromeTabsApi,
|
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
...this.recording,
|
||||||
signal,
|
signal,
|
||||||
});
|
})
|
||||||
|
: recordingRuntimeUnavailable();
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
code: "unknown_method",
|
code: "unknown_method",
|
||||||
@@ -704,6 +694,13 @@ function isRpcError(v: unknown): v is RpcError {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordingRuntimeUnavailable(): RpcError {
|
||||||
|
return {
|
||||||
|
code: "protocol_error",
|
||||||
|
message: "recording runtime is unavailable",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||||
switch (req.method) {
|
switch (req.method) {
|
||||||
case "tool.tab_create":
|
case "tool.tab_create":
|
||||||
|
|||||||
@@ -292,6 +292,10 @@ export interface RecordDeps {
|
|||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
|
||||||
|
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
|
||||||
|
};
|
||||||
|
|
||||||
let defaultDeps: RecordDeps | null = null;
|
let defaultDeps: RecordDeps | null = null;
|
||||||
function getDefaultDeps(): RecordDeps {
|
function getDefaultDeps(): RecordDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
|
|||||||
Reference in new issue
Block a user