mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(screenshot): hide overlays during capture so shots only contain page content (#64)
This commit is contained in:
@@ -0,0 +1,125 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { CAPTURE_SUPPRESS, type CaptureSuppressAck } from "@/lib/capture-suppress-bridge";
|
||||||
|
import { CAPTURE_HIDDEN_ATTR, createCaptureSuppressController } from "../capture-suppress";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* rAF is stubbed with a manual queue so tests can drive the compositor
|
||||||
|
* frame-by-frame and assert the ack only lands after the second frame.
|
||||||
|
*/
|
||||||
|
describe("capture-suppress controller", () => {
|
||||||
|
let rafQueue: FrameRequestCallback[];
|
||||||
|
let host: HTMLElement;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
rafQueue = [];
|
||||||
|
vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => {
|
||||||
|
rafQueue.push(cb);
|
||||||
|
return rafQueue.length;
|
||||||
|
});
|
||||||
|
vi.stubGlobal("cancelAnimationFrame", vi.fn());
|
||||||
|
document.body.innerHTML = "";
|
||||||
|
host = document.createElement("div");
|
||||||
|
document.body.append(host);
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Run every queued rAF callback, draining microtasks between rounds. */
|
||||||
|
async function flushFrames(rounds = 4): Promise<void> {
|
||||||
|
for (let round = 0; round < rounds && rafQueue.length > 0; round += 1) {
|
||||||
|
const callbacks = rafQueue.splice(0);
|
||||||
|
for (const cb of callbacks) cb(0);
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("begin hides the host immediately but only acks after two frames", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
|
||||||
|
const needsAsync = controller.handleMessage(
|
||||||
|
{ type: CAPTURE_SUPPRESS, phase: "begin" },
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(true);
|
||||||
|
expect(controller.pendingCount).toBe(1);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
expect(sendResponse).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// First frame commits the style change; the ack must still be pending.
|
||||||
|
await flushFrames(1);
|
||||||
|
expect(sendResponse).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// Second frame: the compositor has produced an overlay-free frame.
|
||||||
|
await flushFrames();
|
||||||
|
const expected: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith(expected);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("end removes the attribute once the count returns to zero", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
await flushFrames();
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
const needsAsync = controller.handleMessage(
|
||||||
|
{ type: CAPTURE_SUPPRESS, phase: "end" },
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(needsAsync).toBe(false);
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reference-counts concurrent begins so one end does not unhide early", async () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
await flushFrames();
|
||||||
|
expect(controller.pendingCount).toBe(2);
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||||
|
expect(controller.pendingCount).toBe(1);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a stray end as a no-op", () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const sendResponse = vi.fn();
|
||||||
|
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, sendResponse);
|
||||||
|
|
||||||
|
expect(controller.pendingCount).toBe(0);
|
||||||
|
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-applies the attribute when the host is remounted mid-capture", () => {
|
||||||
|
let currentHost: HTMLElement | null = host;
|
||||||
|
const controller = createCaptureSuppressController(() => currentHost);
|
||||||
|
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||||
|
|
||||||
|
// Host lost and rebuilt before `end` arrives.
|
||||||
|
currentHost = null;
|
||||||
|
const rebuilt = document.createElement("div");
|
||||||
|
controller.onHostMounted(rebuilt);
|
||||||
|
|
||||||
|
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves a remounted host visible once suppression has ended", () => {
|
||||||
|
const controller = createCaptureSuppressController(() => host);
|
||||||
|
const rebuilt = document.createElement("div");
|
||||||
|
|
||||||
|
controller.onHostMounted(rebuilt);
|
||||||
|
|
||||||
|
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
/**
|
||||||
|
* Content-script side of the capture-suppress bridge: hides the overlay
|
||||||
|
* shadow host while the background captures a screenshot.
|
||||||
|
*
|
||||||
|
* `begin` sets `data-bsk-capture-hidden` on the host (overlay.css maps it
|
||||||
|
* to `display: none !important`, which removes the whole shadow subtree
|
||||||
|
* from compositing) and only acks after two animation frames — the first
|
||||||
|
* frame commits the style change, the second guarantees the compositor
|
||||||
|
* has produced a frame without the overlay, so the capture API cannot
|
||||||
|
* pick up a stale frame that still contains it. `end` is
|
||||||
|
* reference-counted so overlapping captures do not unhide the host early.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
CAPTURE_SUPPRESS,
|
||||||
|
type CaptureSuppressAck,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
} from "@/lib/capture-suppress-bridge";
|
||||||
|
|
||||||
|
export const CAPTURE_HIDDEN_ATTR = "data-bsk-capture-hidden";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Safety net for hidden/background tabs where rAF callbacks never fire:
|
||||||
|
* the attribute is already set synchronously, so any frame the compositor
|
||||||
|
* produces from now on excludes the overlay. Waiting longer would just
|
||||||
|
* deadlock the capture.
|
||||||
|
*/
|
||||||
|
const FRAME_FALLBACK_MS = 500;
|
||||||
|
|
||||||
|
function nextFrame(): Promise<void> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const raf = requestAnimationFrame(() => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
cancelAnimationFrame(raf);
|
||||||
|
resolve();
|
||||||
|
}, FRAME_FALLBACK_MS);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureSuppressController {
|
||||||
|
/**
|
||||||
|
* Handle one bridge message. Returns true when the ack is sent
|
||||||
|
* asynchronously and the caller must keep the message channel open.
|
||||||
|
*/
|
||||||
|
handleMessage(
|
||||||
|
message: CaptureSuppressMessage,
|
||||||
|
sendResponse: (ack: CaptureSuppressAck) => void,
|
||||||
|
): boolean;
|
||||||
|
/** Re-apply the hidden attribute when the overlay host is (re)mounted. */
|
||||||
|
onHostMounted(host: HTMLElement): void;
|
||||||
|
/** Number of in-flight `begin` phases (exposed for tests). */
|
||||||
|
readonly pendingCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createCaptureSuppressController(
|
||||||
|
getHost: () => HTMLElement | null,
|
||||||
|
): CaptureSuppressController {
|
||||||
|
let pending = 0;
|
||||||
|
|
||||||
|
const ack: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
|
||||||
|
function setHidden(host: HTMLElement | null, hidden: boolean): void {
|
||||||
|
if (!host) return;
|
||||||
|
if (hidden) host.setAttribute(CAPTURE_HIDDEN_ATTR, "");
|
||||||
|
else host.removeAttribute(CAPTURE_HIDDEN_ATTR);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
get pendingCount() {
|
||||||
|
return pending;
|
||||||
|
},
|
||||||
|
handleMessage(message, sendResponse) {
|
||||||
|
if (message.phase === "begin") {
|
||||||
|
pending += 1;
|
||||||
|
setHidden(getHost(), true);
|
||||||
|
void (async () => {
|
||||||
|
await nextFrame();
|
||||||
|
await nextFrame();
|
||||||
|
sendResponse(ack);
|
||||||
|
})();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// `end` without a matching `begin` (e.g. a retried message) must not
|
||||||
|
// drive the counter negative or clear a concurrent capture's flag.
|
||||||
|
pending = Math.max(0, pending - 1);
|
||||||
|
if (pending === 0) setHidden(getHost(), false);
|
||||||
|
sendResponse(ack);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
onHostMounted(host) {
|
||||||
|
// Host lost + rebuilt mid-capture: the fresh host must stay hidden
|
||||||
|
// until the last `end` arrives.
|
||||||
|
setHidden(host, pending > 0);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -14,3 +14,10 @@
|
|||||||
:host([data-bsk-overlay-surface][data-bsk-overlay-blocking]) {
|
:host([data-bsk-overlay-surface][data-bsk-overlay-blocking]) {
|
||||||
pointer-events: auto !important;
|
pointer-events: auto !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Screenshot suppression: the host lives in the main DOM, so hiding it
|
||||||
|
removes the whole shadow subtree from compositing and captured frames
|
||||||
|
only contain page content. */
|
||||||
|
:host([data-bsk-capture-hidden]) {
|
||||||
|
display: none !important;
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { flushSync } from "react-dom";
|
|||||||
import ReactDOM from "react-dom/client";
|
import ReactDOM from "react-dom/client";
|
||||||
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||||
|
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||||
import overlayCss from "@/content/overlay.css?inline";
|
import overlayCss from "@/content/overlay.css?inline";
|
||||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||||
@@ -14,6 +15,11 @@ import {
|
|||||||
isRecordContentMessage,
|
isRecordContentMessage,
|
||||||
type RecordCaptureController,
|
type RecordCaptureController,
|
||||||
} from "@/content/record-capture";
|
} from "@/content/record-capture";
|
||||||
|
import {
|
||||||
|
type CaptureSuppressAck,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
isCaptureSuppressMessage,
|
||||||
|
} from "@/lib/capture-suppress-bridge";
|
||||||
import {
|
import {
|
||||||
HELP_ACK,
|
HELP_ACK,
|
||||||
HELP_FINISH,
|
HELP_FINISH,
|
||||||
@@ -70,6 +76,8 @@ export default defineContentScript({
|
|||||||
let hostLossReported = false;
|
let hostLossReported = false;
|
||||||
let remountInProgress = false;
|
let remountInProgress = false;
|
||||||
|
|
||||||
|
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
||||||
|
|
||||||
const ui = await createShadowRootUi(ctx, {
|
const ui = await createShadowRootUi(ctx, {
|
||||||
name: "browser-skill-overlay",
|
name: "browser-skill-overlay",
|
||||||
position: "inline",
|
position: "inline",
|
||||||
@@ -81,6 +89,8 @@ export default defineContentScript({
|
|||||||
overlayHost = shadowHost;
|
overlayHost = shadowHost;
|
||||||
overlayContainer = container;
|
overlayContainer = container;
|
||||||
hostLossReported = false;
|
hostLossReported = false;
|
||||||
|
// A host rebuilt mid-capture must stay hidden until `end` arrives.
|
||||||
|
captureSuppress.onHostMounted(shadowHost);
|
||||||
const app = document.createElement("div");
|
const app = document.createElement("div");
|
||||||
app.className = "bsk-overlay-root";
|
app.className = "bsk-overlay-root";
|
||||||
container.append(app);
|
container.append(app);
|
||||||
@@ -238,12 +248,17 @@ export default defineContentScript({
|
|||||||
| BorrowCancelMessage
|
| BorrowCancelMessage
|
||||||
| HelpRequestMessage
|
| HelpRequestMessage
|
||||||
| HelpCancelMessage
|
| HelpCancelMessage
|
||||||
|
| CaptureSuppressMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||||
) => {
|
) => {
|
||||||
|
if (isCaptureSuppressMessage(message)) {
|
||||||
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
message,
|
message,
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
CAPTURE_SUPPRESS,
|
||||||
|
type CaptureSuppressMessage,
|
||||||
|
withOverlaysHiddenForCapture,
|
||||||
|
} from "../capture-suppress-bridge";
|
||||||
|
|
||||||
|
function recordingSendToTab(events: string[]) {
|
||||||
|
return vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||||
|
events.push(message.phase);
|
||||||
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("withOverlaysHiddenForCapture", () => {
|
||||||
|
it("sends begin → fn → end and returns fn's result", async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(
|
||||||
|
7,
|
||||||
|
async () => {
|
||||||
|
events.push("capture");
|
||||||
|
return "shot";
|
||||||
|
},
|
||||||
|
sendToTab,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||||
|
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "end" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still sends end when fn throws, then rethrows", async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const sendToTab = recordingSendToTab(events);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
withOverlaysHiddenForCapture(
|
||||||
|
7,
|
||||||
|
async () => {
|
||||||
|
events.push("capture");
|
||||||
|
throw new Error("capture exploded");
|
||||||
|
},
|
||||||
|
sendToTab,
|
||||||
|
),
|
||||||
|
).rejects.toThrow("capture exploded");
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures directly when the tab has no content script", async () => {
|
||||||
|
const sendToTab = vi.fn(async () => {
|
||||||
|
throw new Error("Could not establish connection. Receiving end does not exist.");
|
||||||
|
});
|
||||||
|
const fn = vi.fn(async () => "shot");
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(fn).toHaveBeenCalledTimes(1);
|
||||||
|
// begin failed → no matching end may be sent.
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("swallows end failures so the capture result stands", async () => {
|
||||||
|
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||||
|
if (message.phase === "end") throw new Error("tab navigated away");
|
||||||
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||||
|
|
||||||
|
expect(result).toBe("shot");
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
/**
|
||||||
|
* Wire protocol and background-side helper for hiding the extension's
|
||||||
|
* in-page overlay while a screenshot is captured, so captured frames only
|
||||||
|
* contain the page itself.
|
||||||
|
*
|
||||||
|
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||||
|
* which sends `begin` (the content script hides the overlay host and only
|
||||||
|
* acks once the compositor has produced an overlay-free frame) and always
|
||||||
|
* follows up with `end`, even when the capture throws. Tabs without the
|
||||||
|
* content script (chrome://, the Web Store, ...) host no overlay, so a
|
||||||
|
* failed `begin` falls through to capturing directly.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const CAPTURE_SUPPRESS = "bsk/capture-suppress";
|
||||||
|
|
||||||
|
export type CaptureSuppressPhase = "begin" | "end";
|
||||||
|
|
||||||
|
export interface CaptureSuppressMessage {
|
||||||
|
type: typeof CAPTURE_SUPPRESS;
|
||||||
|
phase: CaptureSuppressPhase;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureSuppressAck {
|
||||||
|
type: typeof CAPTURE_SUPPRESS;
|
||||||
|
ok: true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isCaptureSuppressMessage(msg: unknown): msg is CaptureSuppressMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return m.type === CAPTURE_SUPPRESS && (m.phase === "begin" || m.phase === "end");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Minimal `chrome.tabs.sendMessage` surface so tests can inject a fake. */
|
||||||
|
export type CaptureSuppressSendToTab = (
|
||||||
|
tabId: number,
|
||||||
|
message: CaptureSuppressMessage,
|
||||||
|
) => Promise<unknown>;
|
||||||
|
|
||||||
|
const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, message);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run `fn` with the target tab's overlay hidden. `end` is sent from a
|
||||||
|
* `finally` so the overlay always reappears, and a missing content script
|
||||||
|
* (or any other `begin` failure) simply skips suppression — there is no
|
||||||
|
* overlay to hide in that tab.
|
||||||
|
*/
|
||||||
|
export async function withOverlaysHiddenForCapture<T>(
|
||||||
|
tabId: number,
|
||||||
|
fn: () => Promise<T>,
|
||||||
|
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||||
|
): Promise<T> {
|
||||||
|
let began = false;
|
||||||
|
try {
|
||||||
|
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||||
|
began = true;
|
||||||
|
} catch {
|
||||||
|
// No content script in the target tab → no overlay can leak into the
|
||||||
|
// capture; proceed without suppression.
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} finally {
|
||||||
|
if (began) {
|
||||||
|
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "end" }).catch((err) => {
|
||||||
|
console.debug("[bsk capture] overlay restore failed", err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { renderVom } from "@browser-skill/vom";
|
import { renderVom } from "@browser-skill/vom";
|
||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge";
|
||||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
|
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
|
||||||
import { SessionManager } from "@/session-manager/manager";
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
import type { CdpRunner } from "@/tools/shared";
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
@@ -345,6 +346,98 @@ describe("handleScreenshot", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleScreenshot overlay suppression", () => {
|
||||||
|
/**
|
||||||
|
* Fake background→content bridge that mirrors the content script's
|
||||||
|
* contract: `begin` hides the overlay, `end` restores it. Records the
|
||||||
|
* phase order so tests can assert begin → capture → end.
|
||||||
|
*/
|
||||||
|
function fakeSuppressBridge(events: string[]) {
|
||||||
|
let overlayHidden = false;
|
||||||
|
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||||
|
events.push(message.phase);
|
||||||
|
overlayHidden = message.phase === "begin";
|
||||||
|
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||||
|
});
|
||||||
|
return { sendToTab, isHidden: () => overlayHidden };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("hides the overlay around a visible-tab capture", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const events: string[] = [];
|
||||||
|
const bridge = fakeSuppressBridge(events);
|
||||||
|
const capture = vi.fn(async (_w: number) => {
|
||||||
|
events.push("capture");
|
||||||
|
// The overlay must already be hidden when the capture runs.
|
||||||
|
expect(bridge.isHidden()).toBe(true);
|
||||||
|
return `data:image/png;base64,${TINY_PNG}`;
|
||||||
|
});
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab: bridge.sendToTab },
|
||||||
|
);
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.image_base64).toBe(TINY_PNG);
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
expect(bridge.isHidden()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("hides the overlay around a CDP element capture", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e5", 999, { tabId: 7 });
|
||||||
|
const events: string[] = [];
|
||||||
|
const bridge = fakeSuppressBridge(events);
|
||||||
|
const { cdp } = makeFakeCdp({
|
||||||
|
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||||
|
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||||
|
"Page.captureScreenshot": () => {
|
||||||
|
events.push("capture");
|
||||||
|
expect(bridge.isHidden()).toBe(true);
|
||||||
|
return { data: TINY_PNG };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||||
|
{
|
||||||
|
...makeScreenshotDeps({
|
||||||
|
cdp,
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(),
|
||||||
|
captureVisibleTab: vi.fn(),
|
||||||
|
}),
|
||||||
|
sendToTab: bridge.sendToTab,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.image_base64).toBe(TINY_PNG);
|
||||||
|
expect(events).toEqual(["begin", "capture", "end"]);
|
||||||
|
expect(bridge.isHidden()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still captures when the tab has no content script to ack", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const sendToTab = vi.fn(async () => {
|
||||||
|
throw new Error("Could not establish connection. Receiving end does not exist.");
|
||||||
|
});
|
||||||
|
const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`);
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab },
|
||||||
|
);
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(res.image_base64).toBe(TINY_PNG);
|
||||||
|
expect(capture).toHaveBeenCalledTimes(1);
|
||||||
|
// The failed begin must not be followed by an end.
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// buildVomScene
|
// buildVomScene
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -11,6 +11,10 @@ import {
|
|||||||
type VomScene,
|
type VomScene,
|
||||||
} from "@browser-skill/vom";
|
} from "@browser-skill/vom";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
|
import {
|
||||||
|
type CaptureSuppressSendToTab,
|
||||||
|
withOverlaysHiddenForCapture,
|
||||||
|
} from "@/lib/capture-suppress-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type {
|
import type {
|
||||||
GetHtmlParams,
|
GetHtmlParams,
|
||||||
@@ -114,6 +118,12 @@ export interface ScreenshotDeps {
|
|||||||
cdp?: SharedCdpRunner;
|
cdp?: SharedCdpRunner;
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
captureApi: ChromeTabsCaptureApi;
|
captureApi: ChromeTabsCaptureApi;
|
||||||
|
/**
|
||||||
|
* Bridge used to hide the in-page overlay while a screenshot is taken,
|
||||||
|
* so captured frames only contain page content. Defaults to
|
||||||
|
* `chrome.tabs.sendMessage`; tests inject a fake.
|
||||||
|
*/
|
||||||
|
sendToTab?: CaptureSuppressSendToTab;
|
||||||
}
|
}
|
||||||
|
|
||||||
function defaultScreenshotDeps(): ScreenshotDeps {
|
function defaultScreenshotDeps(): ScreenshotDeps {
|
||||||
@@ -192,7 +202,12 @@ export async function handleScreenshot(
|
|||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||||
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
|
const cdp = deps.cdp;
|
||||||
|
const captured = await withOverlaysHiddenForCapture(
|
||||||
|
target.tabId,
|
||||||
|
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId),
|
||||||
|
deps.sendToTab,
|
||||||
|
);
|
||||||
if (isRpcError(captured)) return captured;
|
if (isRpcError(captured)) return captured;
|
||||||
return withShotDialogs({
|
return withShotDialogs({
|
||||||
image_base64: captured.image_base64,
|
image_base64: captured.image_base64,
|
||||||
@@ -212,7 +227,11 @@ export async function handleScreenshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
|
const dataUrl = await withOverlaysHiddenForCapture(
|
||||||
|
target.tabId,
|
||||||
|
() => deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }),
|
||||||
|
deps.sendToTab,
|
||||||
|
);
|
||||||
const image_base64 = stripDataUrlPrefix(dataUrl);
|
const image_base64 = stripDataUrlPrefix(dataUrl);
|
||||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||||
return withShotDialogs({
|
return withShotDialogs({
|
||||||
|
|||||||
Reference in New Issue
Block a user