fix(screenshot): hide overlays during capture so shots only contain page content (#64)

This commit is contained in:
BB-fat
2026-08-08 22:48:50 +08:00
committed by GitHub
parent 37d3e2b22f
commit 1268761611
8 changed files with 509 additions and 3 deletions
@@ -0,0 +1,125 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { CAPTURE_SUPPRESS, type CaptureSuppressAck } from "@/lib/capture-suppress-bridge";
import { CAPTURE_HIDDEN_ATTR, createCaptureSuppressController } from "../capture-suppress";
/**
* rAF is stubbed with a manual queue so tests can drive the compositor
* frame-by-frame and assert the ack only lands after the second frame.
*/
describe("capture-suppress controller", () => {
let rafQueue: FrameRequestCallback[];
let host: HTMLElement;
beforeEach(() => {
rafQueue = [];
vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => {
rafQueue.push(cb);
return rafQueue.length;
});
vi.stubGlobal("cancelAnimationFrame", vi.fn());
document.body.innerHTML = "";
host = document.createElement("div");
document.body.append(host);
});
/** Run every queued rAF callback, draining microtasks between rounds. */
async function flushFrames(rounds = 4): Promise<void> {
for (let round = 0; round < rounds && rafQueue.length > 0; round += 1) {
const callbacks = rafQueue.splice(0);
for (const cb of callbacks) cb(0);
await Promise.resolve();
await Promise.resolve();
}
}
it("begin hides the host immediately but only acks after two frames", async () => {
const controller = createCaptureSuppressController(() => host);
const sendResponse = vi.fn();
const needsAsync = controller.handleMessage(
{ type: CAPTURE_SUPPRESS, phase: "begin" },
sendResponse,
);
expect(needsAsync).toBe(true);
expect(controller.pendingCount).toBe(1);
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
expect(sendResponse).not.toHaveBeenCalled();
// First frame commits the style change; the ack must still be pending.
await flushFrames(1);
expect(sendResponse).not.toHaveBeenCalled();
// Second frame: the compositor has produced an overlay-free frame.
await flushFrames();
const expected: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
expect(sendResponse).toHaveBeenCalledWith(expected);
});
it("end removes the attribute once the count returns to zero", async () => {
const controller = createCaptureSuppressController(() => host);
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
await flushFrames();
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
const sendResponse = vi.fn();
const needsAsync = controller.handleMessage(
{ type: CAPTURE_SUPPRESS, phase: "end" },
sendResponse,
);
expect(needsAsync).toBe(false);
expect(controller.pendingCount).toBe(0);
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
});
it("reference-counts concurrent begins so one end does not unhide early", async () => {
const controller = createCaptureSuppressController(() => host);
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
await flushFrames();
expect(controller.pendingCount).toBe(2);
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
expect(controller.pendingCount).toBe(1);
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
expect(controller.pendingCount).toBe(0);
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
});
it("treats a stray end as a no-op", () => {
const controller = createCaptureSuppressController(() => host);
const sendResponse = vi.fn();
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, sendResponse);
expect(controller.pendingCount).toBe(0);
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
});
it("re-applies the attribute when the host is remounted mid-capture", () => {
let currentHost: HTMLElement | null = host;
const controller = createCaptureSuppressController(() => currentHost);
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
// Host lost and rebuilt before `end` arrives.
currentHost = null;
const rebuilt = document.createElement("div");
controller.onHostMounted(rebuilt);
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
});
it("leaves a remounted host visible once suppression has ended", () => {
const controller = createCaptureSuppressController(() => host);
const rebuilt = document.createElement("div");
controller.onHostMounted(rebuilt);
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
});
});
@@ -0,0 +1,99 @@
/**
* Content-script side of the capture-suppress bridge: hides the overlay
* shadow host while the background captures a screenshot.
*
* `begin` sets `data-bsk-capture-hidden` on the host (overlay.css maps it
* to `display: none !important`, which removes the whole shadow subtree
* from compositing) and only acks after two animation frames — the first
* frame commits the style change, the second guarantees the compositor
* has produced a frame without the overlay, so the capture API cannot
* pick up a stale frame that still contains it. `end` is
* reference-counted so overlapping captures do not unhide the host early.
*/
import {
CAPTURE_SUPPRESS,
type CaptureSuppressAck,
type CaptureSuppressMessage,
} from "@/lib/capture-suppress-bridge";
export const CAPTURE_HIDDEN_ATTR = "data-bsk-capture-hidden";
/**
* Safety net for hidden/background tabs where rAF callbacks never fire:
* the attribute is already set synchronously, so any frame the compositor
* produces from now on excludes the overlay. Waiting longer would just
* deadlock the capture.
*/
const FRAME_FALLBACK_MS = 500;
function nextFrame(): Promise<void> {
return new Promise((resolve) => {
const raf = requestAnimationFrame(() => {
clearTimeout(timer);
resolve();
});
const timer = setTimeout(() => {
cancelAnimationFrame(raf);
resolve();
}, FRAME_FALLBACK_MS);
});
}
export interface CaptureSuppressController {
/**
* Handle one bridge message. Returns true when the ack is sent
* asynchronously and the caller must keep the message channel open.
*/
handleMessage(
message: CaptureSuppressMessage,
sendResponse: (ack: CaptureSuppressAck) => void,
): boolean;
/** Re-apply the hidden attribute when the overlay host is (re)mounted. */
onHostMounted(host: HTMLElement): void;
/** Number of in-flight `begin` phases (exposed for tests). */
readonly pendingCount: number;
}
export function createCaptureSuppressController(
getHost: () => HTMLElement | null,
): CaptureSuppressController {
let pending = 0;
const ack: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
function setHidden(host: HTMLElement | null, hidden: boolean): void {
if (!host) return;
if (hidden) host.setAttribute(CAPTURE_HIDDEN_ATTR, "");
else host.removeAttribute(CAPTURE_HIDDEN_ATTR);
}
return {
get pendingCount() {
return pending;
},
handleMessage(message, sendResponse) {
if (message.phase === "begin") {
pending += 1;
setHidden(getHost(), true);
void (async () => {
await nextFrame();
await nextFrame();
sendResponse(ack);
})();
return true;
}
// `end` without a matching `begin` (e.g. a retried message) must not
// drive the counter negative or clear a concurrent capture's flag.
pending = Math.max(0, pending - 1);
if (pending === 0) setHidden(getHost(), false);
sendResponse(ack);
return false;
},
onHostMounted(host) {
// Host lost + rebuilt mid-capture: the fresh host must stay hidden
// until the last `end` arrives.
setHidden(host, pending > 0);
},
};
}
+7
View File
@@ -14,3 +14,10 @@
:host([data-bsk-overlay-surface][data-bsk-overlay-blocking]) {
pointer-events: auto !important;
}
/* Screenshot suppression: the host lives in the main DOM, so hiding it
removes the whole shadow subtree from compositing and captured frames
only contain page content. */
:host([data-bsk-capture-hidden]) {
display: none !important;
}
+16 -1
View File
@@ -5,6 +5,7 @@ import { flushSync } from "react-dom";
import ReactDOM from "react-dom/client";
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay";
import { createCaptureSuppressController } from "@/content/capture-suppress";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
@@ -14,6 +15,11 @@ import {
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
type CaptureSuppressAck,
type CaptureSuppressMessage,
isCaptureSuppressMessage,
} from "@/lib/capture-suppress-bridge";
import {
HELP_ACK,
HELP_FINISH,
@@ -70,6 +76,8 @@ export default defineContentScript({
let hostLossReported = false;
let remountInProgress = false;
const captureSuppress = createCaptureSuppressController(() => overlayHost);
const ui = await createShadowRootUi(ctx, {
name: "browser-skill-overlay",
position: "inline",
@@ -81,6 +89,8 @@ export default defineContentScript({
overlayHost = shadowHost;
overlayContainer = container;
hostLossReported = false;
// A host rebuilt mid-capture must stay hidden until `end` arrives.
captureSuppress.onHostMounted(shadowHost);
const app = document.createElement("div");
app.className = "bsk-overlay-root";
container.append(app);
@@ -238,12 +248,17 @@ export default defineContentScript({
| BorrowCancelMessage
| HelpRequestMessage
| HelpCancelMessage
| CaptureSuppressMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
) => {
if (isCaptureSuppressMessage(message)) {
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
@@ -0,0 +1,77 @@
import { describe, expect, it, vi } from "vitest";
import {
CAPTURE_SUPPRESS,
type CaptureSuppressMessage,
withOverlaysHiddenForCapture,
} from "../capture-suppress-bridge";
function recordingSendToTab(events: string[]) {
return vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
events.push(message.phase);
return { type: CAPTURE_SUPPRESS, ok: true };
});
}
describe("withOverlaysHiddenForCapture", () => {
it("sends begin → fn → end and returns fn's result", async () => {
const events: string[] = [];
const sendToTab = recordingSendToTab(events);
const result = await withOverlaysHiddenForCapture(
7,
async () => {
events.push("capture");
return "shot";
},
sendToTab,
);
expect(result).toBe("shot");
expect(events).toEqual(["begin", "capture", "end"]);
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "begin" });
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "end" });
});
it("still sends end when fn throws, then rethrows", async () => {
const events: string[] = [];
const sendToTab = recordingSendToTab(events);
await expect(
withOverlaysHiddenForCapture(
7,
async () => {
events.push("capture");
throw new Error("capture exploded");
},
sendToTab,
),
).rejects.toThrow("capture exploded");
expect(events).toEqual(["begin", "capture", "end"]);
});
it("captures directly when the tab has no content script", async () => {
const sendToTab = vi.fn(async () => {
throw new Error("Could not establish connection. Receiving end does not exist.");
});
const fn = vi.fn(async () => "shot");
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
expect(result).toBe("shot");
expect(fn).toHaveBeenCalledTimes(1);
// begin failed → no matching end may be sent.
expect(sendToTab).toHaveBeenCalledTimes(1);
});
it("swallows end failures so the capture result stands", async () => {
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
if (message.phase === "end") throw new Error("tab navigated away");
return { type: CAPTURE_SUPPRESS, ok: true };
});
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
expect(result).toBe("shot");
expect(sendToTab).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,71 @@
/**
* Wire protocol and background-side helper for hiding the extension's
* in-page overlay while a screenshot is captured, so captured frames only
* contain the page itself.
*
* The background wraps every capture in `withOverlaysHiddenForCapture`,
* which sends `begin` (the content script hides the overlay host and only
* acks once the compositor has produced an overlay-free frame) and always
* follows up with `end`, even when the capture throws. Tabs without the
* content script (chrome://, the Web Store, ...) host no overlay, so a
* failed `begin` falls through to capturing directly.
*/
export const CAPTURE_SUPPRESS = "bsk/capture-suppress";
export type CaptureSuppressPhase = "begin" | "end";
export interface CaptureSuppressMessage {
type: typeof CAPTURE_SUPPRESS;
phase: CaptureSuppressPhase;
}
export interface CaptureSuppressAck {
type: typeof CAPTURE_SUPPRESS;
ok: true;
}
export function isCaptureSuppressMessage(msg: unknown): msg is CaptureSuppressMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === CAPTURE_SUPPRESS && (m.phase === "begin" || m.phase === "end");
}
/** Minimal `chrome.tabs.sendMessage` surface so tests can inject a fake. */
export type CaptureSuppressSendToTab = (
tabId: number,
message: CaptureSuppressMessage,
) => Promise<unknown>;
const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
chrome.tabs.sendMessage(tabId, message);
/**
* Run `fn` with the target tab's overlay hidden. `end` is sent from a
* `finally` so the overlay always reappears, and a missing content script
* (or any other `begin` failure) simply skips suppression — there is no
* overlay to hide in that tab.
*/
export async function withOverlaysHiddenForCapture<T>(
tabId: number,
fn: () => Promise<T>,
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
): Promise<T> {
let began = false;
try {
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "begin" });
began = true;
} catch {
// No content script in the target tab → no overlay can leak into the
// capture; proceed without suppression.
}
try {
return await fn();
} finally {
if (began) {
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "end" }).catch((err) => {
console.debug("[bsk capture] overlay restore failed", err);
});
}
}
}
@@ -1,5 +1,6 @@
import { renderVom } from "@browser-skill/vom";
import { describe, expect, it, vi } from "vitest";
import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
@@ -345,6 +346,98 @@ describe("handleScreenshot", () => {
});
});
describe("handleScreenshot overlay suppression", () => {
/**
* Fake background→content bridge that mirrors the content script's
* contract: `begin` hides the overlay, `end` restores it. Records the
* phase order so tests can assert begin → capture → end.
*/
function fakeSuppressBridge(events: string[]) {
let overlayHidden = false;
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
events.push(message.phase);
overlayHidden = message.phase === "begin";
return { type: CAPTURE_SUPPRESS, ok: true };
});
return { sendToTab, isHidden: () => overlayHidden };
}
it("hides the overlay around a visible-tab capture", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const events: string[] = [];
const bridge = fakeSuppressBridge(events);
const capture = vi.fn(async (_w: number) => {
events.push("capture");
// The overlay must already be hidden when the capture runs.
expect(bridge.isHidden()).toBe(true);
return `data:image/png;base64,${TINY_PNG}`;
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11" },
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab: bridge.sendToTab },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.image_base64).toBe(TINY_PNG);
expect(events).toEqual(["begin", "capture", "end"]);
expect(bridge.isHidden()).toBe(false);
});
it("hides the overlay around a CDP element capture", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 });
const events: string[] = [];
const bridge = fakeSuppressBridge(events);
const { cdp } = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => {
events.push("capture");
expect(bridge.isHidden()).toBe(true);
return { data: TINY_PNG };
},
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
{
...makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
sendToTab: bridge.sendToTab,
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.image_base64).toBe(TINY_PNG);
expect(events).toEqual(["begin", "capture", "end"]);
expect(bridge.isHidden()).toBe(false);
});
it("still captures when the tab has no content script to ack", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const sendToTab = vi.fn(async () => {
throw new Error("Could not establish connection. Receiving end does not exist.");
});
const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`);
const res = await handleScreenshot(
sm,
{ session_id: "aa11" },
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.image_base64).toBe(TINY_PNG);
expect(capture).toHaveBeenCalledTimes(1);
// The failed begin must not be followed by an end.
expect(sendToTab).toHaveBeenCalledTimes(1);
});
});
// ---------------------------------------------------------------------------
// buildVomScene
// ---------------------------------------------------------------------------
+21 -2
View File
@@ -11,6 +11,10 @@ import {
type VomScene,
} from "@browser-skill/vom";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import {
type CaptureSuppressSendToTab,
withOverlaysHiddenForCapture,
} from "@/lib/capture-suppress-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type {
GetHtmlParams,
@@ -114,6 +118,12 @@ export interface ScreenshotDeps {
cdp?: SharedCdpRunner;
tabsApi: ChromeTabsApi;
captureApi: ChromeTabsCaptureApi;
/**
* Bridge used to hide the in-page overlay while a screenshot is taken,
* so captured frames only contain page content. Defaults to
* `chrome.tabs.sendMessage`; tests inject a fake.
*/
sendToTab?: CaptureSuppressSendToTab;
}
function defaultScreenshotDeps(): ScreenshotDeps {
@@ -192,7 +202,12 @@ export async function handleScreenshot(
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
const cdp = deps.cdp;
const captured = await withOverlaysHiddenForCapture(
target.tabId,
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId),
deps.sendToTab,
);
if (isRpcError(captured)) return captured;
return withShotDialogs({
image_base64: captured.image_base64,
@@ -212,7 +227,11 @@ export async function handleScreenshot(
}
try {
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
const dataUrl = await withOverlaysHiddenForCapture(
target.tabId,
() => deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }),
deps.sendToTab,
);
const image_base64 = stripDataUrlPrefix(dataUrl);
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
return withShotDialogs({