mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): expose record-safe semantic observations
Share the VOM capture pipeline with recording, enrich rendered refs, and support value redaction without regressing cooperative cancellation.
This commit is contained in:
@@ -8,6 +8,7 @@ import {
|
|||||||
buildFrameVomScene,
|
buildFrameVomScene,
|
||||||
buildVomScene,
|
buildVomScene,
|
||||||
type CdpAxNode,
|
type CdpAxNode,
|
||||||
|
captureVomObservation,
|
||||||
handleGetHtml,
|
handleGetHtml,
|
||||||
handleObserve,
|
handleObserve,
|
||||||
handleScreenshot,
|
handleScreenshot,
|
||||||
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||||
const axNodes: CdpAxNode[] = [
|
const axNodes: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
nodeId: "1",
|
nodeId: "1",
|
||||||
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
id: 10,
|
id: 10,
|
||||||
sensitive: false,
|
sensitive: true,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1659,7 +1660,9 @@ describe("buildVomScene", () => {
|
|||||||
);
|
);
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "close"');
|
expect(rendered.text).toContain('@e1 button "close"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||||
@@ -1803,7 +1806,9 @@ describe("buildVomScene", () => {
|
|||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||||
@@ -2908,6 +2913,41 @@ describe("handleSnapshot", () => {
|
|||||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
it("exposes the frame-aware observation result for recording", async () => {
|
||||||
|
const ax: CdpAxNode[] = [
|
||||||
|
{
|
||||||
|
nodeId: "root",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
backendDOMNodeId: 11,
|
||||||
|
childIds: ["password"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
nodeId: "password",
|
||||||
|
parentId: "root",
|
||||||
|
role: { type: "role", value: "textbox" },
|
||||||
|
name: { type: "computedString", value: "Password" },
|
||||||
|
value: { value: "hunter2" },
|
||||||
|
backendDOMNodeId: 13,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||||
|
|
||||||
|
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||||
|
redactValues: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.text).not.toContain("hunter2");
|
||||||
|
expect(result.text).toContain("•••");
|
||||||
|
expect(result.refs[0]).toMatchObject({
|
||||||
|
backendNodeId: 13,
|
||||||
|
role: "textbox",
|
||||||
|
name: "Password",
|
||||||
|
line: expect.any(Number),
|
||||||
|
});
|
||||||
|
expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true);
|
||||||
|
expect(result.documents).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
export {
|
||||||
|
type CaptureVomObservationOptions,
|
||||||
|
type CaptureVomObservationResult,
|
||||||
|
captureVomObservation,
|
||||||
|
} from "./observation";
|
||||||
@@ -8,8 +8,10 @@ import {
|
|||||||
applyVomInteractionRecovery,
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
isVomReferenceNode,
|
isVomReferenceNode,
|
||||||
|
type RenderedRef,
|
||||||
renderVom,
|
renderVom,
|
||||||
type VomNode,
|
type VomNode,
|
||||||
|
type VomOptions,
|
||||||
type VomScene,
|
type VomScene,
|
||||||
} from "@browser-skill/vom";
|
} from "@browser-skill/vom";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
@@ -48,6 +50,7 @@ import {
|
|||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import {
|
import {
|
||||||
type CapturedNode,
|
type CapturedNode,
|
||||||
|
type CapturedSurfaceProbe,
|
||||||
type CapturedViewModel,
|
type CapturedViewModel,
|
||||||
captureViewModel,
|
captureViewModel,
|
||||||
collectOverlayExcludedBackendIds,
|
collectOverlayExcludedBackendIds,
|
||||||
@@ -411,7 +414,15 @@ function isModalSignal(
|
|||||||
|
|
||||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||||
const attrs = capturedNode?.attrs ?? {};
|
const attrs = capturedNode?.attrs ?? {};
|
||||||
return (attrs.type ?? "").toLowerCase() === "password";
|
const type = (attrs.type ?? "").toLowerCase();
|
||||||
|
if (type === "password") return true;
|
||||||
|
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||||
|
return (
|
||||||
|
autocomplete.startsWith("cc-") ||
|
||||||
|
autocomplete === "one-time-code" ||
|
||||||
|
autocomplete === "current-password" ||
|
||||||
|
autocomplete === "new-password"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AxNodeSignals {
|
interface AxNodeSignals {
|
||||||
@@ -461,7 +472,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
|||||||
"columnheader",
|
"columnheader",
|
||||||
"rowheader",
|
"rowheader",
|
||||||
]);
|
]);
|
||||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||||
|
"password",
|
||||||
|
"credit-card",
|
||||||
|
"one-time-code",
|
||||||
|
"current-password",
|
||||||
|
"new-password",
|
||||||
|
]);
|
||||||
|
|
||||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||||
const prop = axNode.properties?.find((item) => item.name === name);
|
const prop = axNode.properties?.find((item) => item.name === name);
|
||||||
@@ -1537,14 +1554,13 @@ async function fallbackCapturedViewModel(
|
|||||||
try {
|
try {
|
||||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
throwIfAborted(signal, "observation");
|
throwIfAborted(signal, "observation");
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
viewport = {
|
viewport = {
|
||||||
width: vpSrc.clientWidth ?? 0,
|
width: source.clientWidth ?? 0,
|
||||||
height: vpSrc.clientHeight ?? 0,
|
height: source.clientHeight ?? 0,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (error) {
|
||||||
if (isAbortError(err)) throw err;
|
if (isAbortError(error)) throw error;
|
||||||
// viewport stays zero-sized
|
|
||||||
}
|
}
|
||||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||||
throwIfAborted(signal, "observation");
|
throwIfAborted(signal, "observation");
|
||||||
@@ -1554,22 +1570,69 @@ async function fallbackCapturedViewModel(
|
|||||||
async function captureForVom(
|
async function captureForVom(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
conditionalSurfaceProbe: boolean,
|
options: CaptureVomObservationOptions,
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
try {
|
try {
|
||||||
return await captureViewModel(cdp, tabId, {
|
return await captureViewModel(cdp, tabId, {
|
||||||
conditionalSurfaceProbe,
|
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||||
hoverProbeBypassOverlay,
|
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||||
signal,
|
signal: options.signal,
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (error) {
|
||||||
if (isAbortError(err)) throw err;
|
if (isAbortError(error)) throw error;
|
||||||
return fallbackCapturedViewModel(cdp, tabId, signal);
|
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationResult {
|
||||||
|
text: string;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
truncated: boolean;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
documents: CapturedFrameDocument<CdpAxNode>[];
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureVomObservation(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
url: string | undefined,
|
||||||
|
options: CaptureVomObservationOptions = {},
|
||||||
|
): Promise<CaptureVomObservationResult> {
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const captured = await captureForVom(cdp, tabId, options);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const scene =
|
||||||
|
captured.rootFrameId && captured.frameNodes?.size
|
||||||
|
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||||
|
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||||
|
const rendered = renderVom(scene, {
|
||||||
|
maxDepth: options.maxDepth,
|
||||||
|
maxTokens: options.maxTokens,
|
||||||
|
redactValues: options.redactValues,
|
||||||
|
activeRegionPolicy: options.activeRegionPolicy,
|
||||||
|
});
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
return {
|
||||||
|
text: rendered.text,
|
||||||
|
refs: rendered.refs,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
captured: captured.nodes,
|
||||||
|
documents,
|
||||||
|
surfaceProbes: captured.surfaceProbes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function handleVomObservation(
|
async function handleVomObservation(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SnapshotParams | ObserveParams,
|
params: SnapshotParams | ObserveParams,
|
||||||
@@ -1599,37 +1662,19 @@ async function handleVomObservation(
|
|||||||
try {
|
try {
|
||||||
throwIfAborted(signal, toolName);
|
throwIfAborted(signal, toolName);
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
|
||||||
throwIfAborted(signal, toolName);
|
|
||||||
const effectiveConditionalSurfaceProbe =
|
const effectiveConditionalSurfaceProbe =
|
||||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||||
const captured = await captureForVom(
|
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
effectiveConditionalSurfaceProbe,
|
|
||||||
deps.hoverProbeBypassOverlay,
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
throwIfAborted(signal, toolName);
|
|
||||||
const frameDocuments = await captureFrameData<CdpAxNode>(
|
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
captured,
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
throwIfAborted(signal, toolName);
|
|
||||||
const scene =
|
|
||||||
captured.rootFrameId && captured.frameNodes?.size
|
|
||||||
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
|
|
||||||
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
|
|
||||||
const rendered = renderVom(scene, {
|
|
||||||
maxDepth: params.max_depth,
|
maxDepth: params.max_depth,
|
||||||
maxTokens: params.max_tokens,
|
maxTokens: params.max_tokens,
|
||||||
activeRegionPolicy: true,
|
activeRegionPolicy: true,
|
||||||
|
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||||
|
signal,
|
||||||
});
|
});
|
||||||
throwIfAborted(signal, toolName);
|
throwIfAborted(signal, toolName);
|
||||||
const targetByFrameId = new Map(
|
const targetByFrameId = new Map(
|
||||||
frameDocuments.map((document) => [document.frameId, document.target]),
|
rendered.documents.map((document) => [document.frameId, document.target]),
|
||||||
);
|
);
|
||||||
ctx.refStore.replace(
|
ctx.refStore.replace(
|
||||||
rendered.refs.map((ref) => {
|
rendered.refs.map((ref) => {
|
||||||
@@ -1653,7 +1698,7 @@ async function handleVomObservation(
|
|||||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
? {
|
? {
|
||||||
debug: {
|
debug: {
|
||||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
|
||||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||||
trigger_action: probe.triggerAction,
|
trigger_action: probe.triggerAction,
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||||
|
return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
|
||||||
|
}
|
||||||
|
|
||||||
describe("renderVom single-layer page", () => {
|
describe("renderVom single-layer page", () => {
|
||||||
it("does not derive handle context across frame scopes", () => {
|
it("does not derive handle context across frame scopes", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
@@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
' @e2 button "加入购物车"',
|
' @e2 button "加入购物车"',
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 3 },
|
{ ref: "e1", backendNodeId: 3 },
|
||||||
{ ref: "e2", backendNodeId: 6 },
|
{ ref: "e2", backendNodeId: 6 },
|
||||||
]);
|
]);
|
||||||
@@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(' heading "Title"');
|
expect(out.text).toContain(' heading "Title"');
|
||||||
expect(out.text).toContain(' img "Logo"');
|
expect(out.text).toContain(' img "Logo"');
|
||||||
expect(out.text).toContain(' @e1 button "Continue"');
|
expect(out.text).toContain(' @e1 button "Continue"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("assigns refs to listbox controls", () => {
|
it("assigns refs to listbox controls", () => {
|
||||||
@@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("annotates external links regardless of role casing", () => {
|
it("annotates external links regardless of role casing", () => {
|
||||||
@@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" alertdialog");
|
expect(out.text).toContain(" alertdialog");
|
||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).toContain(' @e1 button "Close"');
|
expect(out.text).toContain(' @e1 button "Close"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("skips generic nodes without adding an extra indentation level", () => {
|
it("skips generic nodes without adding an extra indentation level", () => {
|
||||||
@@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).not.toContain("Too deep");
|
expect(out.text).not.toContain("Too deep");
|
||||||
expect(out.text).toContain(' @e1 button "Later"');
|
expect(out.text).toContain(' @e1 button "Later"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||||
@@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Open settings"');
|
expect(out.text).toContain('@e1 button "Open settings"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||||
@@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Card"');
|
expect(out.text).not.toContain('@e1 button "Card"');
|
||||||
expect(out.text).toContain('@e1 button "Details"');
|
expect(out.text).toContain('@e1 button "Details"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not recover clickable wrappers around native controls", () => {
|
it("does not recover clickable wrappers around native controls", () => {
|
||||||
@@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||||
expect(out.text).toContain('@e1 button "Pay"');
|
expect(out.text).toContain('@e1 button "Pay"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("adds context to duplicate weak action labels", () => {
|
it("adds context to duplicate weak action labels", () => {
|
||||||
@@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain("@layers 1 focus=L1");
|
expect(out.text).toContain("@layers 1 focus=L1");
|
||||||
expect(out.text).toContain('@e1 button "Background"');
|
expect(out.text).toContain('@e1 button "Background"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||||
@@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).not.toContain("active-region");
|
expect(out.text).not.toContain("active-region");
|
||||||
expect(out.text).not.toContain("Background");
|
expect(out.text).not.toContain("Background");
|
||||||
expect(out.text).toContain('@e1 button "Foreground"');
|
expect(out.text).toContain('@e1 button "Foreground"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps paint-order comparisons inside their frame document", () => {
|
it("keeps paint-order comparisons inside their frame document", () => {
|
||||||
@@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||||
@@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
" Bob - great sound",
|
" Bob - great sound",
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside named ref controls", () => {
|
it("does not render redundant children inside named ref controls", () => {
|
||||||
@@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Save"');
|
expect(out.text).toContain('@e1 button "Save"');
|
||||||
expect(out.text).not.toContain("disk icon");
|
expect(out.text).not.toContain("disk icon");
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside native text inputs", () => {
|
it("does not render redundant children inside native text inputs", () => {
|
||||||
@@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => {
|
|||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.text).not.toContain("底层按钮");
|
expect(out.text).not.toContain("底层按钮");
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 4 },
|
{ ref: "e1", backendNodeId: 4 },
|
||||||
{ ref: "e2", backendNodeId: 5 },
|
{ ref: "e2", backendNodeId: 5 },
|
||||||
]);
|
]);
|
||||||
@@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => {
|
|||||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("redacts form values and returns semantic metadata for rendered refs", () => {
|
||||||
|
const out = renderVom(
|
||||||
|
scene([
|
||||||
|
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||||
|
node({
|
||||||
|
id: 2,
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentId: 1,
|
||||||
|
role: "textbox",
|
||||||
|
name: "Email",
|
||||||
|
value: "user@example.com",
|
||||||
|
inputState: "filled",
|
||||||
|
tag: "input",
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(out.text).toContain('[filled] ="•••"');
|
||||||
|
expect(out.text).not.toContain("user@example.com");
|
||||||
|
expect(out.refs[0]).toMatchObject({
|
||||||
|
ref: "e1",
|
||||||
|
backendNodeId: 42,
|
||||||
|
frameId: "child-frame",
|
||||||
|
role: "textbox",
|
||||||
|
name: "Email",
|
||||||
|
line: expect.any(Number),
|
||||||
|
});
|
||||||
|
expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export type {
|
|||||||
CondSurface,
|
CondSurface,
|
||||||
LayerKind,
|
LayerKind,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
Viewport,
|
Viewport,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
|
|||||||
@@ -4,9 +4,9 @@ import type {
|
|||||||
BlockingLayer,
|
BlockingLayer,
|
||||||
CondSurface,
|
CondSurface,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
VomRef,
|
|
||||||
VomResult,
|
VomResult,
|
||||||
VomScene,
|
VomScene,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
@@ -615,6 +615,7 @@ function renderNodeLine(
|
|||||||
ref: string | undefined,
|
ref: string | undefined,
|
||||||
context: string[] = [],
|
context: string[] = [],
|
||||||
surface: CondSurface | undefined = undefined,
|
surface: CondSurface | undefined = undefined,
|
||||||
|
redactValues = false,
|
||||||
): string {
|
): string {
|
||||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||||
|
|
||||||
@@ -636,13 +637,14 @@ function renderNodeLine(
|
|||||||
}
|
}
|
||||||
const placeholder = cleaned(node.placeholder);
|
const placeholder = cleaned(node.placeholder);
|
||||||
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
||||||
const sensitiveHasValue =
|
const hasValue =
|
||||||
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
|
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
|
||||||
const value = node.sensitive
|
const value =
|
||||||
? sensitiveHasValue
|
node.sensitive || redactValues
|
||||||
? SENSITIVE_MASK
|
? hasValue
|
||||||
: undefined
|
? SENSITIVE_MASK
|
||||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
: undefined
|
||||||
|
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||||
|
|
||||||
if (surface && surface.subItems.length > 0) {
|
if (surface && surface.subItems.length > 0) {
|
||||||
@@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
|
|||||||
|
|
||||||
interface RenderState {
|
interface RenderState {
|
||||||
lines: string[];
|
lines: string[];
|
||||||
refs: VomRef[];
|
refs: RenderedRef[];
|
||||||
nextRef: number;
|
nextRef: number;
|
||||||
tokens: number;
|
tokens: number;
|
||||||
maxDepth: number;
|
maxDepth: number;
|
||||||
maxTokens: number;
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
stopped: boolean;
|
stopped: boolean;
|
||||||
children: Map<number | null, VomNode[]>;
|
children: Map<number | null, VomNode[]>;
|
||||||
@@ -764,7 +767,7 @@ function renderTree(
|
|||||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||||
const context = ref ? handleContext(node, state) : [];
|
const context = ref ? handleContext(node, state) : [];
|
||||||
const surface = state.surfaceMap.get(node.id);
|
const surface = state.surfaceMap.get(node.id);
|
||||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||||
const nextTokens = state.tokens + estimateTokens(line);
|
const nextTokens = state.tokens + estimateTokens(line);
|
||||||
if (nextTokens > state.maxTokens) {
|
if (nextTokens > state.maxTokens) {
|
||||||
state.truncated = true;
|
state.truncated = true;
|
||||||
@@ -775,10 +778,15 @@ function renderTree(
|
|||||||
state.lines.push(line);
|
state.lines.push(line);
|
||||||
state.tokens = nextTokens;
|
state.tokens = nextTokens;
|
||||||
if (ref) {
|
if (ref) {
|
||||||
|
const name = cleaned(node.name);
|
||||||
state.refs.push({
|
state.refs.push({
|
||||||
ref,
|
ref,
|
||||||
backendNodeId: node.backendNodeId ?? node.id,
|
backendNodeId: node.backendNodeId ?? node.id,
|
||||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||||
|
...(node.role ? { role: node.role } : {}),
|
||||||
|
...(name ? { name } : {}),
|
||||||
|
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||||
|
line: state.lines.length - 1,
|
||||||
});
|
});
|
||||||
state.nextRef += 1;
|
state.nextRef += 1;
|
||||||
}
|
}
|
||||||
@@ -808,6 +816,7 @@ function renderNodes(
|
|||||||
tokens: estimateTokens(initialLines.join("\n")),
|
tokens: estimateTokens(initialLines.join("\n")),
|
||||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||||
|
redactValues: options.redactValues ?? false,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
stopped: false,
|
stopped: false,
|
||||||
children,
|
children,
|
||||||
|
|||||||
@@ -64,6 +64,10 @@ export interface VomScene {
|
|||||||
export interface VomOptions {
|
export interface VomOptions {
|
||||||
maxDepth?: number;
|
maxDepth?: number;
|
||||||
maxTokens?: number;
|
maxTokens?: number;
|
||||||
|
/**
|
||||||
|
* When true, form values are rendered as masks instead of literal values.
|
||||||
|
*/
|
||||||
|
redactValues?: boolean;
|
||||||
/**
|
/**
|
||||||
* Experimental: filter refs that are geometrically blocked by foreground
|
* Experimental: filter refs that are geometrically blocked by foreground
|
||||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||||
@@ -90,9 +94,17 @@ export interface VomRef {
|
|||||||
frameId?: string;
|
frameId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RenderedRef extends VomRef {
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
ctx?: string;
|
||||||
|
/** Zero-based line index in `VomResult.text`. */
|
||||||
|
line: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface VomResult {
|
export interface VomResult {
|
||||||
text: string;
|
text: string;
|
||||||
refs: VomRef[];
|
refs: RenderedRef[];
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user