figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extension): record user flows as trace v3 page-state observations

Browse Source 
Capture settled page observations into a states/steps trace, match targets by
geometry with capture fallbacks, and reuse VOM rendering with richer refs and
optional value redaction for record bundles.
This commit is contained in:
“haonan”
2026-08-13 19:34:37 +08:00
parent 1508839bf9
commit 13a61b2688
28 changed files with 3993 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/vom/src/__tests__/render.test.ts
+39 -14
View File
@@ -24,6 +24,11 @@ function scene(nodes: VomNode[]): VomScene {
};
}
/** Compare refs without recording-metadata fields (role/name/ctx/line). */
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
return refs.map(({ ref: id, backendNodeId }) => ({ ref: id, backendNodeId }));
}
describe("renderVom single-layer page", () => {
it("always emits a complete @vom single-layer document when there is no blocker", () => {
const out = renderVom(
@@ -51,7 +56,7 @@ describe("renderVom single-layer page", () => {
' @e2 button "加入购物车"',
].join("\n"),
);
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 3 },
{ ref: "e2", backendNodeId: 6 },
]);
@@ -72,7 +77,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(' heading "Title"');
expect(out.text).toContain(' img "Logo"');
expect(out.text).toContain(' @e1 button "Continue"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("assigns refs to listbox controls", () => {
@@ -84,7 +89,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain(' @e1 listbox "Choices"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("annotates external links regardless of role casing", () => {
@@ -121,7 +126,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" alertdialog");
expect(out.text).toContain(" section");
expect(out.text).toContain(' @e1 button "Close"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
});
it("skips generic nodes without adding an extra indentation level", () => {
@@ -192,7 +197,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" section");
expect(out.text).not.toContain("Too deep");
expect(out.text).toContain(' @e1 button "Later"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
@@ -227,7 +232,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain('@e1 button "Open settings"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
@@ -257,7 +262,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Card"');
expect(out.text).toContain('@e1 button "Details"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("does not recover clickable wrappers around native controls", () => {
@@ -280,7 +285,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Checkout"');
expect(out.text).toContain('@e1 button "Pay"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("adds context to duplicate weak action labels", () => {
@@ -343,7 +348,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('@e1 button "Background"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("can filter refs blocked by active positioned regions without changing layer output", () => {
@@ -386,7 +391,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("active-region");
expect(out.text).not.toContain("Background");
expect(out.text).toContain('@e1 button "Foreground"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("renders conditional surface items inline on the trigger", () => {
@@ -405,7 +410,7 @@ describe("renderVom single-layer page", () => {
});
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("injects active scope blocks immediately after their trigger without refs", () => {
@@ -431,7 +436,7 @@ describe("renderVom single-layer page", () => {
" Bob - great sound",
].join("\n"),
);
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside named ref controls", () => {
@@ -445,7 +450,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain('@e1 button "Save"');
expect(out.text).not.toContain("disk icon");
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside native text inputs", () => {
@@ -612,7 +617,7 @@ describe("renderVom double-layer page", () => {
].join("\n"),
);
expect(out.text).not.toContain("底层按钮");
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 4 },
{ ref: "e2", backendNodeId: 5 },
]);
@@ -664,4 +669,24 @@ describe("renderVom double-layer page", () => {
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
});
it("redactValues omits literal form values", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Form" }),
node({
id: 2,
parentId: 1,
role: "textbox",
name: "邮箱",
value: "user@example.com",
inputState: "filled",
tag: "input",
}),
]),
{ redactValues: true },
);
expect(out.text).toContain("[filled]");
expect(out.text).not.toContain("user@example.com");
});
});