源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)

packages/agent/gui/workbench/browser-element-context/BrowserElementContextAction.tsx

@@ -0,0 +1,78 @@
+import { useCallback, useState } from "react";
+import type { ReactNode } from "react";
+import {
+  Button,
+  InspectIcon,
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+  cn
+} from "@tutti-os/ui-system";
+import { useActiveBrowserNodeWebview } from "@tutti-os/browser-node/react";
+import { useEngineSelector } from "../../shared/engine/useEngineSelector.ts";
+import { BrowserElementContextSelectionController } from "./browserElementContextSelectionController.ts";
+
+export interface BrowserElementContextCopy {
+  cancel: string;
+  failed: string;
+  select: string;
+}
+
+export function BrowserElementContextAction({
+  copy,
+  onAppendMention,
+  onError,
+  workspaceId
+}: {
+  copy: BrowserElementContextCopy;
+  onAppendMention: (mention: string) => void;
+  onError: (message: string) => void;
+  workspaceId: string;
+}): ReactNode {
+  const activeWebview = useActiveBrowserNodeWebview();
+  const [controller] = useState(
+    () =>
+      new BrowserElementContextSelectionController({
+        failedCopy: copy.failed,
+        onAppendMention,
+        onError,
+        workspaceId
+      })
+  );
+  controller.configure({
+    failedCopy: copy.failed,
+    onAppendMention,
+    onError,
+    workspaceId
+  });
+  const state = useEngineSelector(controller, (snapshot) => snapshot.state);
+  const bindAction = useCallback(
+    (node: HTMLButtonElement | null) =>
+      controller.bindAction(node, activeWebview),
+    [activeWebview, controller]
+  );
+  const label = state === "selecting" ? copy.cancel : copy.select;
+  return (
+    <Tooltip>
+      <TooltipTrigger asChild>
+        <Button
+          aria-label={label}
+          aria-pressed={state === "selecting"}
+          className={cn(
+            "nodrag shrink-0 rounded-md",
+            state === "selecting" &&
+              "bg-[var(--transparency-block)] text-[var(--text-primary)]"
+          )}
+          size="icon-sm"
+          ref={bindAction}
+          type="button"
+          variant="chrome"
+          onClick={() => controller.toggle(activeWebview)}
+        >
+          <InspectIcon className="size-[15px]" />
+        </Button>
+      </TooltipTrigger>
+      <TooltipContent side="bottom">{label}</TooltipContent>
+    </Tooltip>
+  );
+}

packages/agent/gui/workbench/browser-element-context/browserElementContextSelectionController.ts

@@ -0,0 +1,249 @@
+import type { BrowserNodeWebviewTag } from "@tutti-os/browser-node/react";
+import {
+  normalizeBrowserElementSelectionResult,
+  serializeBrowserElementSnapshot
+} from "./browserElementSnapshot.ts";
+import {
+  browserElementSelectorScript,
+  cancelBrowserElementSelectorScript
+} from "./browserElementSelectorScript.ts";
+import {
+  cancelBrowserElementWebviewSelection,
+  executeBrowserElementWebviewScript
+} from "./browserElementWebview.ts";
+import { createBrowserElementMentionMarkdown } from "./browserElementMention.ts";
+
+export interface BrowserElementSelectionControllerInput {
+  failedCopy: string;
+  onAppendMention: (mention: string) => void;
+  onError: (message: string) => void;
+  workspaceId: string;
+}
+
+export interface BrowserElementSelectionSnapshot {
+  state: "idle" | "selecting";
+}
+
+interface BrowserElementSelectionSession {
+  attempt: number;
+  navigationPending: boolean;
+  webview: BrowserNodeWebviewTag | null;
+}
+
+export class BrowserElementContextSelectionController {
+  private activeWebview: BrowserNodeWebviewTag | null = null;
+  private detachFrame: number | null = null;
+  private input: BrowserElementSelectionControllerInput;
+  private readonly listeners = new Set<
+    (snapshot: BrowserElementSelectionSnapshot) => void
+  >();
+  private session: BrowserElementSelectionSession | null = null;
+  private snapshot: BrowserElementSelectionSnapshot = { state: "idle" };
+  private subscribedWebview: BrowserNodeWebviewTag | null = null;
+
+  constructor(input: BrowserElementSelectionControllerInput) {
+    this.input = input;
+  }
+
+  readonly getSnapshot = (): BrowserElementSelectionSnapshot => this.snapshot;
+
+  readonly subscribe = (
+    listener: (snapshot: BrowserElementSelectionSnapshot) => void
+  ): (() => void) => {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  };
+
+  configure(input: BrowserElementSelectionControllerInput): void {
+    this.input = input;
+  }
+
+  bindAction(
+    node: HTMLButtonElement | null,
+    activeWebview: BrowserNodeWebviewTag | null
+  ): void {
+    if (node) {
+      if (this.detachFrame !== null) {
+        cancelAnimationFrame(this.detachFrame);
+        this.detachFrame = null;
+      }
+      this.moveToActiveWebview(activeWebview);
+      return;
+    }
+    if (this.detachFrame !== null) cancelAnimationFrame(this.detachFrame);
+    this.detachFrame = requestAnimationFrame(() => {
+      this.detachFrame = null;
+      this.dispose();
+    });
+  }
+
+  toggle(activeWebview: BrowserNodeWebviewTag | null): void {
+    if (this.session) {
+      this.endSelection(this.session);
+      return;
+    }
+    if (!activeWebview?.executeJavaScript) {
+      this.input.onError(this.input.failedCopy);
+      return;
+    }
+    const session: BrowserElementSelectionSession = {
+      attempt: 0,
+      navigationPending: false,
+      webview: null
+    };
+    this.session = session;
+    this.setState("selecting");
+    this.moveToActiveWebview(activeWebview);
+  }
+
+  private readonly handleStartLoading = (): void => {
+    if (this.session) this.session.navigationPending = true;
+  };
+
+  private readonly handleDomReady = (): void => {
+    const session = this.session;
+    const webview = this.activeWebview;
+    if (!session || !webview) return;
+    session.navigationPending = false;
+    void this.moveSelectionToWebview(session, webview, true);
+  };
+
+  private moveToActiveWebview(webview: BrowserNodeWebviewTag | null): void {
+    this.activeWebview = webview;
+    const session = this.session;
+    if (!session) return;
+    if (!webview) {
+      const previousWebview = session.webview;
+      session.webview = null;
+      session.attempt += 1;
+      this.unsubscribeFromWebview();
+      void cancelBrowserElementWebviewSelection(
+        previousWebview,
+        cancelBrowserElementSelectorScript
+      );
+      return;
+    }
+    this.subscribeToWebview(webview);
+    void this.moveSelectionToWebview(session, webview);
+  }
+
+  private subscribeToWebview(webview: BrowserNodeWebviewTag): void {
+    if (this.subscribedWebview === webview) return;
+    this.unsubscribeFromWebview();
+    this.subscribedWebview = webview;
+    webview.addEventListener("did-start-loading", this.handleStartLoading);
+    webview.addEventListener("dom-ready", this.handleDomReady);
+  }
+
+  private unsubscribeFromWebview(): void {
+    this.subscribedWebview?.removeEventListener(
+      "did-start-loading",
+      this.handleStartLoading
+    );
+    this.subscribedWebview?.removeEventListener(
+      "dom-ready",
+      this.handleDomReady
+    );
+    this.subscribedWebview = null;
+  }
+
+  private async moveSelectionToWebview(
+    session: BrowserElementSelectionSession,
+    webview: BrowserNodeWebviewTag,
+    force = false
+  ): Promise<void> {
+    if (this.session !== session) return;
+    if (!force && session.webview === webview && session.attempt > 0) return;
+    const previousWebview = session.webview;
+    session.webview = webview;
+    session.navigationPending = false;
+    const attempt = ++session.attempt;
+    await cancelBrowserElementWebviewSelection(
+      previousWebview && (force || previousWebview !== webview)
+        ? previousWebview
+        : null,
+      cancelBrowserElementSelectorScript
+    );
+    if (!this.isCurrentSelectionAttempt(session, attempt)) return;
+    void this.runSelectionAttempt(session, webview, attempt);
+  }
+
+  private async runSelectionAttempt(
+    session: BrowserElementSelectionSession,
+    webview: BrowserNodeWebviewTag,
+    attempt: number
+  ): Promise<void> {
+    try {
+      const rawResult = await executeBrowserElementWebviewScript(
+        webview,
+        browserElementSelectorScript,
+        true
+      );
+      if (!this.isCurrentSelectionAttempt(session, attempt)) return;
+      const result = normalizeBrowserElementSelectionResult(rawResult);
+      if (!result || result.status === "cancelled") {
+        this.endSelection(session);
+        return;
+      }
+      const content = serializeBrowserElementSnapshot(result.snapshot);
+      const mention = createBrowserElementMentionMarkdown({
+        context: content,
+        id: createBrowserElementReferenceId(),
+        tagName: result.snapshot.element.tagName,
+        workspaceId: this.input.workspaceId
+      });
+      if (!mention)
+        throw new Error("Browser element mention could not be created");
+      this.input.onAppendMention(mention);
+      this.endSelection(session);
+    } catch {
+      if (!this.isCurrentSelectionAttempt(session, attempt)) return;
+      // A guest navigation destroys the injected Promise. Keep the selection
+      // session alive until the new document emits dom-ready.
+      if (session.navigationPending) return;
+      this.input.onError(this.input.failedCopy);
+      this.endSelection(session);
+    }
+  }
+
+  private isCurrentSelectionAttempt(
+    session: BrowserElementSelectionSession,
+    attempt: number
+  ): boolean {
+    return this.session === session && session.attempt === attempt;
+  }
+
+  private endSelection(session: BrowserElementSelectionSession): void {
+    if (this.session !== session) return;
+    this.session = null;
+    session.attempt += 1;
+    const webview = session.webview;
+    session.webview = null;
+    this.unsubscribeFromWebview();
+    void cancelBrowserElementWebviewSelection(
+      webview,
+      cancelBrowserElementSelectorScript
+    );
+    this.setState("idle");
+  }
+
+  private setState(state: BrowserElementSelectionSnapshot["state"]): void {
+    if (this.snapshot.state === state) return;
+    this.snapshot = { state };
+    for (const listener of this.listeners) listener(this.snapshot);
+  }
+
+  private dispose(): void {
+    const session = this.session;
+    if (session) this.endSelection(session);
+    this.unsubscribeFromWebview();
+    this.activeWebview = null;
+  }
+}
+
+function createBrowserElementReferenceId(): string {
+  return `browser-element:${
+    globalThis.crypto?.randomUUID?.() ??
+    `${Date.now()}-${Math.random().toString(36).slice(2)}`
+  }`;
+}

packages/agent/gui/workbench/browser-element-context/browserElementMention.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+import { parseRichTextMentionHref } from "@tutti-os/ui-rich-text/core";
+import {
+  browserElementMentionLabel,
+  createBrowserElementMentionMarkdown,
+  presentBrowserElementMention
+} from "./browserElementMention.ts";
+
+describe("browser element mentions", () => {
+  it("preserve prompt text and show only the DOM tag", () => {
+    const context = [
+      "DOM Path: #app > div",
+      "Position: top=0px",
+      "HTML Element: <div>Hello</div>"
+    ].join("\n");
+    const markdown = createBrowserElementMentionMarkdown({
+      context,
+      id: "browser-element:1",
+      tagName: "A",
+      workspaceId: "workspace-1"
+    });
+    const href = markdown.slice(markdown.indexOf("(") + 1, -1);
+    const mention = parseRichTextMentionHref(href);
+
+    expect(markdown).toMatch(/^\[@a\]\(/u);
+    expect(browserElementMentionLabel(mention?.scope?.tag ?? "")).toBe("<a>");
+    expect(mention?.providerId).toBe("browser-element");
+    expect(mention?.scope?.context).toBe(context);
+    expect(mention?.scope?.workspaceId).toBe("workspace-1");
+  });
+
+  it("reject incomplete references", () => {
+    expect(
+      createBrowserElementMentionMarkdown({
+        context: "",
+        id: "browser-element:1",
+        tagName: "div",
+        workspaceId: "workspace-1"
+      })
+    ).toBe("");
+  });
+
+  it("keep historical references presentable without inline context", () => {
+    expect(
+      presentBrowserElementMention({
+        label: "a",
+        scope: { tag: "a", workspaceId: "workspace-1" }
+      })
+    ).toEqual({ name: "<a>", workspaceId: "workspace-1" });
+  });
+});

packages/agent/gui/workbench/browser-element-context/browserElementMention.ts

@@ -0,0 +1,66 @@
+import { createRichTextMentionMarkdown } from "@tutti-os/ui-rich-text/core";
+
+export const browserElementMentionKind = "browser-element";
+
+export interface BrowserElementMentionReference {
+  context: string;
+  id: string;
+  tagName: string;
+  workspaceId: string;
+}
+
+export interface BrowserElementMentionIdentity {
+  label: string;
+  scope?: Readonly<Record<string, string>>;
+}
+
+export interface BrowserElementMentionPresentation {
+  name: string;
+  workspaceId?: string;
+}
+
+export function createBrowserElementMentionMarkdown(
+  reference: BrowserElementMentionReference
+): string {
+  const tagName = normalizeBrowserElementTagName(reference.tagName);
+  const context = reference.context.trim();
+  const workspaceId = reference.workspaceId.trim();
+  if (!tagName || !context || !workspaceId) {
+    return "";
+  }
+  return createRichTextMentionMarkdown({
+    providerId: browserElementMentionKind,
+    entityId: reference.id,
+    label: tagName,
+    scope: { context, tag: tagName, workspaceId }
+  });
+}
+
+export function browserElementMentionLabel(tagName: string): string {
+  const normalized = normalizeBrowserElementTagName(tagName);
+  return normalized ? `<${normalized}>` : "";
+}
+
+export function presentBrowserElementMention(
+  mention: BrowserElementMentionIdentity
+): BrowserElementMentionPresentation | null {
+  const label = browserElementMentionLabel(
+    mention.scope?.tag?.trim() || mention.label
+  );
+  if (!label) {
+    return null;
+  }
+  const workspaceId = mention.scope?.workspaceId?.trim() ?? "";
+  return {
+    name: label,
+    ...(workspaceId ? { workspaceId } : {})
+  };
+}
+
+function normalizeBrowserElementTagName(value: string): string {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9:-]/gu, "")
+    .slice(0, 100);
+}

packages/agent/gui/workbench/browser-element-context/browserElementSelectorScript.ts

@@ -0,0 +1,302 @@
+function runTuttiBrowserElementSelector(): Promise<unknown> {
+  type SelectorGlobal = typeof globalThis & {
+    __tuttiBrowserElementSelector?: { cancel(): void };
+  };
+  const selectorGlobal = globalThis as SelectorGlobal;
+  selectorGlobal.__tuttiBrowserElementSelector?.cancel();
+
+  return new Promise((resolve) => {
+    const overlay = document.createElement("div");
+    overlay.setAttribute("data-tutti-browser-element-highlight", "true");
+    Object.assign(overlay.style, {
+      background: "rgba(86, 156, 214, 0.12)",
+      border: "2px solid rgb(86, 156, 214)",
+      borderRadius: "3px",
+      boxSizing: "border-box",
+      display: "none",
+      left: "0",
+      pointerEvents: "none",
+      position: "fixed",
+      top: "0",
+      zIndex: "2147483647"
+    });
+    document.documentElement.append(overlay);
+    const previousCursor = document.documentElement.style.cursor;
+    document.documentElement.style.cursor = "crosshair";
+
+    const finish = (value: unknown): void => {
+      document.removeEventListener("mousemove", handleMouseMove, true);
+      document.removeEventListener("click", handleClick, true);
+      document.removeEventListener("keydown", handleKeyDown, true);
+      document.documentElement.style.cursor = previousCursor;
+      overlay.remove();
+      delete selectorGlobal.__tuttiBrowserElementSelector;
+      resolve(value);
+    };
+    const cancel = (): void => finish({ status: "cancelled" });
+    selectorGlobal.__tuttiBrowserElementSelector = { cancel };
+
+    function handleMouseMove(event: MouseEvent): void {
+      const target = event.target;
+      if (!(target instanceof Element) || target === overlay) return;
+      const rect = target.getBoundingClientRect();
+      Object.assign(overlay.style, {
+        display: "block",
+        height: `${Math.max(0, rect.height)}px`,
+        transform: `translate(${rect.left}px, ${rect.top}px)`,
+        width: `${Math.max(0, rect.width)}px`
+      });
+    }
+
+    function handleClick(event: MouseEvent): void {
+      const target = event.target;
+      if (!(target instanceof Element) || target === overlay) return;
+      event.preventDefault();
+      event.stopImmediatePropagation();
+      finish({ snapshot: captureSnapshot(target), status: "selected" });
+    }
+
+    function handleKeyDown(event: KeyboardEvent): void {
+      if (event.key !== "Escape") return;
+      event.preventDefault();
+      event.stopImmediatePropagation();
+      cancel();
+    }
+
+    function captureSnapshot(target: Element): Record<string, unknown> {
+      const rect = target.getBoundingClientRect();
+      const rawText = isFormControl(target)
+        ? ""
+        : (target.textContent ?? "").replace(/\s+/gu, " ").trim();
+      const clone = target.cloneNode(true) as Element;
+      clone
+        .querySelectorAll("script,style,noscript,template")
+        .forEach((item) => item.remove());
+      sanitizeElement(clone);
+      clone.querySelectorAll("*").forEach(sanitizeElement);
+      const rawHtml = clone.outerHTML;
+      const computed = getComputedStyle(target);
+      const styleNames = [
+        "display",
+        "position",
+        "color",
+        "background-color",
+        "font-family",
+        "font-size",
+        "font-weight",
+        "line-height",
+        "margin",
+        "padding",
+        "border",
+        "border-radius",
+        "width",
+        "height",
+        "flex",
+        "grid-template-columns",
+        "align-items",
+        "justify-content",
+        "overflow",
+        "opacity",
+        "visibility"
+      ];
+      return {
+        format: "tutti.browser-element.v1",
+        capturedAt: new Date().toISOString(),
+        page: {
+          title: document.title,
+          url: sanitizeUrl(location.href)
+        },
+        element: {
+          ariaLabel: target.getAttribute("aria-label"),
+          attributes: readAttributes(target),
+          bounds: {
+            height: rect.height,
+            width: rect.width,
+            x: rect.x,
+            y: rect.y
+          },
+          classes: [...target.classList].slice(0, 24),
+          domPath: domPathFor(target),
+          html: rawHtml.slice(0, 32_000),
+          htmlTruncated: rawHtml.length > 32_000,
+          id: target.id || null,
+          role: target.getAttribute("role"),
+          selector: selectorFor(target),
+          styles: Object.fromEntries(
+            styleNames.map((name) => [name, computed.getPropertyValue(name)])
+          ),
+          tagName: target.tagName.toLowerCase(),
+          text: rawText.slice(0, 4_000),
+          textTruncated: rawText.length > 4_000
+        },
+        ancestors: readAncestors(target),
+        viewport: {
+          devicePixelRatio: globalThis.devicePixelRatio,
+          height: globalThis.innerHeight,
+          scrollX: globalThis.scrollX,
+          scrollY: globalThis.scrollY,
+          width: globalThis.innerWidth
+        }
+      };
+    }
+
+    function sanitizeElement(element: Element): void {
+      if (
+        element instanceof HTMLInputElement ||
+        element instanceof HTMLTextAreaElement ||
+        element instanceof HTMLSelectElement ||
+        element instanceof HTMLOptionElement
+      ) {
+        element.removeAttribute("value");
+        if ("value" in element) element.value = "";
+        if (element instanceof HTMLTextAreaElement) element.textContent = "";
+      }
+      for (const attribute of [...element.attributes]) {
+        const name = attribute.name.toLowerCase();
+        if (
+          name.startsWith("on") ||
+          name === "nonce" ||
+          name === "integrity" ||
+          name === "srcdoc" ||
+          /(?:token|auth|password|secret|session|cookie|api[-_]?key)/iu.test(
+            name
+          )
+        ) {
+          element.setAttribute(attribute.name, "[redacted]");
+          continue;
+        }
+        if (name === "href" || name === "src" || name === "action") {
+          element.setAttribute(attribute.name, sanitizeUrl(attribute.value));
+        }
+      }
+    }
+
+    function isFormControl(element: Element): boolean {
+      return (
+        element instanceof HTMLInputElement ||
+        element instanceof HTMLTextAreaElement ||
+        element instanceof HTMLSelectElement
+      );
+    }
+
+    function readAttributes(element: Element): Record<string, string> {
+      const result: Record<string, string> = {};
+      for (const attribute of [...element.attributes].slice(0, 40)) {
+        const name = attribute.name.toLowerCase();
+        if (name.startsWith("on") || name === "value" || name === "srcdoc") {
+          continue;
+        }
+        result[attribute.name] =
+          /(?:token|auth|password|secret|session|cookie|api[-_]?key)/iu.test(
+            name
+          )
+            ? "[redacted]"
+            : name === "href" || name === "src" || name === "action"
+              ? sanitizeUrl(attribute.value)
+              : attribute.value.slice(0, 2_000);
+      }
+      return result;
+    }
+
+    function sanitizeUrl(value: string): string {
+      try {
+        const url = new URL(value, location.href);
+        url.username = "";
+        url.password = "";
+        url.hash = "";
+        for (const key of [...url.searchParams.keys()]) {
+          if (
+            /(?:token|auth|password|secret|session|cookie|key|signature)/iu.test(
+              key
+            )
+          ) {
+            url.searchParams.set(key, "[redacted]");
+          }
+        }
+        return url.toString();
+      } catch {
+        return value.slice(0, 4_000);
+      }
+    }
+
+    function selectorFor(element: Element): string {
+      if (element.id) return `#${escapeCss(element.id)}`;
+      const segments: string[] = [];
+      let current: Element | null = element;
+      while (current && segments.length < 8) {
+        let segment = current.tagName.toLowerCase();
+        const parentElement: Element | null = current.parentElement;
+        if (parentElement) {
+          const currentTagName = current.tagName;
+          const siblings = [...parentElement.children].filter(
+            (item) => item.tagName === currentTagName
+          );
+          if (siblings.length > 1) {
+            segment += `:nth-of-type(${siblings.indexOf(current) + 1})`;
+          }
+        }
+        segments.unshift(segment);
+        current = parentElement;
+      }
+      return segments.join(" > ");
+    }
+
+    function domPathFor(element: Element): string {
+      const appRoot = document.querySelector("#app");
+      const root =
+        appRoot?.contains(element) === true
+          ? appRoot
+          : (document.body ?? document.documentElement);
+      const segments: string[] = [];
+      let current: Element | null = element;
+      while (current) {
+        segments.unshift(domPathSegment(current, current === root));
+        if (current === root) break;
+        current = current.parentElement;
+      }
+      return segments.join(" > ").slice(0, 4_000);
+    }
+
+    function domPathSegment(element: Element, isRoot: boolean): string {
+      if (isRoot && element.id === "app") return "#app";
+      const tagName = element.tagName.toLowerCase();
+      const classNames = [...element.classList]
+        .filter(Boolean)
+        .slice(0, 6)
+        .map((name) => `.${name}`)
+        .join("");
+      if (classNames) return `${tagName}${classNames}`;
+      if (element.id) return `${tagName}#${element.id}`;
+      return tagName;
+    }
+
+    function escapeCss(value: string): string {
+      return globalThis.CSS?.escape
+        ? globalThis.CSS.escape(value)
+        : value.replace(/[^a-zA-Z0-9_-]/gu, (character) => `\\${character}`);
+    }
+
+    function readAncestors(element: Element): Array<Record<string, unknown>> {
+      const result: Array<Record<string, unknown>> = [];
+      let current = element.parentElement;
+      while (current && result.length < 8) {
+        result.push({
+          id: current.id || null,
+          role: current.getAttribute("role"),
+          selector: selectorFor(current),
+          tagName: current.tagName.toLowerCase()
+        });
+        current = current.parentElement;
+      }
+      return result;
+    }
+
+    document.addEventListener("mousemove", handleMouseMove, true);
+    document.addEventListener("click", handleClick, true);
+    document.addEventListener("keydown", handleKeyDown, true);
+  });
+}
+
+export const browserElementSelectorScript = `(${runTuttiBrowserElementSelector.toString()})()`;
+export const cancelBrowserElementSelectorScript =
+  "globalThis.__tuttiBrowserElementSelector?.cancel();";

packages/agent/gui/workbench/browser-element-context/browserElementSnapshot.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import {
+  browserElementSnapshotFormat,
+  browserElementSnapshotMaxHtmlChars,
+  normalizeBrowserElementSelectionResult,
+  serializeBrowserElementSnapshot
+} from "./browserElementSnapshot.ts";
+
+describe("browser element snapshots", () => {
+  it("bound content and redact secret-looking URL parameters", () => {
+    const result = normalizeBrowserElementSelectionResult({
+      status: "selected",
+      snapshot: {
+        capturedAt: "2026-07-15T00:00:00.000Z",
+        page: {
+          title: "Example",
+          url: "https://example.com/page?token=secret&tab=main#private"
+        },
+        element: {
+          attributes: {},
+          bounds: { height: 10, width: 20, x: 1, y: 2 },
+          classes: ["primary"],
+          domPath: "#app > main.page > button.primary",
+          html: "x".repeat(browserElementSnapshotMaxHtmlChars + 100),
+          selector: "#submit",
+          styles: {},
+          tagName: "BUTTON",
+          text: "Submit"
+        },
+        viewport: { height: 800, width: 1200 }
+      }
+    });
+
+    expect(result?.status).toBe("selected");
+    if (result?.status !== "selected") return;
+    expect(result.snapshot.format).toBe(browserElementSnapshotFormat);
+    expect(result.snapshot.element.html.length).toBe(
+      browserElementSnapshotMaxHtmlChars
+    );
+    expect(result.snapshot.element.tagName).toBe("button");
+    expect(result.snapshot.page.url).toBe(
+      "https://example.com/page?token=%5Bredacted%5D&tab=main"
+    );
+  });
+
+  it("serialize to Cursor's three-field text format", () => {
+    const result = normalizeBrowserElementSelectionResult({
+      status: "selected",
+      snapshot: {
+        page: { title: "Example", url: "https://example.com" },
+        element: {
+          bounds: { height: 40.125, width: 120.5, x: 8, y: -0 },
+          domPath: "#app > div.page-wrapper-n1Pp9 > a.nav-link",
+          html: '<a href="https://example.com">\n  Example home\n</a>',
+          tagName: "a"
+        }
+      }
+    });
+
+    expect(result?.status).toBe("selected");
+    if (result?.status !== "selected") return;
+    expect(serializeBrowserElementSnapshot(result.snapshot)).toBe(
+      [
+        "DOM Path: #app > div.page-wrapper-n1Pp9 > a.nav-link",
+        "Position: top=0px, left=8px, width=120.5px, height=40.13px",
+        'HTML Element: <a href="https://example.com"> Example home </a>'
+      ].join("\n")
+    );
+  });
+
+  it("reject selected results without identity", () => {
+    expect(
+      normalizeBrowserElementSelectionResult({
+        status: "selected",
+        snapshot: { element: {}, page: {}, viewport: {} }
+      })
+    ).toBeNull();
+  });
+});

packages/agent/gui/workbench/browser-element-context/browserElementSnapshot.ts

@@ -0,0 +1,214 @@
+export const browserElementSnapshotFormat = "tutti.browser-element.v1";
+export const browserElementSnapshotMaxHtmlChars = 32_000;
+export const browserElementSnapshotMaxTextChars = 4_000;
+
+export interface BrowserElementSnapshot {
+  format: typeof browserElementSnapshotFormat;
+  capturedAt: string;
+  page: {
+    title: string;
+    url: string;
+  };
+  element: {
+    ariaLabel: string | null;
+    attributes: Record<string, string>;
+    bounds: { height: number; width: number; x: number; y: number };
+    classes: string[];
+    domPath: string;
+    html: string;
+    htmlTruncated: boolean;
+    id: string | null;
+    role: string | null;
+    selector: string;
+    styles: Record<string, string>;
+    tagName: string;
+    text: string;
+    textTruncated: boolean;
+  };
+  ancestors: Array<{
+    id: string | null;
+    role: string | null;
+    selector: string;
+    tagName: string;
+  }>;
+  viewport: {
+    devicePixelRatio: number;
+    height: number;
+    scrollX: number;
+    scrollY: number;
+    width: number;
+  };
+}
+
+export type BrowserElementSelectionResult =
+  | { status: "cancelled" }
+  | { snapshot: BrowserElementSnapshot; status: "selected" };
+
+export function normalizeBrowserElementSelectionResult(
+  value: unknown
+): BrowserElementSelectionResult | null {
+  if (!isRecord(value)) return null;
+  if (value.status === "cancelled") return { status: "cancelled" };
+  if (value.status !== "selected" || !isRecord(value.snapshot)) return null;
+  const snapshot = value.snapshot;
+  const page = isRecord(snapshot.page) ? snapshot.page : {};
+  const element = isRecord(snapshot.element) ? snapshot.element : {};
+  const bounds = isRecord(element.bounds) ? element.bounds : {};
+  const viewport = isRecord(snapshot.viewport) ? snapshot.viewport : {};
+  const normalized: BrowserElementSnapshot = {
+    format: browserElementSnapshotFormat,
+    capturedAt:
+      boundedString(snapshot.capturedAt, 64) || new Date().toISOString(),
+    page: {
+      title: boundedString(page.title, 500),
+      url: sanitizeSnapshotUrl(boundedString(page.url, 4_000))
+    },
+    element: {
+      ariaLabel: nullableBoundedString(element.ariaLabel, 1_000),
+      attributes: boundedStringRecord(element.attributes, 40, 200, 2_000),
+      bounds: {
+        height: finiteNumber(bounds.height),
+        width: finiteNumber(bounds.width),
+        x: finiteNumber(bounds.x),
+        y: finiteNumber(bounds.y)
+      },
+      classes: boundedStringArray(element.classes, 24, 200),
+      domPath:
+        boundedString(element.domPath, 4_000) ||
+        boundedString(element.selector, 2_000),
+      html: boundedString(element.html, browserElementSnapshotMaxHtmlChars),
+      htmlTruncated:
+        element.htmlTruncated === true ||
+        (typeof element.html === "string" &&
+          element.html.length > browserElementSnapshotMaxHtmlChars),
+      id: nullableBoundedString(element.id, 500),
+      role: nullableBoundedString(element.role, 200),
+      selector: boundedString(element.selector, 2_000),
+      styles: boundedStringRecord(element.styles, 32, 100, 500),
+      tagName: boundedString(element.tagName, 100).toLowerCase(),
+      text: boundedString(element.text, browserElementSnapshotMaxTextChars),
+      textTruncated:
+        element.textTruncated === true ||
+        (typeof element.text === "string" &&
+          element.text.length > browserElementSnapshotMaxTextChars)
+    },
+    ancestors: Array.isArray(snapshot.ancestors)
+      ? snapshot.ancestors.slice(0, 8).flatMap((item) =>
+          isRecord(item)
+            ? [
+                {
+                  id: nullableBoundedString(item.id, 500),
+                  role: nullableBoundedString(item.role, 200),
+                  selector: boundedString(item.selector, 2_000),
+                  tagName: boundedString(item.tagName, 100).toLowerCase()
+                }
+              ]
+            : []
+        )
+      : [],
+    viewport: {
+      devicePixelRatio: finiteNumber(viewport.devicePixelRatio, 1),
+      height: finiteNumber(viewport.height),
+      scrollX: finiteNumber(viewport.scrollX),
+      scrollY: finiteNumber(viewport.scrollY),
+      width: finiteNumber(viewport.width)
+    }
+  };
+  if (!normalized.element.tagName || !normalized.element.domPath) return null;
+  return { snapshot: normalized, status: "selected" };
+}
+
+export function serializeBrowserElementSnapshot(
+  snapshot: BrowserElementSnapshot
+): string {
+  const { bounds } = snapshot.element;
+  return [
+    `DOM Path: ${snapshot.element.domPath}`,
+    `Position: top=${formatPixel(bounds.y)}, left=${formatPixel(bounds.x)}, width=${formatPixel(bounds.width)}, height=${formatPixel(bounds.height)}`,
+    `HTML Element: ${compactHtmlFragment(snapshot.element.html)}`
+  ].join("\n");
+}
+
+function formatPixel(value: number): string {
+  const rounded = Math.round(value * 100) / 100;
+  return `${Object.is(rounded, -0) ? 0 : rounded}px`;
+}
+
+function compactHtmlFragment(value: string): string {
+  return value
+    .replace(/\r?\n/gu, " ")
+    .replace(/\s{2,}/gu, " ")
+    .trim();
+}
+
+function sanitizeSnapshotUrl(value: string): string {
+  try {
+    const url = new URL(value);
+    url.username = "";
+    url.password = "";
+    url.hash = "";
+    for (const key of [...url.searchParams.keys()]) {
+      if (
+        /(?:token|auth|password|secret|session|cookie|key|signature)/iu.test(
+          key
+        )
+      ) {
+        url.searchParams.set(key, "[redacted]");
+      }
+    }
+    return url.toString();
+  } catch {
+    return boundedString(value, 4_000);
+  }
+}
+
+function boundedString(value: unknown, maxChars: number): string {
+  return typeof value === "string" ? value.slice(0, maxChars) : "";
+}
+
+function nullableBoundedString(
+  value: unknown,
+  maxChars: number
+): string | null {
+  const normalized = boundedString(value, maxChars).trim();
+  return normalized || null;
+}
+
+function boundedStringArray(
+  value: unknown,
+  maxItems: number,
+  maxChars: number
+): string[] {
+  return Array.isArray(value)
+    ? value
+        .slice(0, maxItems)
+        .filter((item): item is string => typeof item === "string")
+        .map((item) => item.slice(0, maxChars))
+    : [];
+}
+
+function boundedStringRecord(
+  value: unknown,
+  maxItems: number,
+  maxKeyChars: number,
+  maxValueChars: number
+): Record<string, string> {
+  if (!isRecord(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value)
+      .slice(0, maxItems)
+      .flatMap(([key, item]) =>
+        typeof item === "string"
+          ? [[key.slice(0, maxKeyChars), item.slice(0, maxValueChars)]]
+          : []
+      )
+  );
+}
+
+function finiteNumber(value: unknown, fallback = 0): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

packages/agent/gui/workbench/browser-element-context/browserElementWebview.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it } from "vitest";
+import type { BrowserNodeWebviewTag } from "@tutti-os/browser-node/react";
+import {
+  cancelBrowserElementWebviewSelection,
+  executeBrowserElementWebviewScript,
+  isBrowserElementWebviewReady,
+  waitForBrowserElementWebviewReady
+} from "./browserElementWebview.ts";
+
+class MockBrowserElementWebview extends EventTarget {
+  isConnected = true;
+  ready = false;
+  executedScripts: string[] = [];
+
+  executeJavaScript<T>(script: string): Promise<T> {
+    this.executedScripts.push(script);
+    return Promise.resolve("selected" as T);
+  }
+
+  getWebContentsId(): number {
+    if (!this.ready) {
+      throw new Error(
+        "The WebView must be attached to the DOM and the dom-ready event emitted"
+      );
+    }
+    return 42;
+  }
+}
+
+function asBrowserWebview(
+  webview: MockBrowserElementWebview
+): BrowserNodeWebviewTag {
+  return webview as unknown as BrowserNodeWebviewTag;
+}
+
+describe("browser element webview", () => {
+  it("waits for the active webview dom-ready event before execution", async () => {
+    const mock = new MockBrowserElementWebview();
+    const webview = asBrowserWebview(mock);
+    const execution = executeBrowserElementWebviewScript<string>(
+      webview,
+      "select()",
+      true
+    );
+
+    expect(mock.executedScripts).toEqual([]);
+    mock.ready = true;
+    mock.dispatchEvent(new Event("dom-ready"));
+
+    expect(await execution).toBe("selected");
+    expect(mock.executedScripts).toEqual(["select()"]);
+  });
+
+  it("fails detached webview readiness without executing", async () => {
+    const mock = new MockBrowserElementWebview();
+    mock.isConnected = false;
+    const webview = asBrowserWebview(mock);
+
+    expect(isBrowserElementWebviewReady(webview)).toBe(false);
+    expect(await waitForBrowserElementWebviewReady(webview, 1)).toBe(false);
+    await expect(
+      executeBrowserElementWebviewScript(webview, "select()")
+    ).rejects.toThrow(/not ready/u);
+    expect(mock.executedScripts).toEqual([]);
+  });
+
+  it("ignores a webview detached during cancellation cleanup", async () => {
+    const mock = new MockBrowserElementWebview();
+    mock.isConnected = false;
+
+    await cancelBrowserElementWebviewSelection(
+      asBrowserWebview(mock),
+      "cancel()"
+    );
+
+    expect(mock.executedScripts).toEqual([]);
+  });
+});

packages/agent/gui/workbench/browser-element-context/browserElementWebview.ts

@@ -0,0 +1,82 @@
+import type { BrowserNodeWebviewTag } from "@tutti-os/browser-node/react";
+
+export const browserElementWebviewReadyTimeoutMs = 2_000;
+
+export function isBrowserElementWebviewReady(
+  webview: BrowserNodeWebviewTag
+): boolean {
+  if (!webview.isConnected) return false;
+  try {
+    const webContentsId = webview.getWebContentsId?.();
+    return typeof webContentsId === "number" && Number.isFinite(webContentsId);
+  } catch {
+    return false;
+  }
+}
+
+export async function waitForBrowserElementWebviewReady(
+  webview: BrowserNodeWebviewTag,
+  timeoutMs = browserElementWebviewReadyTimeoutMs
+): Promise<boolean> {
+  if (isBrowserElementWebviewReady(webview)) return true;
+  if (!webview.isConnected) return false;
+
+  return new Promise((resolve) => {
+    let settled = false;
+    let timeoutSignal: AbortSignal | null = null;
+    const finish = (ready: boolean): void => {
+      if (settled) return;
+      settled = true;
+      timeoutSignal?.removeEventListener("abort", handleTimeout);
+      webview.removeEventListener("dom-ready", handleDomReady);
+      resolve(ready);
+    };
+    const handleDomReady = (): void => {
+      finish(isBrowserElementWebviewReady(webview));
+    };
+    timeoutSignal = AbortSignal.timeout(timeoutMs);
+    const handleTimeout = (): void => finish(false);
+    timeoutSignal.addEventListener("abort", handleTimeout, { once: true });
+    webview.addEventListener("dom-ready", handleDomReady, { once: true });
+
+    // Covers a dom-ready event emitted between the initial check and listener
+    // registration without adding a second BrowserNode lifecycle owner.
+    if (isBrowserElementWebviewReady(webview)) finish(true);
+  });
+}
+
+export async function executeBrowserElementWebviewScript<T>(
+  webview: BrowserNodeWebviewTag,
+  script: string,
+  userGesture = false
+): Promise<T> {
+  if (!webview.executeJavaScript) {
+    throw new Error("Browser webview JavaScript execution is unavailable");
+  }
+  if (!(await waitForBrowserElementWebviewReady(webview))) {
+    throw new Error("Browser webview is not ready");
+  }
+  return webview.executeJavaScript<T>(script, userGesture);
+}
+
+export async function cancelBrowserElementWebviewSelection(
+  webview: BrowserNodeWebviewTag | null,
+  script: string,
+  onError?: (error: unknown) => void
+): Promise<void> {
+  if (!webview?.executeJavaScript || !webview.isConnected) return;
+  try {
+    await webview.executeJavaScript(script);
+  } catch (error) {
+    // Cancellation is best-effort because navigation and unmount detach the
+    // guest before React cleanup runs.
+    if (onError) {
+      onError(error);
+    } else {
+      console.warn(
+        "[agent-gui] browser element selection cleanup failed",
+        error
+      );
+    }
+  }
+}

packages/agent/gui/workbench/browser-element-context/index.ts

@@ -0,0 +1,14 @@
+export {
+  BrowserElementContextAction,
+  type BrowserElementContextCopy
+} from "./BrowserElementContextAction";
+export {
+  browserElementMentionKind,
+  browserElementMentionLabel,
+  createBrowserElementMentionMarkdown,
+  presentBrowserElementMention,
+  type BrowserElementMentionIdentity,
+  type BrowserElementMentionPresentation,
+  type BrowserElementMentionReference
+} from "./browserElementMention";
+export { registerBrowserElementMention } from "./registerBrowserElementMention";

packages/agent/gui/workbench/browser-element-context/registerBrowserElementMention.tsx

@@ -0,0 +1,29 @@
+import { Badge } from "@tutti-os/ui-system";
+import { InspectIcon } from "@tutti-os/ui-system/icons";
+import { registerAgentCustomMentionKind } from "../../custom-mention";
+import {
+  browserElementMentionKind,
+  presentBrowserElementMention
+} from "./browserElementMention";
+
+/** Registers browser DOM references before the first AgentGUI surface mounts. */
+export function registerBrowserElementMention(): void {
+  registerAgentCustomMentionKind({
+    kind: browserElementMentionKind,
+    materializePromptText: (mention) => {
+      const context = mention.scope?.context?.trim();
+      return context ? `\n${context}\n` : null;
+    },
+    present: presentBrowserElementMention,
+    renderChip: ({ name }) => (
+      <Badge
+        className="h-5 max-w-full gap-1.5 rounded-[6px] px-2 py-0 font-[var(--font-mono)] text-[13px] font-semibold leading-5"
+        data-agent-browser-element-chip="true"
+        variant="accent"
+      >
+        <InspectIcon aria-hidden="true" className="size-3.5 shrink-0" />
+        <span className="truncate">{name}</span>
+      </Badge>
+    )
+  });
+}