BrowserSkill/apps/extension/src/content/record-capture.ts

import {
  describeEventTarget,
  describeTarget,
  resolveClickableElement,
  resolveHoverElement,
  type TargetDescriptor,
} from "@/lib/describe-target";
import {
  evaluateHoverTrigger,
  type HoverTriggerDecision,
  type HoverTriggerRect,
} from "@/lib/hover-trigger-policy";
import {
  isRecordCancelMessage,
  isRecordStartMessage,
  isRecordStopMessage,
  RECORD_CANCEL,
  RECORD_START,
  RECORD_STEP,
  RECORD_STOP,
  type RecordCancelMessage,
  type RecordStartAck,
  type RecordStartMessage,
  type RecordStepPayload,
  type RecordStopAck,
  type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { decideHoverSurfaceRelation } from "./record-hover-surface";

const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const knownRecordRequests = new Set<string>();

function sendRecordStep(requestId: string, step: RecordStepPayload): void {
  const payload = { type: RECORD_STEP, requestId, step };
  const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
    () => true,
    () => {
      failedStepDeliveries.add(requestId);
      return false;
    },
  );
  const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
  sends.add(pending);
  pendingStepSends.set(requestId, sends);
  void pending.then(() => {
    sends.delete(pending);
    if (sends.size === 0) pendingStepSends.delete(requestId);
  });
}

export interface RecordCaptureController {
  dispose(): void;
}

interface FillSession {
  element: FillableElement;
  target: TargetDescriptor;
  baselineValue: string;
  lastValue: string;
}

interface PendingHover {
  element: Element;
  target: TargetDescriptor;
  recordedAt: number;
  score: number;
  reasons: string[];
}

type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;

const HOVER_BEFORE_CLICK_MAX_MS = 10_000;
const HOVER_REPLACE_SCORE_MARGIN = 50;

function eventTarget(event: Event): EventTarget | null {
  return event.composedPath()[0] ?? event.target;
}

function isOverlayTarget(target: EventTarget | null): boolean {
  if (!(target instanceof Node)) return false;
  const root = document.documentElement;
  let node: Node | null = target;
  while (node && node !== root) {
    if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) {
      return true;
    }
    const rootNode: Node | Document | ShadowRoot = node.getRootNode();
    if (rootNode instanceof ShadowRoot) {
      const host: Element = rootNode.host;
      if (host.hasAttribute("data-bsk-overlay")) {
        return true;
      }
      node = host;
    } else {
      node = node.parentNode;
    }
  }
  return false;
}

function isTextFillable(el: Element): el is FillableElement {
  if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
    return true;
  }
  if (el instanceof HTMLTextAreaElement) return true;
  if (!(el instanceof HTMLInputElement)) return false;
  const type = el.type.toLowerCase();
  return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button";
}

function fillableFromTarget(target: EventTarget | null): FillableElement | null {
  if (!(target instanceof Element)) return null;
  if (isTextFillable(target)) return target;
  const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])');
  if (el && isTextFillable(el)) return el;
  return null;
}

/** Clicks on search chrome that only focus the nearby input should not become steps. */
function nearbyFillableFromSearchChrome(target: Element): FillableElement | null {
  const container = target.closest(
    '[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]',
  );
  if (!container) return null;
  const fillable = container.querySelector(
    'textarea, input[type="search"], input[name="q"], #chat-textarea',
  );
  if (
    fillable instanceof HTMLElement &&
    isTextFillable(fillable) &&
    fillable !== target &&
    !fillable.contains(target)
  ) {
    return fillable;
  }
  return null;
}

function fillableValue(el: FillableElement): string {
  if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
    return el.value;
  }
  return el.textContent ?? "";
}

function hoverTriggerAttrs(el: Element): Record<string, string | undefined> {
  return {
    id: el.id || undefined,
    class: typeof el.className === "string" ? el.className || undefined : undefined,
    "data-testid": el.getAttribute("data-testid") ?? undefined,
    "data-test": el.getAttribute("data-test") ?? undefined,
    "data-cy": el.getAttribute("data-cy") ?? undefined,
    "aria-label": el.getAttribute("aria-label") ?? undefined,
    "aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined,
    "aria-controls": el.getAttribute("aria-controls") ?? undefined,
    "aria-expanded": el.getAttribute("aria-expanded") ?? undefined,
    "aria-hidden": el.getAttribute("aria-hidden") ?? undefined,
    "aria-disabled": el.getAttribute("aria-disabled") ?? undefined,
    contenteditable: el.getAttribute("contenteditable") ?? undefined,
    disabled: el.hasAttribute("disabled") ? "" : undefined,
    hidden: el.hasAttribute("hidden") ? "" : undefined,
    inert: el.hasAttribute("inert") ? "" : undefined,
    onclick: el.getAttribute("onclick") ?? undefined,
    onmouseenter: el.getAttribute("onmouseenter") ?? undefined,
    onmouseover: el.getAttribute("onmouseover") ?? undefined,
    role: el.getAttribute("role") ?? undefined,
    tabindex: el.getAttribute("tabindex") ?? undefined,
    title: el.getAttribute("title") ?? undefined,
  };
}

function hoverTriggerRect(el: Element): HoverTriggerRect {
  const rect = el.getBoundingClientRect();
  return { x: rect.left, y: rect.top, w: rect.width, h: rect.height };
}

function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } {
  if (!(el instanceof HTMLElement)) return {};
  const style = getComputedStyle(el);
  return { cursor: style.cursor, pointerEvents: style.pointerEvents };
}

function hasHoverGraphicDescendant(el: Element): boolean {
  return el.querySelector("img,svg,use,path,i") !== null;
}

function isWeakHoverTarget(target: TargetDescriptor): boolean {
  return !target.role && !target.name && target.tag === "div";
}

function looksLikeAvatarElement(el: Element): boolean {
  const className = typeof el.className === "string" ? el.className : "";
  return /\b(avatar|user-avatar)\b/i.test(className);
}

function normalizeHoverTarget(
  el: Element,
  desc: TargetDescriptor,
  decision: HoverTriggerDecision,
): TargetDescriptor {
  if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
    return { ...desc, name: "image" };
  }
  if (
    isWeakHoverTarget(desc) &&
    (looksLikeAvatarElement(el) ||
      (hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only")))
  ) {
    return { tag: "img", role: "img", name: "image" };
  }
  return desc;
}

function evaluateHoverTriggerElement(
  el: Element,
  desc: TargetDescriptor,
): HoverTriggerDecision | null {
  if (!(el instanceof HTMLElement)) return null;
  const style = hoverTriggerStyle(el);
  return evaluateHoverTrigger({
    tag: el.tagName.toLowerCase(),
    role: desc.role,
    label: desc.name,
    attrs: hoverTriggerAttrs(el),
    rect: hoverTriggerRect(el),
    cursor: style.cursor,
    pointerEvents: style.pointerEvents,
    hasGraphicDescendant: hasHoverGraphicDescendant(el),
  });
}

function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null {
  if (!(target instanceof Element)) return null;
  const element = resolveHoverElement(target);
  if (!element) return null;
  const desc = describeTarget(element);
  const decision = evaluateHoverTriggerElement(element, desc);
  if (!decision?.eligible) return null;
  const normalizedTarget = normalizeHoverTarget(element, desc, decision);
  return {
    element,
    target: normalizedTarget,
    recordedAt: Date.now(),
    score: decision.score,
    reasons: decision.reasons,
  };
}

function shouldReplacePendingHover(
  current: PendingHover,
  next: PendingHover,
  now: number,
): boolean {
  if (next.element === current.element) return false;
  if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true;
  if (current.element.contains(next.element)) return false;
  if (next.element.contains(current.element)) return true;
  return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN;
}

/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
  if (!session || !(target instanceof Element)) return false;
  if (session.element.contains(target)) return false;

  const option = target.closest('[role="option"]');
  if (option?.closest('[role="listbox"]')) return true;

  const suggestionRoot = target.closest(
    [
      '[id*="Sug"]',
      '[id*="sug"]',
      '[class*="suggest"]',
      '[class*="autocomplete"]',
      '[class*="typeahead"]',
      "[data-autocomplete]",
    ].join(", "),
  );
  if (suggestionRoot && !suggestionRoot.contains(session.element)) return true;

  return false;
}

function scheduleInputCompletionCommit(
  sessionElement: FillableElement,
  syncFillSessionValue: (el: FillableElement) => void,
  commitFillSession: () => void,
  hasSessionFor: (el: FillableElement) => boolean,
): void {
  const syncAndCommit = () => {
    if (!hasSessionFor(sessionElement)) return;
    syncFillSessionValue(sessionElement);
    commitFillSession();
  };
  syncFillSessionValue(sessionElement);
  queueMicrotask(syncAndCommit);
  setTimeout(syncAndCommit, 0);
}

export function startRecordCapture(
  _requestId: string,
  sendStep: (step: RecordStepPayload) => void,
): RecordCaptureController {
  const emitStep = (step: RecordStepPayload) => {
    sendStep({ page_url: location.href, ...step });
  };
  let fillSession: FillSession | null = null;
  let composing = false;
  let lastUrl = location.href;
  let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
  let pendingHover: PendingHover | null = null;
  let generatedControlClick: Element | null = null;
  let navigationActionPending = false;
  let navigationActionVersion = 0;
  const committedValues = new WeakMap<FillableElement, string>();

  const markNavigationAction = () => {
    navigationActionPending = true;
    const version = ++navigationActionVersion;
    setTimeout(() => {
      if (navigationActionVersion === version) {
        navigationActionPending = false;
      }
    }, 0);
  };

  const emitFill = (session: FillSession) => {
    if (session.lastValue === session.baselineValue) return;
    const isPassword =
      session.element instanceof HTMLInputElement && session.element.type === "password";
    const value = isPassword ? "***" : session.lastValue;
    emitStep({
      op: "fill",
      target: session.target,
      value,
      ...(isPassword ? { redacted: true } : {}),
    });
  };

  const commitFillSession = () => {
    if (!fillSession || composing) return;
    const session = fillSession;
    fillSession = null;
    emitFill(session);
    committedValues.set(session.element, session.lastValue);
  };

  const syncFillSessionValue = (el: FillableElement) => {
    if (!fillSession || fillSession.element !== el) return;
    fillSession.lastValue = fillableValue(el);
  };

  const ensureFillSession = (el: FillableElement) => {
    if (fillSession?.element === el) return;
    if (fillSession) commitFillSession();
    const currentValue = fillableValue(el);
    const baselineValue = committedValues.get(el) ?? currentValue;
    fillSession = {
      element: el,
      target: describeTarget(el),
      baselineValue,
      lastValue: currentValue,
    };
  };

  const emitNavigateIfChanged = (causedByAction?: boolean) => {
    if (location.href === lastUrl) return;
    commitFillSession();
    lastUrl = location.href;
    emitStep({
      op: "navigate",
      url: location.href,
      ...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}),
    });
  };

  const emitClick = (event: MouseEvent) => {
    // Only record clicks an LLM can re-identify (named interactive controls).
    const target = describeEventTarget(eventTarget(event));
    if (!target) return;
    markNavigationAction();
    emitStep({
      op: "click",
      target,
      expects_navigation: true,
    });
  };

  const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
    if (!pendingHover || !(actionTarget instanceof Element)) return;
    if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
      pendingHover = null;
      return;
    }
    const actionElement = resolveClickableElement(actionTarget);
    if (actionElement === pendingHover.element) {
      return;
    }
    const relationTarget = actionElement ?? actionTarget;
    if (!actionElement && pendingHover.element.contains(relationTarget)) {
      return;
    }
    const relation = decideHoverSurfaceRelation(
      {
        triggerElement: pendingHover.element,
      },
      relationTarget,
    );
    if (!relation.related) {
      pendingHover = null;
      return;
    }
    emitStep({
      op: "hover",
      target: pendingHover.target,
    });
    pendingHover = null;
  };

  const onMouseOver = (event: MouseEvent) => {
    const target = eventTarget(event);
    if (isOverlayTarget(target)) return;
    if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) {
      if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return;
      pendingHover = null;
    }
    const hover = hoverTargetFromEvent(target);
    if (!hover) return;
    const now = Date.now();
    if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) {
      return;
    }
    pendingHover = hover;
  };

  const onClick = (event: MouseEvent) => {
    if (event.button !== 0) return;
    const target = eventTarget(event);
    if (isOverlayTarget(target)) return;
    if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) {
      generatedControlClick = null;
      return;
    }
    if (
      event.detail === 0 &&
      keyboardActivation?.target === target &&
      Date.now() - keyboardActivation.recordedAt < 500
    ) {
      keyboardActivation = undefined;
      return;
    }

    const label = target instanceof Element ? target.closest("label") : null;
    const nestedInteractive =
      target instanceof Element ? target.closest("a,button,input,select,textarea") : null;
    if (label instanceof HTMLLabelElement && !nestedInteractive) {
      commitFillSession();
      generatedControlClick = label.control;
      emitClick(event);
      return;
    }

    const fillable = fillableFromTarget(target);
    if (fillable) {
      emitPendingHoverBeforeAction(fillable);
      ensureFillSession(fillable);
      return;
    }

    if (target instanceof Element) {
      const nearbyFillable = nearbyFillableFromSearchChrome(target);
      if (nearbyFillable) {
        emitPendingHoverBeforeAction(nearbyFillable);
        ensureFillSession(nearbyFillable);
        return;
      }
    }

    if (isInputCompletionClick(target, fillSession)) {
      markNavigationAction();
      const sessionElement = fillSession!.element;
      scheduleInputCompletionCommit(
        sessionElement,
        syncFillSessionValue,
        commitFillSession,
        (el) => fillSession?.element === el,
      );
      return;
    }

    commitFillSession();
    if (target instanceof Element && target.closest("select")) return;
    emitPendingHoverBeforeAction(target);
    emitClick(event);
  };

  const onFocusIn = (event: FocusEvent) => {
    const target = fillableFromTarget(eventTarget(event));
    if (target) ensureFillSession(target);
  };

  const onFocusOut = (event: FocusEvent) => {
    const target = fillableFromTarget(eventTarget(event));
    if (!target) return;
    if (!fillSession || fillSession.element !== target) return;
    syncFillSessionValue(target);
    commitFillSession();
  };

  const onInput = (event: Event) => {
    const target = fillableFromTarget(eventTarget(event));
    if (!target) return;
    if (composing) return;
    ensureFillSession(target);
    syncFillSessionValue(target);
  };

  const onCompositionStart = () => {
    composing = true;
  };

  const onCompositionEnd = (event: CompositionEvent) => {
    composing = false;
    const target = fillableFromTarget(eventTarget(event));
    if (target) {
      ensureFillSession(target);
      syncFillSessionValue(target);
    }
  };

  const onChange = (event: Event) => {
    commitFillSession();
    const target = eventTarget(event);
    if (target instanceof HTMLSelectElement) {
      emitPendingHoverBeforeAction(target);
      const values = Array.from(target.selectedOptions).map((opt) => opt.value);
      const labels = Array.from(target.selectedOptions).map((opt) =>
        (opt.label || opt.textContent || opt.value).trim(),
      );
      const desc = describeTarget(target);
      markNavigationAction();
      emitStep({
        op: "select",
        target: desc,
        values,
        labels,
        expects_navigation: true,
      });
    }
  };

  const onKeyDown = (event: KeyboardEvent) => {
    if (isOverlayTarget(eventTarget(event))) return;
    const target = eventTarget(event);
    const fillable = fillableFromTarget(target);
    if (fillable) {
      ensureFillSession(fillable);
      syncFillSessionValue(fillable);
    }
    const modifiers = [
      ...(event.altKey ? (["alt"] as const) : []),
      ...(event.ctrlKey ? (["ctrl"] as const) : []),
      ...(event.metaKey ? (["meta"] as const) : []),
      ...(event.shiftKey ? (["shift"] as const) : []),
    ];
    if (!shouldRecordPress(event.key, modifiers)) {
      return;
    }
    markNavigationAction();
    if (event.key === "Enter" || event.key === " ") {
      const submitTarget =
        event.key === "Enter" && fillable
          ? fillable
              .closest("form")
              ?.querySelector(
                'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]',
              )
          : null;
      keyboardActivation = {
        target: submitTarget ?? target,
        recordedAt: Date.now(),
      };
    }
    if (fillable) {
      commitFillSession();
    }
    const desc = describeEventTarget(target);
    if (!desc && !event.key) return;
    emitPendingHoverBeforeAction(target);
    emitStep({
      op: "press",
      key: event.key,
      ...(desc ? { target: desc } : {}),
      ...(modifiers.length ? { modifiers } : {}),
      expects_navigation: event.key === "Enter",
    });
  };

  document.addEventListener("click", onClick, true);
  document.addEventListener("mouseover", onMouseOver, true);
  document.addEventListener("focusin", onFocusIn, true);
  document.addEventListener("focusout", onFocusOut, true);
  document.addEventListener("input", onInput, true);
  document.addEventListener("compositionstart", onCompositionStart, true);
  document.addEventListener("compositionend", onCompositionEnd, true);
  document.addEventListener("change", onChange, true);
  document.addEventListener("keydown", onKeyDown, true);

  const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
  urlObserver.observe(document, { subtree: true, childList: true });
  const onUrlEvent = () => emitNavigateIfChanged();
  window.addEventListener("hashchange", onUrlEvent);
  window.addEventListener("popstate", onUrlEvent);
  window.addEventListener("pagehide", commitFillSession);

  const originalPushState = history.pushState;
  const originalReplaceState = history.replaceState;
  history.pushState = function (...args: Parameters<History["pushState"]>) {
    originalPushState.apply(this, args);
    emitNavigateIfChanged(navigationActionPending ? true : undefined);
  };
  history.replaceState = function (...args: Parameters<History["replaceState"]>) {
    originalReplaceState.apply(this, args);
    emitNavigateIfChanged(navigationActionPending ? true : undefined);
  };

  return {
    dispose() {
      commitFillSession();
      document.removeEventListener("click", onClick, true);
      document.removeEventListener("mouseover", onMouseOver, true);
      document.removeEventListener("focusin", onFocusIn, true);
      document.removeEventListener("focusout", onFocusOut, true);
      document.removeEventListener("input", onInput, true);
      document.removeEventListener("compositionstart", onCompositionStart, true);
      document.removeEventListener("compositionend", onCompositionEnd, true);
      document.removeEventListener("change", onChange, true);
      document.removeEventListener("keydown", onKeyDown, true);
      urlObserver.disconnect();
      window.removeEventListener("hashchange", onUrlEvent);
      window.removeEventListener("popstate", onUrlEvent);
      window.removeEventListener("pagehide", commitFillSession);
      history.pushState = originalPushState;
      history.replaceState = originalReplaceState;
    },
  };
}

export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;

export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
  return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}

export function handleRecordContentMessage(
  message: RecordContentMessage,
  state: {
    activeRequestId: string | null;
    capture: RecordCaptureController | null;
    setActiveRequestId(id: string | null): void;
    setCapture(capture: RecordCaptureController | null): void;
    onStart(requestId: string): void;
    onStop(): void;
  },
  sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
  if (isRecordStartMessage(message)) {
    if (!knownRecordRequests.has(message.requestId)) {
      knownRecordRequests.add(message.requestId);
      failedStepDeliveries.delete(message.requestId);
    }
    state.capture?.dispose();
    state.setCapture(
      startRecordCapture(message.requestId, (step) => {
        sendRecordStep(message.requestId, step);
      }),
    );
    state.setActiveRequestId(message.requestId);
    state.onStart(message.requestId);
    sendResponse?.({ ok: true });
    return sendResponse !== undefined;
  }

  if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
    // Require an active recording that matches this requestId — otherwise a
    // stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
    // clear overlay state even though nothing was capturing.
    if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
      return false;
    }
    state.capture?.dispose();
    state.setCapture(null);
    const finishStop = () => {
      state.onStop();
      state.setActiveRequestId(null);
    };
    if (isRecordStopMessage(message) && sendResponse) {
      const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
      void Promise.all(pending).then((delivered) => {
        finishStop();
        const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
        if (succeeded) {
          failedStepDeliveries.delete(message.requestId);
          knownRecordRequests.delete(message.requestId);
        }
        sendResponse(
          succeeded
            ? { ok: true }
            : {
                ok: false,
                error: "failed to deliver one or more recorded steps",
              },
        );
      });
      return true;
    }
    if (isRecordCancelMessage(message)) {
      failedStepDeliveries.delete(message.requestId);
      knownRecordRequests.delete(message.requestId);
    }
    finishStop();
    return false;
  }

  return false;
}

export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };