源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)

2026-09-01 19:05:14 +08:00
commit 7be535059e
3358 changed files with 1306357 additions and 0 deletions
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { randomUUID } from 'node:crypto';
+import type {
+  ComputerUseBoundAction,
+  ComputerUseFrameIdentity,
+  ComputerUseObservationIdentity,
+  ComputerUseRect,
+  ComputerUseWindowIdentity,
+  CuAction,
+  CuPoint,
+} from '@maka/core/computer-use';
+
+export type CuaFrameIdentity = ComputerUseFrameIdentity;
+export type CuaObservation = ComputerUseObservationIdentity;
+export type CuaBoundAction = ComputerUseBoundAction & {
+  fingerprint: string;
+};
+
+export interface CuaObservationSnapshot {
+  capturedAt: number;
+  screenshotWidthPx?: number;
+  screenshotHeightPx?: number;
+  displays: ComputerUseObservationIdentity['displays'];
+  target: ComputerUseWindowIdentity;
+}
+
+export type CuaActionRejectionReason =
+  | 'invalid_binding'
+  | 'no_active_frame'
+  | 'stale_epoch'
+  | 'stale_frame'
+  | 'duplicate_action'
+  // The same action again, where the first attempt was refused before anything
+  // was dispatched. Its own label because the recovery is the opposite one:
+  // `duplicate_action` means "it may already have taken effect, look", and this
+  // means "it did not, and the window is as it was". Both used to come back as
+  // `duplicate_action`, so the sentence that followed a retired action told the
+  // model to observe for a change that provably had not happened — directly
+  // contradicting the refusal it had just been given, which said the frame was
+  // still current and observing again was the round trip to skip.
+  | 'retired_action'
+  | 'action_not_claimed';
+
+export type CuaActionClaimResult = { ok: true } | { ok: false; reason: CuaActionRejectionReason };
+
+export type CuaActionConfirmationResult =
+  | { ok: true; epoch: number }
+  | { ok: false; reason: CuaActionRejectionReason };
+
+export class CuaFrameState {
+  private epoch = 0;
+  private currentFrame: CuaObservation | undefined;
+  private readonly claimedActions = new Set<string>();
+  private readonly consumedActions = new Set<string>();
+  /**
+   * The subset of `consumedActions` that never reached the window.
+   *
+   * Kept apart because the two produce opposite instructions, and the model was
+   * being given the wrong one half the time.
+   */
+  private readonly retiredActions = new Set<string>();
+
+  constructor(private readonly createFrameId: (epoch: number) => string = () => randomUUID()) {}
+
+  observe(snapshot: CuaObservationSnapshot): CuaObservation {
+    const frame = {
+      frameId: this.createFrameId(this.epoch),
+      epoch: this.epoch,
+      ...snapshot,
+    };
+    this.currentFrame = frame;
+    this.claimedActions.clear();
+    return frame;
+  }
+
+  activeObservation(): CuaObservation | undefined {
+    return this.currentFrame;
+  }
+
+  invalidate(): number {
+    this.epoch += 1;
+    this.currentFrame = undefined;
+    this.claimedActions.clear();
+    return this.epoch;
+  }
+
+  claimAction(action: CuaBoundAction): CuaActionClaimResult {
+    if (this.consumedActions.has(action.fingerprint)) {
+      return {
+        ok: false,
+        reason: this.retiredActions.has(action.fingerprint) ? 'retired_action' : 'duplicate_action',
+      };
+    }
+    const rejection = this.validateAction(action);
+    if (rejection) return { ok: false, reason: rejection };
+    if (this.claimedActions.has(action.fingerprint)) {
+      return { ok: false, reason: 'duplicate_action' };
+    }
+    this.claimedActions.add(action.fingerprint);
+    return { ok: true };
+  }
+
+  confirmAction(action: CuaBoundAction): CuaActionConfirmationResult {
+    const rejection = this.validateAction(action);
+    if (rejection) return { ok: false, reason: rejection };
+    if (!this.claimedActions.has(action.fingerprint)) {
+      return { ok: false, reason: 'action_not_claimed' };
+    }
+    this.consumedActions.add(action.fingerprint);
+    this.retiredActions.delete(action.fingerprint);
+    return { ok: true, epoch: this.invalidate() };
+  }
+
+  /**
+   * Record that an action was tried and leave the frame alive.
+   *
+   * For a refusal the executor never dispatched. `confirmAction` invalidates,
+   * because an action that ran may have changed the window the frame describes
+   * — but one that was refused before any dispatch changed nothing, and
+   * invalidating on its behalf costs the model an `observe` to get back a frame
+   * that was never stale. Measured on a real save-as-PDF run: three rounds of
+   * `click_element` → refused → `click_element` → `reobserve_required` →
+   * `observe`, 9 of 23 calls, before it found the route.
+   *
+   * The action is still retired rather than released, so sending the same one
+   * again is `duplicate_action` — which is the truth, and is a better answer
+   * than letting it be refused identically forever.
+   */
+  retireAction(action: CuaBoundAction): CuaActionConfirmationResult {
+    const rejection = this.validateAction(action);
+    if (rejection) return { ok: false, reason: rejection };
+    if (!this.claimedActions.has(action.fingerprint)) {
+      return { ok: false, reason: 'action_not_claimed' };
+    }
+    this.claimedActions.delete(action.fingerprint);
+    this.consumedActions.add(action.fingerprint);
+    this.retiredActions.add(action.fingerprint);
+    return { ok: true, epoch: this.epoch };
+  }
+
+  isConsumed(frame: CuaFrameIdentity, actionFingerprint: string): boolean {
+    return this.consumedActions.has(
+      bindCuaAction(frame, actionFingerprint, this.requireTarget(frame)).fingerprint,
+    );
+  }
+
+  /**
+   * Whether a consumed action was retired rather than dispatched.
+   *
+   * The companion to `isConsumed`, for the pre-check that refuses a repeat
+   * before it is bound: "already sent" and "already refused without being sent"
+   * are the same lookup and opposite instructions.
+   */
+  wasRetired(frame: CuaFrameIdentity, actionFingerprint: string): boolean {
+    return this.retiredActions.has(
+      bindCuaAction(frame, actionFingerprint, this.requireTarget(frame)).fingerprint,
+    );
+  }
+
+  private requireTarget(frame: CuaFrameIdentity): ComputerUseWindowIdentity {
+    if (
+      this.currentFrame &&
+      this.currentFrame.frameId === frame.frameId &&
+      this.currentFrame.epoch === frame.epoch
+    ) {
+      return this.currentFrame.target;
+    }
+    return { pid: -1, windowId: -1 };
+  }
+
+  private validateAction(action: CuaBoundAction): CuaActionRejectionReason | undefined {
+    if (fingerprintBoundAction(action) !== action.fingerprint) {
+      return 'invalid_binding';
+    }
+    if (!this.currentFrame) return 'no_active_frame';
+    if (action.epoch !== this.epoch) return 'stale_epoch';
+    if (action.frameId !== this.currentFrame.frameId) return 'stale_frame';
+    return undefined;
+  }
+}
+
+export function bindCuaAction(
+  frame: CuaFrameIdentity,
+  actionFingerprint: string,
+  target: ComputerUseWindowIdentity,
+  binding: Omit<
+    ComputerUseBoundAction,
+    keyof CuaFrameIdentity | 'actionFingerprint' | 'target'
+  > = {},
+): CuaBoundAction {
+  const action: CuaBoundAction = {
+    ...frame,
+    actionFingerprint,
+    target,
+    ...binding,
+    fingerprint: '',
+  };
+  return { ...action, fingerprint: fingerprintBoundAction(action) };
+}
+
+export function fingerprintCuaAction(action: CuAction): string {
+  return JSON.stringify(action);
+}
+
+export function fingerprintCuaSemanticAction(
+  type: string,
+  elementId?: string,
+  value?: string,
+): string {
+  return JSON.stringify([type, elementId, value]);
+}
+
+/**
+ * Where the cursor should be shown for an element action, from the element's
+ * own observed frame.
+ *
+ * Element frames arrive in the same screen coordinates as the window's bounds
+ * — `validateSemanticElementVisibility` compares one against the other
+ * directly — so the centre needs no transform, only the containment check that
+ * validator applies. Outside the window the frame is stale or the element has
+ * moved, and no point at all is better than sending the cursor somewhere the
+ * action will be refused from anyway.
+ */
+function elementPresentationPoint(
+  observation: CuaObservation,
+  frame: ComputerUseRect | undefined,
+): CuPoint | undefined {
+  const bounds = observation.target.bounds;
+  if (!frame || !bounds) return undefined;
+  if (frame.width <= 0 || frame.height <= 0) return undefined;
+  const point = { x: frame.x + frame.width / 2, y: frame.y + frame.height / 2 };
+  if (point.x < bounds.x || point.x >= bounds.x + bounds.width) return undefined;
+  if (point.y < bounds.y || point.y >= bounds.y + bounds.height) return undefined;
+  return point;
+}
+
+export function bindCuaSemanticActionToObservation(
+  observation: CuaObservation,
+  input: {
+    type: string;
+    elementId?: string;
+    value?: string;
+    /** The observed frame of `elementId`, when the observation reported one. */
+    elementFrame?: ComputerUseRect;
+  },
+): CuaBoundAction {
+  const presentationScreenPoint = elementPresentationPoint(observation, input.elementFrame);
+  return bindCuaAction(
+    observation,
+    fingerprintCuaSemanticAction(input.type, input.elementId, input.value),
+    observation.target,
+    {
+      ...(input.elementId ? { elementId: input.elementId } : {}),
+      ...(presentationScreenPoint ? { presentationScreenPoint } : {}),
+    },
+  );
+}
+
+export function bindCuaActionToObservation(
+  observation: CuaObservation,
+  action: CuAction,
+): CuaBoundAction | undefined {
+  const base = bindCuaAction(observation, fingerprintCuaAction(action), observation.target);
+  if (action.type === 'zoom') {
+    const start = bindWindowPoint(observation, {
+      x: Math.min(action.region.x1, action.region.x2),
+      y: Math.min(action.region.y1, action.region.y2),
+    });
+    const end = bindWindowPoint(observation, {
+      x: Math.max(action.region.x1, action.region.x2),
+      y: Math.max(action.region.y1, action.region.y2),
+    });
+    if (!start || !end) return undefined;
+    return {
+      ...finalizeBoundAction({
+        ...base,
+        sourceStartCoordinate: start,
+        sourceCoordinate: end,
+        windowStartCoordinate: start,
+        windowCoordinate: end,
+        coordinateSpace: 'window-screenshot-local',
+      }),
+    };
+  }
+  if ('coordinate' in action) {
+    const end = bindWindowPoint(observation, action.coordinate);
+    if (!end) return undefined;
+    if (action.type === 'left_click_drag') {
+      const start = bindWindowPoint(observation, action.startCoordinate);
+      if (!start) return undefined;
+      return finalizeBoundAction({
+        ...base,
+        sourceStartCoordinate: start,
+        sourceCoordinate: end,
+        windowStartCoordinate: start,
+        windowCoordinate: end,
+        coordinateSpace: 'window-screenshot-local',
+      });
+    }
+    return finalizeBoundAction({
+      ...base,
+      sourceCoordinate: end,
+      windowCoordinate: end,
+      coordinateSpace: 'window-screenshot-local',
+    });
+  }
+  return base;
+}
+
+function bindWindowPoint(observation: CuaObservation, point: CuPoint): CuPoint | undefined {
+  const width = observation.screenshotWidthPx ?? observation.target.sourceBoundsPx?.width ?? 0;
+  const height = observation.screenshotHeightPx ?? observation.target.sourceBoundsPx?.height ?? 0;
+  return width > 0 &&
+    height > 0 &&
+    point.x >= 0 &&
+    point.y >= 0 &&
+    point.x < width &&
+    point.y < height
+    ? point
+    : undefined;
+}
+
+function finalizeBoundAction(
+  action: Omit<CuaBoundAction, 'fingerprint'> & { fingerprint?: string },
+): CuaBoundAction {
+  const withPlaceholder = { ...action, fingerprint: '' };
+  return {
+    ...withPlaceholder,
+    fingerprint: fingerprintBoundAction(withPlaceholder),
+  };
+}
+
+function fingerprintBoundAction(
+  action: Omit<CuaBoundAction, 'fingerprint'> | CuaBoundAction,
+): string {
+  return JSON.stringify([
+    action.frameId,
+    action.epoch,
+    action.actionFingerprint,
+    action.target.pid,
+    action.target.windowId,
+    action.elementId ?? null,
+    action.sourceStartCoordinate ?? null,
+    action.sourceCoordinate ?? null,
+  ]);
+}