mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): emit pending hover before actions
This commit is contained in:
1 parent
73724617f9
commit
d685c75d25
4 files changed
+183
-19
No files matched your search
@@ -470,6 +470,92 @@ describe("record-capture semantic", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("records hover before filling a field inside the hover surface", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||||
|
<div class="user-menu dropdown-menu">
|
||||||
|
<label for="nickname">Nickname</label>
|
||||||
|
<input id="nickname" />
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: 900,
|
||||||
|
y: 8,
|
||||||
|
top: 8,
|
||||||
|
left: 900,
|
||||||
|
right: 932,
|
||||||
|
bottom: 40,
|
||||||
|
width: 32,
|
||||||
|
height: 32,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "Ada";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "fill",
|
||||||
|
target: { tag: "input" },
|
||||||
|
value: "Ada",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records hover before selecting inside the hover surface", () => {
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||||
|
<div class="user-menu dropdown-menu">
|
||||||
|
<select aria-label="Status">
|
||||||
|
<option value="online">Online</option>
|
||||||
|
<option value="away">Away</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step));
|
||||||
|
const trigger = document.querySelector("button")!;
|
||||||
|
const select = document.querySelector("select")!;
|
||||||
|
vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({
|
||||||
|
x: 900,
|
||||||
|
y: 8,
|
||||||
|
top: 8,
|
||||||
|
left: 900,
|
||||||
|
right: 932,
|
||||||
|
bottom: 40,
|
||||||
|
width: 32,
|
||||||
|
height: 32,
|
||||||
|
toJSON: () => ({}),
|
||||||
|
});
|
||||||
|
|
||||||
|
trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true }));
|
||||||
|
select.value = "away";
|
||||||
|
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["hover", "select"]);
|
||||||
|
expect(steps[0]).toMatchObject({
|
||||||
|
op: "hover",
|
||||||
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
});
|
||||||
|
expect(steps[1]).toMatchObject({
|
||||||
|
op: "select",
|
||||||
|
target: { role: "combobox", name: "Status" },
|
||||||
|
values: ["away"],
|
||||||
|
labels: ["Away"],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("records hover before clicking an unlabelled positioned floating surface", () => {
|
it("records hover before clicking an unlabelled positioned floating surface", () => {
|
||||||
document.body.innerHTML = `
|
document.body.innerHTML = `
|
||||||
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
<button type="button" aria-label="Open user navigation menu">avatar</button>
|
||||||
|
|||||||
@@ -388,18 +388,18 @@ export function startRecordCapture(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => {
|
const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => {
|
||||||
if (!pendingHover || !(clickTarget instanceof Element)) return;
|
if (!pendingHover || !(actionTarget instanceof Element)) return;
|
||||||
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
|
if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) {
|
||||||
pendingHover = null;
|
pendingHover = null;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const clickElement = resolveClickableElement(clickTarget);
|
const actionElement = resolveClickableElement(actionTarget);
|
||||||
if (clickElement === pendingHover.element) {
|
if (actionElement === pendingHover.element) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const relationTarget = clickElement ?? clickTarget;
|
const relationTarget = actionElement ?? actionTarget;
|
||||||
if (!clickElement && pendingHover.element.contains(relationTarget)) {
|
if (!actionElement && pendingHover.element.contains(relationTarget)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const relation = decideHoverSurfaceRelation(
|
const relation = decideHoverSurfaceRelation(
|
||||||
@@ -460,6 +460,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
const fillable = fillableFromTarget(target);
|
const fillable = fillableFromTarget(target);
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
|
emitPendingHoverBeforeAction(fillable);
|
||||||
ensureFillSession(fillable);
|
ensureFillSession(fillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -467,6 +468,7 @@ export function startRecordCapture(
|
|||||||
if (target instanceof Element) {
|
if (target instanceof Element) {
|
||||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||||
if (nearbyFillable) {
|
if (nearbyFillable) {
|
||||||
|
emitPendingHoverBeforeAction(nearbyFillable);
|
||||||
ensureFillSession(nearbyFillable);
|
ensureFillSession(nearbyFillable);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -486,7 +488,7 @@ export function startRecordCapture(
|
|||||||
|
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
if (target instanceof Element && target.closest("select")) return;
|
if (target instanceof Element && target.closest("select")) return;
|
||||||
emitPendingHoverBeforeClick(target);
|
emitPendingHoverBeforeAction(target);
|
||||||
emitClick(event);
|
emitClick(event);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -528,6 +530,7 @@ export function startRecordCapture(
|
|||||||
commitFillSession();
|
commitFillSession();
|
||||||
const target = eventTarget(event);
|
const target = eventTarget(event);
|
||||||
if (target instanceof HTMLSelectElement) {
|
if (target instanceof HTMLSelectElement) {
|
||||||
|
emitPendingHoverBeforeAction(target);
|
||||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||||
(opt.label || opt.textContent || opt.value).trim(),
|
(opt.label || opt.textContent || opt.value).trim(),
|
||||||
@@ -581,6 +584,7 @@ export function startRecordCapture(
|
|||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
|
emitPendingHoverBeforeAction(target);
|
||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
|
|||||||
@@ -456,6 +456,60 @@ describe("ToolDispatcher", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("limits default-target hover reassertion to the active tab", async () => {
|
||||||
|
vi.stubGlobal("chrome", {
|
||||||
|
tabs: {
|
||||||
|
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||||
|
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||||
|
sendMessage: vi.fn(async () => undefined),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { transport } = fakeTransport();
|
||||||
|
const sessions = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: vi.fn(async () => 4242),
|
||||||
|
remove: vi.fn(async () => {}),
|
||||||
|
ensureActiveTab: vi.fn(async () => {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await sessions.start("aa11");
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async <T>() => ({}) as T),
|
||||||
|
detachSession: vi.fn(async () => {}),
|
||||||
|
ensureNetworkCapture: vi.fn(async () => {}),
|
||||||
|
networkEntriesSince: vi.fn(() => ({
|
||||||
|
tab_id: 9,
|
||||||
|
entries: [],
|
||||||
|
next_since: 0,
|
||||||
|
truncated: false,
|
||||||
|
})),
|
||||||
|
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||||
|
setUserAgentOverride: vi.fn(async () => {}),
|
||||||
|
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||||
|
const helpers = dispatcher as unknown as {
|
||||||
|
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||||
|
withHoverReassert: <T>(
|
||||||
|
params: { session_id: string; tab_id?: number },
|
||||||
|
work: () => Promise<T>,
|
||||||
|
) => Promise<T>;
|
||||||
|
};
|
||||||
|
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||||
|
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||||
|
|
||||||
|
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||||
|
|
||||||
|
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||||
|
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||||
|
type: "mouseMoved",
|
||||||
|
x: 30,
|
||||||
|
y: 40,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("releases the current session hover latch before navigation-style work", async () => {
|
it("releases the current session hover latch before navigation-style work", async () => {
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
tabs: {
|
tabs: {
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ import {
|
|||||||
type SessionStartParams,
|
type SessionStartParams,
|
||||||
type SessionStopParams,
|
type SessionStopParams,
|
||||||
} from "./session";
|
} from "./session";
|
||||||
import { chromeTabsApi } from "./shared";
|
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||||
import {
|
import {
|
||||||
type BorrowConfirmationApprover,
|
type BorrowConfirmationApprover,
|
||||||
handleTabBorrow,
|
handleTabBorrow,
|
||||||
@@ -90,6 +90,11 @@ interface HoverLatch {
|
|||||||
y: number;
|
y: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HoverLatchScope {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DispatcherDeps {
|
export interface DispatcherDeps {
|
||||||
transport: Transport;
|
transport: Transport;
|
||||||
sessions: SessionManager;
|
sessions: SessionManager;
|
||||||
@@ -341,23 +346,24 @@ export class ToolDispatcher {
|
|||||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
case "tool.observe":
|
case "tool.observe": {
|
||||||
return this.withHoverReassert(req.params as ObserveParams, () =>
|
const params = req.params as ObserveParams;
|
||||||
|
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||||
|
return this.withHoverReassert(params, () =>
|
||||||
handleObserve(
|
handleObserve(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
req.params as ObserveParams,
|
params,
|
||||||
this.cdp
|
this.cdp
|
||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsCaptureApi,
|
tabsApi: chromeTabsCaptureApi,
|
||||||
conditionalSurfaceProbe: !this.hasHoverLatchForRequest(
|
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||||
req.params as ObserveParams,
|
|
||||||
),
|
|
||||||
hoverProbeBypassOverlay: bypassOverlay,
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
case "tool.get_html":
|
case "tool.get_html":
|
||||||
return handleGetHtml(
|
return handleGetHtml(
|
||||||
this.sessions,
|
this.sessions,
|
||||||
@@ -551,8 +557,8 @@ export class ToolDispatcher {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean {
|
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||||
return this.hoverLatchesForRequest(params).length > 0;
|
return this.hoverLatchesForRequest(scope).length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async withHoverReassert<T>(
|
private async withHoverReassert<T>(
|
||||||
@@ -560,12 +566,13 @@ export class ToolDispatcher {
|
|||||||
work: () => Promise<T>,
|
work: () => Promise<T>,
|
||||||
options: { releaseAfter?: boolean } = {},
|
options: { releaseAfter?: boolean } = {},
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
await this.reassertHover(params);
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
await this.reassertHover(scope);
|
||||||
try {
|
try {
|
||||||
return await work();
|
return await work();
|
||||||
} finally {
|
} finally {
|
||||||
if (options.releaseAfter) {
|
if (options.releaseAfter) {
|
||||||
await this.releaseHoverLatch(params.session_id, params.tab_id);
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -574,10 +581,23 @@ export class ToolDispatcher {
|
|||||||
params: { session_id: string; tab_id?: number },
|
params: { session_id: string; tab_id?: number },
|
||||||
work: () => Promise<T>,
|
work: () => Promise<T>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
await this.releaseHoverLatch(params.session_id, params.tab_id);
|
const scope = await this.resolveHoverLatchScope(params);
|
||||||
|
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||||
return work();
|
return work();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async resolveHoverLatchScope(params: {
|
||||||
|
session_id: string;
|
||||||
|
tab_id?: number;
|
||||||
|
}): Promise<HoverLatchScope> {
|
||||||
|
if (params.tab_id !== undefined) return params;
|
||||||
|
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||||
|
if (isRpcError(ctx)) return params;
|
||||||
|
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||||
|
if (isRpcError(target)) return params;
|
||||||
|
return { session_id: params.session_id, tab_id: target.tabId };
|
||||||
|
}
|
||||||
|
|
||||||
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||||
return [...this.hoverLatches.values()].filter((latch) => {
|
return [...this.hoverLatches.values()].filter((latch) => {
|
||||||
if (latch.sessionId !== params.session_id) return false;
|
if (latch.sessionId !== params.session_id) return false;
|
||||||
|
|||||||
Reference in new issue
Block a user