mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
63338cd98e | ||
|
|
bd193ceb5d | ||
|
|
5ed828736f | ||
|
|
ee620acf64 | ||
|
|
05a36075b8 | ||
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
641d934337 | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
506adbe708 | ||
|
|
984d017b0f | ||
|
|
5df93143c5 | ||
|
|
7163f2f9b0 | ||
|
|
e9f0584af3 | ||
|
|
2997f495db | ||
|
|
f96bd72c3c | ||
|
|
329c68054b | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
60fbf30e24 |
@@ -325,3 +325,180 @@ jobs:
|
||||
|
||||
echo "Chrome Web Store submission created successfully:"
|
||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||
|
||||
publish-edge-addons:
|
||||
needs: [resolve, build-extension]
|
||||
runs-on: ubuntu-latest
|
||||
environment: edge-addons
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: extension-zip
|
||||
path: dist
|
||||
|
||||
- name: Validate Microsoft Edge Add-ons configuration
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$CLIENT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$PRODUCT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||
if [ -z "$ARCHIVE" ]; then
|
||||
echo "::error::No extension zip found in the downloaded artifact"
|
||||
exit 1
|
||||
fi
|
||||
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||
|
||||
operation_id_from_headers() {
|
||||
local headers="$1"
|
||||
local location
|
||||
location="$(awk '
|
||||
tolower($1) == "location:" {
|
||||
sub(/\r$/, "")
|
||||
sub(/^[^:]*:[[:space:]]*/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' "$headers")"
|
||||
location="${location%%\?*}"
|
||||
printf '%s' "${location##*/}"
|
||||
}
|
||||
|
||||
poll_operation() {
|
||||
local operation_name="$1"
|
||||
local status_url="$2"
|
||||
local status="InProgress"
|
||||
local http_status
|
||||
|
||||
for attempt in $(seq 1 30); do
|
||||
sleep 5
|
||||
http_status="$(curl --silent --show-error \
|
||||
--output "$STATUS_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
"$status_url")"
|
||||
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||
echo "$operation_name status check $attempt/30: $status"
|
||||
case "$status" in
|
||||
Succeeded)
|
||||
return 0
|
||||
;;
|
||||
InProgress)
|
||||
;;
|
||||
Failed)
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$UPLOAD_RESPONSE" \
|
||||
--dump-header "$UPLOAD_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/zip" \
|
||||
--data-binary "@$ARCHIVE" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||
cat "$UPLOAD_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"package upload" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$PUBLISH_RESPONSE" \
|
||||
--dump-header "$PUBLISH_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||
cat "$PUBLISH_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"submission" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||
|
||||
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||
|
||||
@@ -7,6 +7,10 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
@@ -247,6 +247,41 @@ describe("App", () => {
|
||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
state: "version_skew",
|
||||
instanceId: "03c3e47f",
|
||||
handshake: {
|
||||
server: "bh",
|
||||
version: mockDaemonVersion,
|
||||
protocol_version: "1.0",
|
||||
},
|
||||
},
|
||||
statusState: "version_skew",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("已连接")).toBeTruthy();
|
||||
expect(screen.getByText("可升级")).toBeTruthy();
|
||||
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||
expect(screen.queryByText("Action needed")).toBeNull();
|
||||
expect(screen.queryByText("兼容")).toBeNull();
|
||||
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||
expect(warning.textContent).toContain("CLI 协议");
|
||||
expect(warning.textContent).toContain("扩展协议");
|
||||
|
||||
openRecordView();
|
||||
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||
fireEvent.click(copyButton);
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("control hints toggle", () => {
|
||||
|
||||
@@ -79,6 +79,7 @@ export function App() {
|
||||
}, [copiedTick]);
|
||||
|
||||
const isSkewed = statusState === "version_skew";
|
||||
const connectionLive = statusState === "connected" || isSkewed;
|
||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
const extensionVersion = snapshot.extensionVersion || "—";
|
||||
@@ -90,7 +91,7 @@ export function App() {
|
||||
setCopiedInstanceId(true);
|
||||
};
|
||||
|
||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
||||
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||
const recordPurpose = purposeDraft.trim();
|
||||
const recordStartUrl = startUrlDraft.trim();
|
||||
const recordCommand = snapshot.instanceId
|
||||
@@ -206,12 +207,12 @@ export function App() {
|
||||
</div>
|
||||
{isSkewed && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
data-slot="popup-version-skew-warning"
|
||||
>
|
||||
{t("popup.versionSkewWarning", {
|
||||
extensionProtocol: PROTOCOL_VERSION,
|
||||
daemonProtocol,
|
||||
cliProtocol: daemonProtocol,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
|
||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||
// AX captures would leave the two halves of one observation describing the
|
||||
// page on either side of that change.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const methodOrder: string[] = [];
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
methodOrder.push(method);
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
// Report the button as a `:hover` rule target so a real hover fires.
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||
}
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", probe_hover: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||
expect(firstHover).toBeGreaterThan(-1);
|
||||
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||
});
|
||||
|
||||
it("runs conditional surface probing only when observe opts in", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const withoutProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if ("code" in withoutProbe) {
|
||||
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||
}
|
||||
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
const withProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||
deps,
|
||||
);
|
||||
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -383,7 +383,11 @@ export class ToolDispatcher {
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
// Active hover probing is opt-in. A held hover latch still
|
||||
// suppresses it, because probing would move the cursor off
|
||||
// the element the caller is deliberately holding.
|
||||
conditionalSurfaceProbe:
|
||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
|
||||
@@ -49,11 +49,14 @@ import {
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
|
||||
export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
supplementalNames: options.supplementalNames,
|
||||
});
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured);
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
|
||||
}
|
||||
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
|
||||
return {
|
||||
...scene,
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
@@ -1010,17 +1018,69 @@ async function captureForVom(
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
/**
|
||||
* Whether hovering surfaced content absent from the static snapshot. Cached
|
||||
* tooltip names count, so this may over-report on repeat observations — it
|
||||
* errs towards telling the caller the snapshot is less fresh than it looks.
|
||||
*/
|
||||
revealedContent: boolean;
|
||||
surfaceProbes: CapturedSurfaceProbe[];
|
||||
tooltipNames: Map<string, string>;
|
||||
}
|
||||
|
||||
const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
performed: false,
|
||||
revealedContent: false,
|
||||
surfaceProbes: [],
|
||||
tooltipNames: new Map(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs both active-hover chains back to back.
|
||||
*
|
||||
* Ordering matters: this happens after DOM *and* accessibility capture so a
|
||||
* hover that opens a menu cannot leave one half of the observation describing
|
||||
* the page before the change and the other half after it. Sharing one overlay
|
||||
* bypass span also means the agent overlay toggles once per observation
|
||||
* instead of once per chain.
|
||||
*/
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<HoverProbeOutcome> {
|
||||
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
|
||||
|
||||
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
|
||||
signal: options.signal,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return {
|
||||
performed: true,
|
||||
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
|
||||
surfaceProbes,
|
||||
tooltipNames,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
});
|
||||
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
||||
const tooltipNames = options.conditionalSurfaceProbe
|
||||
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
|
||||
signal: options.signal,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
})
|
||||
: new Map<string, string>();
|
||||
const hoverProbes = await runHoverProbes(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
normalizedDocuments,
|
||||
staticSemantics,
|
||||
options,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene = projectSemanticGraph(
|
||||
normalizeSemanticStructure(
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(
|
||||
scene,
|
||||
normalizedDocuments,
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
frameDocuments: normalizedDocuments,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
surfaceProbes: hoverProbes.surfaceProbes,
|
||||
hoverProbe: {
|
||||
performed: hoverProbes.performed,
|
||||
revealedContent: hoverProbes.revealedContent,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && observation.hoverProbe?.performed
|
||||
? {
|
||||
hover_probe: {
|
||||
performed: true,
|
||||
revealed_content: observation.hoverProbe.revealedContent,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
@@ -1179,5 +1258,13 @@ export async function handleObserve(
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
return handleVomObservation(
|
||||
manager,
|
||||
params,
|
||||
"observe",
|
||||
"transient_input",
|
||||
params.probe_hover === true,
|
||||
deps,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
|
||||
|
||||
afterEach(() => {
|
||||
resetOverlayBypassDepth();
|
||||
});
|
||||
|
||||
describe("ProbeBudget", () => {
|
||||
it("refuses a candidate that cannot finish inside the budget", () => {
|
||||
const budget = new ProbeBudget(500);
|
||||
|
||||
expect(budget.canAfford(400)).toBe(true);
|
||||
expect(budget.canAfford(600)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
|
||||
const budget = new ProbeBudget(60);
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
|
||||
// A top-of-loop `elapsed > total` check would still admit this candidate
|
||||
// and then overshoot by its full cost.
|
||||
expect(budget.canAfford(50)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withOverlayBypass", () => {
|
||||
it("toggles the overlay once around nested probe phases", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("restores the overlay when the probe throws", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
withOverlayBypass(bypass, 4, async () => {
|
||||
throw new Error("probe exploded");
|
||||
}),
|
||||
).rejects.toThrow("probe exploded");
|
||||
|
||||
expect(bypass).toHaveBeenLastCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("reports a failed restore instead of swallowing it", async () => {
|
||||
const onRestoreFailure = vi.fn();
|
||||
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
|
||||
|
||||
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not pin the overlay off for the tab when a restore fails", async () => {
|
||||
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
|
||||
|
||||
const bypass = vi.fn(async () => {});
|
||||
await withOverlayBypass(bypass, 4, async () => undefined);
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("runs the probe untouched when no bypass is wired up", async () => {
|
||||
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [
|
||||
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
|
||||
];
|
||||
const graph = semantics(documents);
|
||||
|
||||
const first = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(first.size).toBe(0);
|
||||
const hoversAfterFirst = vi
|
||||
.mocked(cdp.send)
|
||||
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
|
||||
expect(hoversAfterFirst).toBeGreaterThan(0);
|
||||
|
||||
const second = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(second.size).toBe(0);
|
||||
expect(
|
||||
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
|
||||
.length,
|
||||
).toBe(hoversAfterFirst);
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
/**
|
||||
* Ceiling for the whole hover-surface phase.
|
||||
*
|
||||
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
|
||||
* so a candidate could start with 1ms left and still run its two settle
|
||||
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
|
||||
* enforced by an up-front affordability check, so the same number of candidates
|
||||
* get probed under a bound that no longer lies.
|
||||
*/
|
||||
const MAX_HOVER_PROBE_MS = 2_600;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
@@ -614,13 +620,35 @@ function confidenceForHover(
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
/**
|
||||
* One candidate costs two settle windows (baseline + post-hover) plus a few
|
||||
* CDP round trips. Used to decide whether the next candidate still fits the
|
||||
* budget before paying for it.
|
||||
*/
|
||||
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
|
||||
|
||||
export interface HoverSurfaceProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hovers a bounded set of likely menu triggers and reports the sub-items each
|
||||
* one reveals.
|
||||
*
|
||||
* Runs after DOM *and* accessibility capture. Hovering can open menus and
|
||||
* change layout, so probing between the two captures would leave the DOM half
|
||||
* of an observation describing the page before the change and the AX half
|
||||
* describing it after.
|
||||
*
|
||||
* The caller owns the overlay bypass span (see `withOverlayBypass`).
|
||||
*/
|
||||
export async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
options: HoverSurfaceProbeOptions = {},
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: [],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wall-clock budget for a probe phase.
|
||||
*
|
||||
* Callers ask whether the *next* candidate still fits before starting it,
|
||||
* rather than checking elapsed time at the top of the loop. A top-of-loop
|
||||
* check lets a candidate start with 1ms of budget left and then run to
|
||||
* completion, overshooting by a full settle window.
|
||||
*/
|
||||
export class ProbeBudget {
|
||||
private readonly startedAt = Date.now();
|
||||
|
||||
constructor(private readonly totalMs: number) {}
|
||||
|
||||
get elapsedMs(): number {
|
||||
return Date.now() - this.startedAt;
|
||||
}
|
||||
|
||||
/** True when a candidate costing `estimatedMs` still fits in the budget. */
|
||||
canAfford(estimatedMs: number): boolean {
|
||||
return this.elapsedMs + estimatedMs <= this.totalMs;
|
||||
}
|
||||
}
|
||||
|
||||
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
|
||||
|
||||
export interface OverlayBypassScopeOptions {
|
||||
/**
|
||||
* Called when the overlay could not be restored. Defaults to `console.error`
|
||||
* so a stuck-bypassed overlay is never silent; tests inject a spy.
|
||||
*/
|
||||
onRestoreFailure?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
const overlayBypassDepth = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
|
||||
*
|
||||
* Reference counted per tab so nested probe phases share a single bypass span
|
||||
* instead of toggling the overlay once per phase. Restore failures are
|
||||
* reported rather than swallowed, and the depth is released even when restore
|
||||
* throws so a failure cannot pin the overlay off for the tab's lifetime.
|
||||
*/
|
||||
export async function withOverlayBypass<T>(
|
||||
bypass: OverlayBypass | undefined,
|
||||
tabId: number,
|
||||
body: () => Promise<T>,
|
||||
options: OverlayBypassScopeOptions = {},
|
||||
): Promise<T> {
|
||||
if (!bypass) return body();
|
||||
|
||||
const depth = overlayBypassDepth.get(tabId) ?? 0;
|
||||
overlayBypassDepth.set(tabId, depth + 1);
|
||||
if (depth === 0) {
|
||||
try {
|
||||
await bypass(tabId, true);
|
||||
} catch (error) {
|
||||
// Failing to hide the overlay only risks probing our own UI, so continue.
|
||||
console.debug("[bsk hover] overlay bypass could not be enabled", error);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await body();
|
||||
} finally {
|
||||
const current = overlayBypassDepth.get(tabId) ?? 1;
|
||||
if (current <= 1) {
|
||||
overlayBypassDepth.delete(tabId);
|
||||
try {
|
||||
await bypass(tabId, false);
|
||||
} catch (error) {
|
||||
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
|
||||
}
|
||||
} else {
|
||||
overlayBypassDepth.set(tabId, current - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRestoreFailure(error: unknown): void {
|
||||
console.error("[bsk hover] agent overlay could not be restored after probing", error);
|
||||
}
|
||||
|
||||
/** Test seam: drops any bypass depth left behind by a crashed probe. */
|
||||
export function resetOverlayBypassDepth(): void {
|
||||
overlayBypassDepth.clear();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||
|
||||
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
|
||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||
const TOOLTIP_SETTLE_MS = 250;
|
||||
const CACHE_TTL_MS = 5 * 60_000;
|
||||
/**
|
||||
* Misses expire sooner than hits: a control that gains a tooltip once the app
|
||||
* finishes booting should still be discoverable, but not at the cost of
|
||||
* re-probing every unlabelled control on every observation.
|
||||
*/
|
||||
const MISS_CACHE_TTL_MS = 60_000;
|
||||
const MAX_CACHE_ENTRIES = 500;
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||
|
||||
interface TooltipCacheEntry {
|
||||
name: string;
|
||||
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
|
||||
name: string | null;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
||||
|
||||
export interface TooltipNameProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
|
||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
function cachedName<T extends FrameOwnedAxNode>(
|
||||
/** Returns the cache entry, or `undefined` when this control was never probed. */
|
||||
function cachedEntry<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
): string | undefined {
|
||||
): TooltipCacheEntry | undefined {
|
||||
const key = cacheKey(document, backendNodeId);
|
||||
const entry = tooltipCache.get(key);
|
||||
if (!entry) return undefined;
|
||||
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
|
||||
tooltipCache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return entry.name;
|
||||
return entry;
|
||||
}
|
||||
|
||||
function storeName<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
name: string,
|
||||
name: string | null,
|
||||
): void {
|
||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||
const oldest = tooltipCache.keys().next().value;
|
||||
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||
name,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
const names = new Map<string, string>();
|
||||
const pending: TooltipCandidate<T>[] = [];
|
||||
for (const candidate of candidates(documents, graph)) {
|
||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
||||
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
|
||||
if (cached) {
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
||||
// A cached miss is still a decision: skip the control instead of re-probing.
|
||||
if (cached.name !== null) {
|
||||
names.set(
|
||||
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
|
||||
cached.name,
|
||||
);
|
||||
}
|
||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
if (pending.length === 0) return names;
|
||||
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
|
||||
const contexts = new Map<string, number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of pending) {
|
||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
||||
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
let contextId = contexts.get(candidate.document.frameId);
|
||||
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||
const name = cleanTooltipDiff(before, after);
|
||||
storeName(candidate.document, candidate.backendNodeId, name ?? null);
|
||||
if (!name) continue;
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||
storeName(candidate.document, candidate.backendNodeId, name);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
} finally {
|
||||
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/**
|
||||
* Whether this observation actively hovered the page, and whether that
|
||||
* revealed content the static tree does not contain. Lets callers judge how
|
||||
* far the page may have drifted from the returned snapshot.
|
||||
*/
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}
|
||||
|
||||
export interface HoverProbeReport {
|
||||
performed: boolean;
|
||||
revealedContent: boolean;
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -318,9 +318,14 @@ export interface SnapshotResult {
|
||||
|
||||
export interface ObserveParams extends SnapshotParams {
|
||||
debug_surfaces?: boolean;
|
||||
probe_hover?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult extends SnapshotResult {
|
||||
hover_probe?: {
|
||||
performed: boolean;
|
||||
revealed_content: boolean;
|
||||
};
|
||||
debug?: {
|
||||
surface_probes?: Array<{
|
||||
trigger_backend_node_id: number;
|
||||
|
||||
+106
-282
@@ -1,327 +1,151 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## Recover without wandering
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// `bsk doctor` check: every connected browser should use the same
|
||||
/// protocol version as the daemon. Minor protocol drift is accepted by
|
||||
/// the daemon but flagged here so the user can see who needs updating
|
||||
/// (M10.4).
|
||||
/// `bsk doctor` check: connected browsers should speak a protocol the
|
||||
/// daemon accepts. A different protocol string is still a live
|
||||
/// connection — report Ok so agents keep working. The detail is an
|
||||
/// upgrade reminder, not a blocker.
|
||||
///
|
||||
/// Review M2 (round-1 minor): when no browsers are connected, the
|
||||
/// check has nothing to compare against, so it now reports
|
||||
@@ -396,13 +396,12 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
CheckResult::fail(
|
||||
CheckResult::ok(
|
||||
name,
|
||||
format!(
|
||||
"{} browser(s) have protocol minor drift from the daemon: {stale}",
|
||||
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
|
||||
status.version_skew_browsers.len()
|
||||
),
|
||||
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -581,7 +580,7 @@ mod m2_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browsers_check_reports_fail_when_skew_present() {
|
||||
fn browsers_check_reports_ok_when_skew_present() {
|
||||
let status = fake_status(
|
||||
vec![BrowserStatusEntry {
|
||||
instance_id: "alpha".into(),
|
||||
@@ -605,9 +604,17 @@ mod m2_tests {
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
assert_eq!(check.status, CheckStatus::Fail);
|
||||
assert_eq!(check.status, CheckStatus::Ok);
|
||||
assert!(check.detail.contains("alpha"));
|
||||
assert!(check.hint.is_some());
|
||||
assert!(
|
||||
check
|
||||
.detail
|
||||
.contains("still usable — continue, and upgrade soon")
|
||||
);
|
||||
assert!(
|
||||
check.hint.is_none(),
|
||||
"a protocol-version note must not hint to stop"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -635,7 +642,7 @@ mod m2_tests {
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
assert_eq!(check.status, CheckStatus::Fail);
|
||||
assert_eq!(check.status, CheckStatus::Ok);
|
||||
assert!(
|
||||
check
|
||||
.detail
|
||||
|
||||
@@ -33,6 +33,12 @@ pub struct ObserveArgs {
|
||||
/// Include conditional surface probe diagnostics in JSON output.
|
||||
#[arg(long = "debug-surfaces")]
|
||||
pub debug_surfaces: bool,
|
||||
|
||||
/// Actively hover page controls to reveal hover-only menus and tooltips.
|
||||
/// Costs seconds of wall clock and touches the live page, so it is off
|
||||
/// unless a static observation looks like it is missing hover content.
|
||||
#[arg(long = "probe-hover")]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
debug_surfaces: args.debug_surfaces,
|
||||
probe_hover: args.probe_hover,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
|
||||
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
|
||||
let bold = "\x1b[1m";
|
||||
let reset = "\x1b[0m";
|
||||
eprintln!(
|
||||
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
|
||||
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
|
||||
s.version_skew_browsers.len()
|
||||
);
|
||||
for skew in &s.version_skew_browsers {
|
||||
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
|
||||
skew.label.clone()
|
||||
};
|
||||
eprintln!(
|
||||
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
|
||||
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
|
||||
skew.instance_id,
|
||||
label,
|
||||
display_protocol(&skew.client_protocol_version),
|
||||
|
||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
tab_id: 13,
|
||||
truncated: false,
|
||||
dialogs: vec![],
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -494,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
max_depth: None,
|
||||
max_tokens: None,
|
||||
debug_surfaces: false,
|
||||
probe_hover: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ObserveParams",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
@@ -30,6 +30,11 @@
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"probe_hover": {
|
||||
"description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -24,6 +24,17 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"hover_probe": {
|
||||
"description": "Present only when `probe_hover` was requested and probing actually ran.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HoverProbeReport"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ref_count": {
|
||||
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
||||
"type": "integer",
|
||||
@@ -46,6 +57,24 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HoverProbeReport": {
|
||||
"description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"performed",
|
||||
"revealed_content"
|
||||
],
|
||||
"properties": {
|
||||
"performed": {
|
||||
"description": "The real cursor was moved over page content during this observation.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"revealed_content": {
|
||||
"description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.",
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
|
||||
@@ -58,8 +58,9 @@ pub struct SnapshotResult {
|
||||
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
||||
///
|
||||
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
||||
/// such as hover-surface discovery. It must not submit input, click,
|
||||
/// navigate, or otherwise commit page state.
|
||||
/// such as hover-surface discovery, but only when `probe_hover` asks for
|
||||
/// them. It must not submit input, click, navigate, or otherwise commit
|
||||
/// page state.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveParams {
|
||||
pub session_id: String,
|
||||
@@ -77,6 +78,16 @@ pub struct ObserveParams {
|
||||
/// Include implementation diagnostics for conditional surface probes.
|
||||
#[serde(default)]
|
||||
pub debug_surfaces: bool,
|
||||
/// Opt in to active hover probing: the extension moves the real cursor
|
||||
/// over a bounded set of controls to discover hover-only menus and
|
||||
/// tooltips.
|
||||
///
|
||||
/// Off by default. Probing costs seconds of wall clock, mutates the live
|
||||
/// page, and pays off only on pages whose content is genuinely hidden
|
||||
/// behind hover. Request it when a static observation looks like it is
|
||||
/// missing hover-revealed structure.
|
||||
#[serde(default)]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -102,6 +113,17 @@ pub struct ObserveDebug {
|
||||
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||
}
|
||||
|
||||
/// Reports what active hover probing did during an observation, so the agent
|
||||
/// can judge how far the live page may have drifted from the returned text.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverProbeReport {
|
||||
/// The real cursor was moved over page content during this observation.
|
||||
pub performed: bool,
|
||||
/// Hovering surfaced content the static tree does not contain, so the page
|
||||
/// reacts to the cursor and may not have fully reverted.
|
||||
pub revealed_content: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveResult {
|
||||
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
||||
@@ -118,6 +140,9 @@ pub struct ObserveResult {
|
||||
pub truncated: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
/// Present only when `probe_hover` was requested and probing actually ran.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub hover_probe: Option<HoverProbeReport>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub debug: Option<ObserveDebug>,
|
||||
}
|
||||
@@ -308,6 +333,7 @@ mod tests {
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
@@ -315,4 +341,30 @@ mod tests {
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_params_default_to_no_hover_probing() {
|
||||
let params: ObserveParams =
|
||||
serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap();
|
||||
assert!(!params.probe_hover);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_result_round_trips_with_hover_probe_report() {
|
||||
let r = ObserveResult {
|
||||
text: "@vom 1\n".into(),
|
||||
ref_count: 0,
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: Some(HoverProbeReport {
|
||||
performed: true,
|
||||
revealed_content: true,
|
||||
}),
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,41 +5,20 @@ npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wx
|
||||
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
|
||||
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
|
||||
|
||||
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
|
||||
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
|
||||
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
|
||||
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | bsk command | Purpose |
|
||||
| Tool | Actions | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `browser_session_start` | `bsk session start` | Open an Agent Window session; optional initial URL, window size, and device emulation preset. Returns the session id and makes it the current session. |
|
||||
| `browser_session_stop` | `bsk session stop` | Stop a session (the current one by default) and close its Agent Window. Only plugin-created sessions can be stopped. |
|
||||
| `browser_session_list` | — (registry only) | List the sessions this plugin created, marking the current one. Foreign daemon sessions are never shown. |
|
||||
| `browser_navigate` | `bsk navigate` | Navigate the active tab, with `waitUntil` / timeout control. |
|
||||
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
|
||||
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
|
||||
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
|
||||
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
|
||||
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
|
||||
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
|
||||
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
|
||||
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
|
||||
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
|
||||
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
|
||||
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
|
||||
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
|
||||
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
|
||||
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
|
||||
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
|
||||
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
|
||||
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
|
||||
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
|
||||
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
|
||||
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
|
||||
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
|
||||
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
|
||||
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
|
||||
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
||||
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
|
||||
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
|
||||
|
||||
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
|
||||
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
|
||||
@@ -63,7 +42,7 @@ seam degrade silently.
|
||||
|
||||
One agent conversation can drive several browser sessions at once:
|
||||
|
||||
- `browser_session_start` returns the session id and makes it the **current session**.
|
||||
- `browser_session` with `action: start` returns the session id and makes it the **current session**.
|
||||
- Every operation tool accepts an optional `session` argument. When omitted, the call acts on the
|
||||
current session (the one most recently started or used); when given, that session becomes current.
|
||||
- Every tool result echoes the session it actually acted on, so the model never has to guess.
|
||||
@@ -72,9 +51,9 @@ One agent conversation can drive several browser sessions at once:
|
||||
|
||||
**Ownership boundary**: the bsk daemon may be shared with other agents, terminals, or dsh
|
||||
instances. The plugin therefore only ever sees and operates on sessions it created itself —
|
||||
an explicit `session` argument naming a foreign or unknown id is rejected, `browser_session_list`
|
||||
shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup can never touch
|
||||
a session owned by another program.
|
||||
an explicit `session` argument naming a foreign or unknown id is rejected, the `list` action on
|
||||
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
|
||||
can never touch a session owned by another program.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -109,7 +88,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
|
||||
# lazyTools: true # reveal browser_* tools only after the skill is invoked
|
||||
```
|
||||
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
|
||||
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
|
||||
`browser-skill` skill is actually invoked — the skill catalog entry is the only
|
||||
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
|
||||
@@ -131,7 +110,7 @@ like the BrowserSkill extension without leaking a single selector into the host
|
||||
shell's theme cannot bleed back in.
|
||||
|
||||
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
|
||||
`browser_session_start`; disappears when all sessions stop (or the plugin unloads).
|
||||
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
|
||||
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
|
||||
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
|
||||
stage while the next one loads, and is kept on errors so the card does not flash),
|
||||
@@ -173,11 +152,11 @@ shell's theme cannot bleed back in.
|
||||
card). Screenshots additionally attach the image itself when the host mounts an attachment store
|
||||
and the active model route declares image input; otherwise the PNG path is returned.
|
||||
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
|
||||
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
|
||||
custom view keeps the terminal block (command + output) and, when the settled result carries an
|
||||
image block, resolves the durable attachment through the client session's authorized
|
||||
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
|
||||
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
|
||||
an image block, resolves the durable attachment through the client session's authorized
|
||||
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
|
||||
Every other `browser_*` tool keeps the stock terminal card. The bundle follows the dsh client
|
||||
The other five tools keep the stock terminal card. The bundle follows the dsh client
|
||||
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
|
||||
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
|
||||
lightningcss.
|
||||
@@ -198,8 +177,9 @@ pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
|
||||
## Publishing
|
||||
|
||||
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
|
||||
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
|
||||
`dsh-plugin-browserskill`.
|
||||
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id, client
|
||||
bundle registration, and `cordis.patch.yml` name are the same specifier so
|
||||
dsh can import and materialize the plugin without a ModuleLoader id mismatch.
|
||||
|
||||
Trigger it by pushing a tag that matches `package.json`'s `version`:
|
||||
|
||||
|
||||
@@ -50,6 +50,10 @@
|
||||
"@deepseek-ai/dsh-client-runtime",
|
||||
"@deepseek-ai/dsh-client-ui-tool",
|
||||
"@deepseek-ai/dsh-client-ui-layout"
|
||||
],
|
||||
"external": [
|
||||
"@deepseek-ai/dsh-client-ui-attachment",
|
||||
"@deepseek-ai/dsh-client-ui-primitives"
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
@@ -18,37 +18,15 @@ function stripFrontmatter(markdown) {
|
||||
}
|
||||
|
||||
const SUPPORTED_BROWSER_TOOLS = [
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_session_list",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_reload",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_snapshot",
|
||||
"browser_observe",
|
||||
"browser_get_html",
|
||||
"browser_screenshot",
|
||||
"browser_console",
|
||||
"browser_network",
|
||||
"browser_click",
|
||||
"browser_hover",
|
||||
"browser_fill",
|
||||
"browser_select",
|
||||
"browser_press",
|
||||
"browser_tab_list",
|
||||
"browser_tab_create",
|
||||
"browser_tab_select",
|
||||
"browser_tab_close",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_return",
|
||||
"browser_request_help",
|
||||
"browser_emulate",
|
||||
"browser_window_resize",
|
||||
"browser_session",
|
||||
"browser_page",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_tabs",
|
||||
"browser_assist",
|
||||
];
|
||||
|
||||
function validateModelFacingSkill(markdown) {
|
||||
function validateModelFacingSkill(markdown, supportedTools) {
|
||||
if (/\bbsk\b/i.test(markdown)) {
|
||||
throw new Error("DSH skill must not expose the internal CLI name");
|
||||
}
|
||||
@@ -57,9 +35,9 @@ function validateModelFacingSkill(markdown) {
|
||||
}
|
||||
|
||||
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
|
||||
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
|
||||
const supported = new Set(supportedTools);
|
||||
const unknown = [...mentioned].filter((name) => !supported.has(name));
|
||||
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
|
||||
const missing = supportedTools.filter((name) => !mentioned.has(name));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
|
||||
}
|
||||
@@ -75,11 +53,11 @@ const DESCRIPTION =
|
||||
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
|
||||
|
||||
const content = `${stripFrontmatter(source).trim()}\n`;
|
||||
validateModelFacingSkill(content);
|
||||
validateModelFacingSkill(content, SUPPORTED_BROWSER_TOOLS);
|
||||
|
||||
const banner =
|
||||
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
|
||||
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
|
||||
"// Source: skill/SKILL.md (DeepSeek Harness six-tool instructions).\n";
|
||||
|
||||
writeFileSync(
|
||||
join(pkg, "src", "skill-content.generated.ts"),
|
||||
|
||||
@@ -1,271 +1,103 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use the injected browser_* tools to automate the user's logged-in Chromium
|
||||
browser in a managed Agent Window.
|
||||
description: Browser automation through six injected domain tools.
|
||||
---
|
||||
|
||||
# browser-skill for DeepSeek Harness
|
||||
|
||||
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
|
||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
||||
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
|
||||
fields, not command-line flags.
|
||||
Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is
|
||||
isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed.
|
||||
|
||||
## Tool availability
|
||||
Loading this skill reveals six tools for the rest of the conversation:
|
||||
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
|
||||
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
|
||||
- `browser_session` owns session lifecycle.
|
||||
- `browser_page` handles navigation and lifecycle waits.
|
||||
- `browser_inspect` reads semantic, visual, console, and network state.
|
||||
- `browser_interact` performs normal page interactions.
|
||||
- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing.
|
||||
- `browser_assist` handles human help, window size, and device emulation.
|
||||
|
||||
All browser operations must use the injected tools directly. Do not translate the examples in this
|
||||
document into terminal commands or invoke another process to control the browser. Direct tool calls
|
||||
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
|
||||
|
||||
## When to use
|
||||
|
||||
- Visit and read pages that require the user's existing browser session.
|
||||
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
|
||||
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
|
||||
- Work with a user tab after explicitly borrowing it into the Agent Window.
|
||||
|
||||
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
|
||||
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
|
||||
personal tab longer than the immediate task requires.
|
||||
Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and
|
||||
parameters; do not guess fields. All browser work must use the injected tools directly so session
|
||||
ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke
|
||||
another process to control the browser.
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every browser task follows this lifecycle:
|
||||
|
||||
1. Call `browser_session_start` and retain its returned `sessionId`.
|
||||
2. Use that session for every browser operation. Pass `session` explicitly when more than one
|
||||
session exists.
|
||||
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
|
||||
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
|
||||
|
||||
Conceptual tool-call sequence:
|
||||
Every task owns a bounded plugin session:
|
||||
|
||||
```text
|
||||
browser_session_start({ url: "https://example.com" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_click({ session: sessionId, target: "@e4" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_session_stop({ session: sessionId })
|
||||
browser_session({ action: "start", ... })
|
||||
... use the returned sessionId for browser work ...
|
||||
browser_session({ action: "stop", session: sessionId })
|
||||
```
|
||||
|
||||
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
|
||||
only those owned sessions. Never guess or reuse a session id from another program. When `session`
|
||||
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
|
||||
for multi-session work.
|
||||
Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another
|
||||
program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep
|
||||
the session open. Stopping also returns borrowed tabs.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Browser work is a bounded task, not open-ended browsing.
|
||||
- Derive a concrete success condition from the user's request.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. Define one observable success condition from the user's request.
|
||||
2. Take the shortest path: observe, act, then perform at most one relevant check.
|
||||
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
|
||||
verification after success.
|
||||
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
|
||||
progress, pause instead of brute-forcing.
|
||||
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
|
||||
user rather than continuing to click.
|
||||
## Observe, act, observe
|
||||
|
||||
## Core interaction loop
|
||||
Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states,
|
||||
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||
also become stale after large DOM changes, so observe again before the next interaction.
|
||||
|
||||
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
|
||||
before interacting:
|
||||
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
|
||||
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||
Do not click the trigger itself unless the user wants the trigger's action.
|
||||
|
||||
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
|
||||
including hints about conditional hover or focus surfaces.
|
||||
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
|
||||
insufficient or a stricter tree is useful.
|
||||
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
|
||||
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
|
||||
4. Observe again after navigation or a meaningful DOM change before using another ref.
|
||||
Escalate reading only as needed:
|
||||
|
||||
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
|
||||
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
|
||||
trigger, observe again, and then act on the newly visible ref.
|
||||
1. `observe` for normal understanding and interaction refs.
|
||||
2. `snapshot` when a stricter static accessibility tree is more useful.
|
||||
3. `html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
4. `screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
Use interaction tools according to their schemas:
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
|
||||
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
|
||||
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
|
||||
intentional.
|
||||
- `browser_select`: pass `values` as an array, including for a single option.
|
||||
- `browser_press`: send a key or combo, optionally focusing `target` first.
|
||||
Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative
|
||||
waits when no navigation is expected. After any page change, discard old refs and observe again.
|
||||
|
||||
Actions may trigger a navigation after the action itself returns. Use
|
||||
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
|
||||
new page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Observation and page reading priority
|
||||
Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs.
|
||||
Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never
|
||||
invent a tab id or keep a personal tab borrowed across unrelated work.
|
||||
|
||||
Escalate page reading only as needed:
|
||||
## Ask the human when needed
|
||||
|
||||
1. `browser_observe` for normal semantic understanding and interaction refs.
|
||||
2. `browser_snapshot` for a stricter static accessibility tree.
|
||||
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
|
||||
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
|
||||
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
|
||||
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
|
||||
Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or
|
||||
another step the user must complete. Give a precise prompt and highlight fresh targets when concrete
|
||||
controls are involved. Use completion criteria only for a clear stable success signal.
|
||||
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
|
||||
return an inline image or a saved path depending on the deployment.
|
||||
Resume only after the user continues or the criteria complete. Treat cancellation as rejection and
|
||||
timeout as a blocker rather than retrying. Observe again after control returns before reasoning about
|
||||
the new state or using refs.
|
||||
|
||||
## Tabs and the Agent Window boundary
|
||||
The same tool can resize the Agent Window or emulate a device when the task requires visual or
|
||||
responsive testing. Emulation is scoped to one tab.
|
||||
|
||||
Write actions normally affect only Agent Window tabs.
|
||||
## Debug and recover without wandering
|
||||
|
||||
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
|
||||
`scope: "user"` to locate a user tab before borrowing it.
|
||||
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
|
||||
for a background tab.
|
||||
- `browser_tab_select` focuses an Agent Window tab.
|
||||
- `browser_tab_close` closes an Agent Window tab.
|
||||
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
|
||||
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
|
||||
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
|
||||
Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics.
|
||||
Continue from returned sequence cursors instead of rereading the same buffer.
|
||||
|
||||
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
|
||||
from page position or title.
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab: list tabs instead of guessing.
|
||||
- Unknown session: list owned sessions or start one; never try foreign ids.
|
||||
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
|
||||
- Unrecoverable failure: report the blocker and stop the owned session.
|
||||
|
||||
## Navigation and lifecycle
|
||||
|
||||
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
|
||||
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
|
||||
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
|
||||
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
|
||||
wait. Avoid speculative waits when no navigation is expected.
|
||||
|
||||
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
|
||||
`browser_snapshot` again.
|
||||
|
||||
## Ask the user for browser help
|
||||
|
||||
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
|
||||
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
|
||||
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
|
||||
adds clarity.
|
||||
|
||||
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
|
||||
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
|
||||
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
|
||||
|
||||
```text
|
||||
browser_request_help({
|
||||
session: sessionId,
|
||||
title: "Sign in required",
|
||||
prompt: "Complete sign-in, then return control after the account page appears.",
|
||||
targets: ["@e7"],
|
||||
timeoutMs: 300000,
|
||||
completionCriteria: {
|
||||
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
|
||||
stableForMs: 1000
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Interpret outcomes deliberately:
|
||||
|
||||
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
|
||||
again before reasoning about the new state or using refs.
|
||||
- `cancelled`: stop the requested flow.
|
||||
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
|
||||
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
|
||||
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
|
||||
|
||||
## Debugging without page scripting
|
||||
|
||||
Use the read-only debugging tools only when relevant to the user's task:
|
||||
|
||||
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
|
||||
Set `includeStack: true` when stack frames are useful.
|
||||
- `browser_network` reads buffered response and failure metadata. It does not return request or
|
||||
response headers or bodies and does not modify traffic.
|
||||
|
||||
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
|
||||
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
|
||||
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
|
||||
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
|
||||
strictly requires either capability, explain that it is not available in this phase.
|
||||
|
||||
## Window and device controls
|
||||
|
||||
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
|
||||
must be from 100 through 7680 CSS pixels.
|
||||
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
|
||||
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
|
||||
inherit it.
|
||||
|
||||
## Tool reference
|
||||
|
||||
### Sessions
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
|
||||
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
|
||||
| `browser_session_list` | List owned sessions and identify the current one. |
|
||||
|
||||
### Navigation and reading
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_navigate` | Navigate the active tab to a URL. |
|
||||
| `browser_navigate_back` | Go back one history entry. |
|
||||
| `browser_navigate_forward` | Go forward one history entry. |
|
||||
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
|
||||
| `browser_observe` | Read the primary semantic page representation with refs. |
|
||||
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
|
||||
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
|
||||
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
|
||||
|
||||
### Interaction
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_click` | Click a ref or CSS selector. |
|
||||
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
|
||||
| `browser_fill` | Fill a text-capable element. |
|
||||
| `browser_select` | Select one or more option values. |
|
||||
| `browser_press` | Send a keyboard key or combo. |
|
||||
|
||||
### Tabs
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_tab_list` | List visible user and Agent Window tabs. |
|
||||
| `browser_tab_create` | Create an Agent Window tab. |
|
||||
| `browser_tab_select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
|
||||
| `browser_tab_return` | Return a borrowed tab to its original window. |
|
||||
|
||||
### Assistance, diagnostics, and display
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_request_help` | Pause for an explicit user browser step. |
|
||||
| `browser_console` | Read buffered console and exception entries. |
|
||||
| `browser_network` | Read buffered response and failure metadata. |
|
||||
| `browser_window_resize` | Resize the Agent Window. |
|
||||
| `browser_emulate` | Apply or clear per-tab device emulation. |
|
||||
|
||||
## Error handling and final checklist
|
||||
|
||||
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
|
||||
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
|
||||
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
|
||||
actions with unchanged state.
|
||||
|
||||
Before finishing a browser task, verify that:
|
||||
|
||||
- The user's observable goal is met, or the blocker is clearly reported.
|
||||
- Any borrowed tab has been returned when practical.
|
||||
- The owned session has been stopped unless the user explicitly asked to keep it open.
|
||||
- No further page action is taken after success.
|
||||
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tools or route around those limits.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* are hidden from every surface, so their Agent Windows would otherwise
|
||||
* linger with no way back to them.
|
||||
*
|
||||
* Ownership lineage: browser_session_start records the calling agent's
|
||||
* Ownership lineage: browser_session action=start records the calling agent's
|
||||
* session id PLUS every ancestor along `header.parentSession` (a subagent's
|
||||
* browsers are reaped when any ancestor is archived, the root conversation
|
||||
* included). Lineage is resolved through the host session store; an
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* The six model-facing BrowserSkill tools. Each action dispatches to a private
|
||||
* operation handler, so the model receives only six schemas while every action
|
||||
* retains the existing ownership, cancellation, queuing, observation,
|
||||
* screenshot, and presentation behavior.
|
||||
*/
|
||||
|
||||
import { defineTool, type ParameterSchemaSpec, type ToolDefinition } from "@deepseek-ai/dsh-tools";
|
||||
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
|
||||
import { createBrowserOperationDefinitions, type ToolDeps } from "./tools";
|
||||
|
||||
const DEVICE_PRESETS = [
|
||||
"iphone-14",
|
||||
"iphone-14-pro-max",
|
||||
"iphone-se",
|
||||
"pixel-7",
|
||||
"galaxy-s23",
|
||||
"ipad-mini",
|
||||
"galaxy-tab-s8",
|
||||
] as const;
|
||||
|
||||
const TARGET_PARAM = {
|
||||
type: "string",
|
||||
description: "Snapshot ref such as @e3, or a CSS selector.",
|
||||
} as const;
|
||||
|
||||
const DOMAIN_RESULT = { type: "json" } as const;
|
||||
|
||||
interface BrowserToolSpec {
|
||||
name: string;
|
||||
description: string;
|
||||
actions: Record<string, string>;
|
||||
parameters: ParameterSchemaSpec;
|
||||
}
|
||||
|
||||
function operationArgs(args: unknown): Record<string, unknown> {
|
||||
const { action: _action, ...rest } = args as Record<string, unknown>;
|
||||
return rest;
|
||||
}
|
||||
|
||||
function definitionFor(
|
||||
definitions: ReadonlyMap<string, ToolDefinition>,
|
||||
actions: Record<string, string>,
|
||||
action: unknown,
|
||||
): ToolDefinition {
|
||||
const operationName = typeof action === "string" ? actions[action] : undefined;
|
||||
const definition = operationName === undefined ? undefined : definitions.get(operationName);
|
||||
if (definition === undefined) throw new Error(`unsupported browser action: ${String(action)}`);
|
||||
return definition;
|
||||
}
|
||||
|
||||
function defineBrowserTool(
|
||||
spec: BrowserToolSpec,
|
||||
definitions: ReadonlyMap<string, ToolDefinition>,
|
||||
): ToolDefinition {
|
||||
const actionNames = Object.keys(spec.actions);
|
||||
return defineTool({
|
||||
name: spec.name,
|
||||
description: spec.description,
|
||||
parameters: {
|
||||
action: {
|
||||
type: "string",
|
||||
required: true,
|
||||
enum: actionNames,
|
||||
description:
|
||||
"Operation to perform. Other arguments are action-dependent and are validated by that operation.",
|
||||
},
|
||||
...spec.parameters,
|
||||
},
|
||||
output: {
|
||||
schema: DOMAIN_RESULT,
|
||||
render(args, value) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.output.render(operationArgs(args), value);
|
||||
},
|
||||
},
|
||||
isConcurrencySafe(args) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.isConcurrencySafe?.(operationArgs(args)) === true;
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return (await definition.execute(operationArgs(args), exec)) as never;
|
||||
},
|
||||
presentCall(args) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.presentCall?.(operationArgs(args));
|
||||
},
|
||||
presentResult(args, result) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.presentResult?.(operationArgs(args), result);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
|
||||
{
|
||||
name: "browser_session",
|
||||
description:
|
||||
"Manage plugin-owned browser sessions. Actions: start opens an Agent Window; stop closes an " +
|
||||
"owned session; list returns owned sessions. For start, url/device/width/height/noFocus/browser " +
|
||||
"are optional. For stop, session is optional and defaults to the current owned session.",
|
||||
actions: {
|
||||
start: "session.start",
|
||||
stop: "session.stop",
|
||||
list: "session.list",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
url: { type: "string", description: "Initial URL for start." },
|
||||
width: { type: "integer", description: "Agent Window width; start requires height too." },
|
||||
height: { type: "integer", description: "Agent Window height; start requires width too." },
|
||||
noFocus: { type: "boolean", description: "Start the Agent Window in the background." },
|
||||
browser: { type: "string", description: "Browser instance id for start." },
|
||||
device: { type: "string", enum: DEVICE_PRESETS, description: "Device preset for start." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_page",
|
||||
description:
|
||||
"Navigate and wait on the active Agent Window tab. Actions: navigate, back, forward, reload, " +
|
||||
"wait. navigate requires url; reload optionally accepts hard; navigation actions accept " +
|
||||
"waitUntil/timeoutMs. Observe again after a meaningful page change before reusing refs.",
|
||||
actions: {
|
||||
navigate: "page.navigate",
|
||||
back: "page.back",
|
||||
forward: "page.forward",
|
||||
reload: "page.reload",
|
||||
wait: "page.wait",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
url: { type: "string", description: "Destination URL; required for navigate." },
|
||||
waitUntil: WAIT_UNTIL_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
hard: { type: "boolean", description: "Bypass cache for reload." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_inspect",
|
||||
description:
|
||||
"Read page state without arbitrary script execution. Actions: observe, snapshot, html, " +
|
||||
"screenshot, console, network. Prefer observe, then snapshot, then bounded html; use screenshot " +
|
||||
"for visual evidence. console/network support cursor fields since/limit/maxTextChars.",
|
||||
actions: {
|
||||
observe: "inspect.observe",
|
||||
snapshot: "inspect.snapshot",
|
||||
html: "inspect.html",
|
||||
screenshot: "inspect.screenshot",
|
||||
console: "inspect.console",
|
||||
network: "inspect.network",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
maxDepth: { type: "integer", description: "Tree depth cap for observe/snapshot." },
|
||||
maxTokens: { type: "integer", description: "Token cap for observe/snapshot." },
|
||||
ref: { type: "string", description: "Fresh ref for scoped html or cropped screenshot." },
|
||||
maxBytes: { type: "integer", description: "HTML byte cap." },
|
||||
since: { type: "integer", description: "Console/network sequence cursor." },
|
||||
limit: { type: "integer", description: "Console/network entry cap." },
|
||||
maxTextChars: { type: "integer", description: "Console/network per-entry text cap." },
|
||||
includeStack: { type: "boolean", description: "Include console stack frames." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_interact",
|
||||
description:
|
||||
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
|
||||
"press. click/hover/fill/select require target; fill also requires value; select requires " +
|
||||
"values; press requires key and may optionally focus target first.",
|
||||
actions: {
|
||||
click: "interact.click",
|
||||
hover: "interact.hover",
|
||||
fill: "interact.fill",
|
||||
select: "interact.select",
|
||||
press: "interact.press",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
target: TARGET_PARAM,
|
||||
button: { type: "string", enum: ["left", "middle", "right"], description: "Click button." },
|
||||
clickCount: { type: "integer", description: "Click count." },
|
||||
value: { type: "string", description: "Text for fill." },
|
||||
noClear: { type: "boolean", description: "Append instead of clearing for fill." },
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
||||
description: "Modifiers held during hover.",
|
||||
},
|
||||
settleMs: { type: "integer", description: "Hover settle delay." },
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
values: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description: "Option values for select; at least one is required.",
|
||||
},
|
||||
key: { type: "string", description: "Key or combo for press." },
|
||||
holdMs: { type: "integer", description: "Key hold duration for press." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_tabs",
|
||||
description:
|
||||
"Manage tabs visible to an owned session. Actions: list, create, select, close, borrow, return. " +
|
||||
"select/close/borrow/return require tabId from list/create. Borrow moves a user tab into the " +
|
||||
"Agent Window; return it as soon as the task finishes.",
|
||||
actions: {
|
||||
list: "tabs.list",
|
||||
create: "tabs.create",
|
||||
select: "tabs.select",
|
||||
close: "tabs.close",
|
||||
borrow: "tabs.borrow",
|
||||
return: "tabs.return",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
scope: {
|
||||
type: "string",
|
||||
enum: ["user", "agent", "all"],
|
||||
description: "Tab scope for list.",
|
||||
},
|
||||
url: { type: "string", description: "Initial URL for create." },
|
||||
active: { type: "boolean", description: "Focus the created tab (default true)." },
|
||||
index: { type: "integer", description: "Insertion index for create." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_assist",
|
||||
description:
|
||||
"Display and human-assistance operations. Actions: resize, emulate, request-help. resize " +
|
||||
"requires width/height; emulate accepts device or width/height/mobile, or off alone; " +
|
||||
"request-help requires prompt and can wait for explicit completion criteria.",
|
||||
actions: {
|
||||
resize: "assist.resize",
|
||||
emulate: "assist.emulate",
|
||||
"request-help": "assist.request-help",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
width: { type: "integer", description: "Window/viewport width." },
|
||||
height: { type: "integer", description: "Window/viewport height." },
|
||||
device: { type: "string", enum: DEVICE_PRESETS, description: "Emulation device preset." },
|
||||
mobile: { type: "boolean", description: "Enable a mobile viewport with width/height." },
|
||||
off: { type: "boolean", description: "Clear emulation; use alone." },
|
||||
prompt: { type: "string", description: "Instructions shown to the user for request-help." },
|
||||
title: { type: "string", description: "Optional request-help title." },
|
||||
targets: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description: "Refs/selectors highlighted for request-help.",
|
||||
},
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
completionCriteria: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
description: "Automatic completion detector for request-help.",
|
||||
properties: {
|
||||
any: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
urlContains: { type: "string" },
|
||||
urlMatches: { type: "string" },
|
||||
selectorExists: { type: "string" },
|
||||
selectorMissing: { type: "string" },
|
||||
textExists: { type: "string" },
|
||||
textMissing: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
all: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
urlContains: { type: "string" },
|
||||
urlMatches: { type: "string" },
|
||||
selectorExists: { type: "string" },
|
||||
selectorMissing: { type: "string" },
|
||||
textExists: { type: "string" },
|
||||
textMissing: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
stableForMs: { type: "integer" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function indexOperations(definitions: ToolDefinition[]): ReadonlyMap<string, ToolDefinition> {
|
||||
const indexed = new Map<string, ToolDefinition>();
|
||||
for (const definition of definitions) {
|
||||
if (indexed.has(definition.name)) {
|
||||
throw new Error(`duplicate browser operation definition: ${definition.name}`);
|
||||
}
|
||||
indexed.set(definition.name, definition);
|
||||
}
|
||||
|
||||
const routed = new Set(BROWSER_TOOL_SPECS.flatMap((spec) => Object.values(spec.actions)));
|
||||
const missing = [...routed].filter((name) => !indexed.has(name));
|
||||
const unreachable = [...indexed.keys()].filter((name) => !routed.has(name));
|
||||
if (missing.length > 0 || unreachable.length > 0) {
|
||||
throw new Error(
|
||||
[
|
||||
missing.length > 0 ? `missing operations: ${missing.join(", ")}` : undefined,
|
||||
unreachable.length > 0 ? `unreachable operations: ${unreachable.join(", ")}` : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("; "),
|
||||
);
|
||||
}
|
||||
return indexed;
|
||||
}
|
||||
|
||||
/** Register the complete six-tool browser suite; returns its combined disposer. */
|
||||
export function registerBrowserTools(deps: ToolDeps): () => void {
|
||||
const definitions = indexOperations(createBrowserOperationDefinitions(deps));
|
||||
const disposers = BROWSER_TOOL_SPECS.map((spec) =>
|
||||
deps.ctx.tools.register(defineBrowserTool(spec, definitions)),
|
||||
).filter((dispose): dispose is () => void => typeof dispose === "function");
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
/* browser_screenshot toolview: disclosure row over a terminal block plus the
|
||||
/* browser_inspect screenshot toolview: disclosure row over a terminal block plus the
|
||||
screenshot image. Semantic tokens only (--dsw-alias-*), no literal colors. */
|
||||
|
||||
.card {
|
||||
+80
-23
@@ -1,8 +1,6 @@
|
||||
// browser_screenshot keyed toolview: the terminal card's command line and
|
||||
// output stay exactly as the host renders them, and a result carrying an
|
||||
// image block additionally renders the screenshot itself (thumbnail with the
|
||||
// shared click-to-open lightbox). Pure function of the frozen call/result
|
||||
// slice — live and replay paths render identically.
|
||||
// browser_inspect keyed toolview: all inspect actions retain a terminal card;
|
||||
// screenshot results additionally render the image attachment. Pure function
|
||||
// of the frozen call/result slice, so live and replay paths stay identical.
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import {
|
||||
@@ -19,13 +17,13 @@ import {
|
||||
} from "@deepseek-ai/dsh-client-ui-primitives";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { useState } from "react";
|
||||
import css from "./ScreenshotToolView.module.css";
|
||||
import css from "./BrowserInspectToolView.module.css";
|
||||
|
||||
/** Resolved loader bound to the owning session at registration time. */
|
||||
export type ScreenshotImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
|
||||
export type BrowserInspectImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
|
||||
|
||||
export type ScreenshotToolViewProps = ToolCallViewProps & {
|
||||
loadImage: ScreenshotImageLoader;
|
||||
export type BrowserInspectToolViewProps = ToolCallViewProps & {
|
||||
loadImage: BrowserInspectImageLoader;
|
||||
};
|
||||
|
||||
type ViewState = "running" | "ok" | "error";
|
||||
@@ -36,6 +34,7 @@ interface ViewModel {
|
||||
readonly output: string | null;
|
||||
readonly image: ImageAttachmentRef | null;
|
||||
readonly summary: string;
|
||||
readonly title: string;
|
||||
}
|
||||
|
||||
const TERMINAL_LABELS: Partial<TerminalBlockLabels> = {
|
||||
@@ -66,18 +65,71 @@ function firstLine(text: string): string {
|
||||
}
|
||||
|
||||
/** Rebuild the command line from the logged arguments (mirrors the host presenter). */
|
||||
function commandOf(argsRaw: string, callId: string): string {
|
||||
const INSPECT_COMMANDS: Record<string, string> = {
|
||||
observe: "observe",
|
||||
snapshot: "snapshot",
|
||||
html: "get-html",
|
||||
screenshot: "screenshot",
|
||||
console: "console",
|
||||
network: "network",
|
||||
};
|
||||
|
||||
function titleOf(action: string): string {
|
||||
return action === "html" ? "HTML" : `${action.slice(0, 1).toUpperCase()}${action.slice(1)}`;
|
||||
}
|
||||
|
||||
interface InspectArgs {
|
||||
action?: unknown;
|
||||
session?: unknown;
|
||||
tabId?: unknown;
|
||||
maxDepth?: unknown;
|
||||
maxTokens?: unknown;
|
||||
ref?: unknown;
|
||||
maxBytes?: unknown;
|
||||
since?: unknown;
|
||||
limit?: unknown;
|
||||
maxTextChars?: unknown;
|
||||
includeStack?: unknown;
|
||||
}
|
||||
|
||||
function appendNumber(parts: string[], flag: string, value: unknown): void {
|
||||
if (typeof value === "number" && Number.isFinite(value)) parts.push(flag, String(value));
|
||||
}
|
||||
|
||||
function commandOf(argsRaw: string, callId: string): { command: string; title: string } {
|
||||
try {
|
||||
const args = JSON.parse(argsRaw) as { session?: unknown; ref?: unknown };
|
||||
const parts = ["bsk", "screenshot", "--session"];
|
||||
const args = JSON.parse(argsRaw) as InspectArgs;
|
||||
const action = typeof args.action === "string" ? args.action : "inspect";
|
||||
const command = INSPECT_COMMANDS[action] ?? action;
|
||||
const parts = ["bsk", command, "--session"];
|
||||
parts.push(
|
||||
typeof args.session === "string" && args.session !== "" ? args.session : "(current)",
|
||||
);
|
||||
if (typeof args.ref === "string" && args.ref !== "") parts.push("--ref", args.ref);
|
||||
return parts.join(" ");
|
||||
if (action === "observe" || action === "snapshot") {
|
||||
appendNumber(parts, "--max-depth", args.maxDepth);
|
||||
appendNumber(parts, "--max-tokens", args.maxTokens);
|
||||
}
|
||||
if (action === "html" || action === "console" || action === "network") {
|
||||
appendNumber(parts, "--tab-id", args.tabId);
|
||||
}
|
||||
if ((action === "html" || action === "screenshot") && typeof args.ref === "string") {
|
||||
if (args.ref !== "") parts.push("--ref", args.ref);
|
||||
}
|
||||
if (action === "html") appendNumber(parts, "--max-bytes", args.maxBytes);
|
||||
if (action === "console" || action === "network") {
|
||||
appendNumber(parts, "--since", args.since);
|
||||
appendNumber(parts, "--limit", args.limit);
|
||||
appendNumber(parts, "--max-text-chars", args.maxTextChars);
|
||||
}
|
||||
if (action === "console" && args.includeStack === true) parts.push("--include-stack");
|
||||
return { command: parts.join(" "), title: titleOf(action) };
|
||||
} catch {
|
||||
// Streaming can expose a truncated JSON prefix; it is still the best label.
|
||||
return argsRaw === "" ? `bsk screenshot (${callId})` : `bsk screenshot ${firstLine(argsRaw)}`;
|
||||
return {
|
||||
command:
|
||||
argsRaw === "" ? `browser_inspect (${callId})` : `browser_inspect ${firstLine(argsRaw)}`,
|
||||
title: "Inspect",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,9 +137,9 @@ function commandOf(argsRaw: string, callId: string): string {
|
||||
export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
|
||||
const settled = "kind" in block;
|
||||
const argsRaw = (settled ? block.call?.argsRaw : block.argsRaw) ?? "";
|
||||
const command = commandOf(argsRaw, block.callId);
|
||||
const { command, title } = commandOf(argsRaw, block.callId);
|
||||
if (!settled) {
|
||||
return { state: "running", command, output: null, image: null, summary: command };
|
||||
return { state: "running", command, output: null, image: null, summary: command, title };
|
||||
}
|
||||
const textBlock = block.content.find((item) => item.type === "text");
|
||||
const imageBlock = block.content.find((item) => item.type === "image");
|
||||
@@ -96,7 +148,7 @@ export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
|
||||
imageBlock !== undefined && imageBlock.type === "image" ? imageBlock.attachment : null;
|
||||
const state: ViewState = block.isError ? "error" : "ok";
|
||||
const summary = output !== null ? firstLine(output) : command;
|
||||
return { state, command, output, image, summary };
|
||||
return { state, command, output, image, summary, title };
|
||||
}
|
||||
|
||||
function dotState(state: ViewState): StateDotState {
|
||||
@@ -111,19 +163,24 @@ function dotState(state: ViewState): StateDotState {
|
||||
}
|
||||
|
||||
/**
|
||||
* Render one browser_screenshot call: a disclosure row over a terminal block
|
||||
* (command + output) plus the screenshot image when the result carries one.
|
||||
* Render one browser_inspect call: a disclosure row over a terminal block,
|
||||
* plus the screenshot image when that action returns an image attachment.
|
||||
*/
|
||||
export function ScreenshotToolView({ block, cwd, inspect, loadImage }: ScreenshotToolViewProps) {
|
||||
export function BrowserInspectToolView({
|
||||
block,
|
||||
cwd,
|
||||
inspect,
|
||||
loadImage,
|
||||
}: BrowserInspectToolViewProps) {
|
||||
const model = viewModelOf(block);
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const expandable = model.state === "running" || model.output !== null || model.image !== null;
|
||||
const open = expanded && expandable;
|
||||
return (
|
||||
<div className={css.card} data-tool="browser_screenshot" data-state={model.state}>
|
||||
<div className={css.card} data-tool="browser_inspect" data-state={model.state}>
|
||||
<DisclosureRow
|
||||
icon={<StateDot state={dotState(model.state)} />}
|
||||
title="Screenshot"
|
||||
title={model.title}
|
||||
open={open}
|
||||
expandable={expandable}
|
||||
onToggle={() => setExpanded((value) => !value)}
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt). The
|
||||
* dsh-plugin-browserskill browser half: the `browser_inspect` keyed toolview
|
||||
* plus the observation overlay (live thumbnails + interrupt). The
|
||||
* default carrier is a floating card on the `shell.overlay` seat; when the
|
||||
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
|
||||
* moves into a sidebar tab instead (see observation-sidebar.tsx).
|
||||
@@ -16,10 +16,10 @@ import { createElement } from "react";
|
||||
// <style> tags; selectors stay unhashed so `cn(..., "bsk-obs")` roots match).
|
||||
import "./bsk-tokens.nomodule.css";
|
||||
import "./bsk-ui.nomodule.css";
|
||||
import { BrowserInspectToolView } from "./BrowserInspectToolView";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
@@ -65,9 +65,9 @@ export function apply(ctx: ClientContext): void {
|
||||
const sessions = ctx.get("sessions") as unknown as ISessions;
|
||||
ctx.slots.inject("tool.call.toolview", () =>
|
||||
ctx.slots.register(
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "tool.call.toolview", key: "browser_inspect" },
|
||||
(props: ToolCallViewProps) =>
|
||||
createElement(ScreenshotToolView, {
|
||||
createElement(BrowserInspectToolView, {
|
||||
...props,
|
||||
loadImage: (attachment: ImageAttachmentRef) =>
|
||||
loadSessionImage(sessions, props.sessionId, attachment),
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
|
||||
* path plus pixel metadata); when the host mounts a durable attachment store
|
||||
* AND the calling route declares image input, the PNG bytes are additionally
|
||||
* committed through `ctx.attachments` so the render step can attach the image
|
||||
* itself to the tool result. Any uncertainty (no store, unknown route,
|
||||
* text-only model) falls back to the path-only form.
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
|
||||
* file path plus pixel metadata); when the host mounts a durable attachment
|
||||
* store AND the calling route declares image input, the capture bytes are
|
||||
* additionally committed through `ctx.attachments` so the render step can
|
||||
* attach the image itself to the tool result. Any uncertainty (no store,
|
||||
* unknown route, text-only model, unrecognized bytes) falls back to the
|
||||
* path-only form.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
@@ -33,6 +34,54 @@ interface LlmLike {
|
||||
): Promise<{ inputModalities?: readonly string[] }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sniff the image media type from magic bytes. The capture path can hand back
|
||||
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
|
||||
* returning JPEG regardless of the requested format), so the declared type can
|
||||
* never be trusted from the file extension alone.
|
||||
* @param data - first bytes of the capture.
|
||||
* @returns the detected media type, or undefined when unrecognized.
|
||||
*/
|
||||
function sniffImageMediaType(data: Uint8Array): string | undefined {
|
||||
if (data.length >= 4) {
|
||||
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
|
||||
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||
return "image/png";
|
||||
}
|
||||
// JPEG SOI: FF D8 FF
|
||||
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
|
||||
return "image/jpeg";
|
||||
}
|
||||
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
|
||||
if (
|
||||
data[0] === 0x52 &&
|
||||
data[1] === 0x49 &&
|
||||
data[2] === 0x46 &&
|
||||
data[3] === 0x46 &&
|
||||
data.length >= 12 &&
|
||||
data[8] === 0x57 &&
|
||||
data[9] === 0x45 &&
|
||||
data[10] === 0x42 &&
|
||||
data[11] === 0x50
|
||||
) {
|
||||
return "image/webp";
|
||||
}
|
||||
// GIF: "GIF87a" or "GIF89a"
|
||||
if (
|
||||
data[0] === 0x47 &&
|
||||
data[1] === 0x49 &&
|
||||
data[2] === 0x46 &&
|
||||
data[3] === 0x38 &&
|
||||
data.length >= 6 &&
|
||||
(data[4] === 0x37 || data[4] === 0x39) &&
|
||||
data[5] === 0x61
|
||||
) {
|
||||
return "image/gif";
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit screenshot bytes to the host attachment store when the composition
|
||||
* supports durable images on the current model route.
|
||||
@@ -46,7 +95,14 @@ export async function trySaveScreenshot(
|
||||
): Promise<ImageAttachmentRef | undefined> {
|
||||
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
|
||||
// Declare the real media type from the bytes, never a hard-coded "image/png".
|
||||
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
|
||||
// silently degrades to path-only; sniffing first lets JPEG captures (returned
|
||||
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
|
||||
// inline correctly.
|
||||
const mediaType = sniffImageMediaType(data);
|
||||
if (mediaType === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
|
||||
if (
|
||||
data.byteLength >
|
||||
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
||||
@@ -55,9 +111,9 @@ export async function trySaveScreenshot(
|
||||
}
|
||||
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
||||
try {
|
||||
return await attachments.saveImage({ data, mediaType: "image/png", name });
|
||||
return await attachments.saveImage({ data, mediaType, name });
|
||||
} catch {
|
||||
// A store hiccup must not fail the tool call; the PNG path still works.
|
||||
// A store hiccup must not fail the tool call; the path-only form still works.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { armArchiveCleanup } from "./archive-cleanup";
|
||||
import { registerBrowserTools } from "./browser-tools";
|
||||
import { armLazyTools } from "./lazy-tools";
|
||||
import { ObservationService } from "./observation";
|
||||
import { registerObservationRoutes } from "./observation-http";
|
||||
@@ -20,9 +21,9 @@ import { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
import type { PluginConfig } from "./tools";
|
||||
|
||||
export const name = "dsh-plugin-browserskill";
|
||||
export const name = "@wxg-prc-cpg/browser-skill-dsh-plugin";
|
||||
export const inject = ["tools"];
|
||||
|
||||
/** Runtime configuration schema (validated and defaulted by Cordis). */
|
||||
@@ -100,11 +101,9 @@ export function apply(
|
||||
// exact agent context at startup so DSH always loads the browser_* protocol
|
||||
// instructions; the shared CLI skill remains untouched for other agents.
|
||||
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
|
||||
const removeSuite = resolved.lazyTools
|
||||
? armLazyTools(ctx, () =>
|
||||
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
||||
)
|
||||
: registerTools({ ctx, runner, registry, config: resolved, observation, queue });
|
||||
const registerSuite = () =>
|
||||
registerBrowserTools({ ctx, runner, registry, config: resolved, observation, queue });
|
||||
const removeSuite = resolved.lazyTools ? armLazyTools(ctx, registerSuite) : registerSuite();
|
||||
// Route registration rides ctx.inject: the webServer service may be provided
|
||||
// AFTER this plugin loads, and in headless compositions it never appears (the
|
||||
// callback simply never runs, leaving the rest of the plugin unaffected).
|
||||
@@ -133,7 +132,7 @@ export function apply(
|
||||
);
|
||||
|
||||
// Unload cleanup: kill in-flight children, then stop every session this
|
||||
// plugin OWNS (created via browser_session_start). Referenced or unknown
|
||||
// plugin OWNS (created via browser_session action=start). Referenced or unknown
|
||||
// sessions belonging to other programs on the shared daemon are never
|
||||
// touched; per-stop failures (already stopped externally, daemon restart)
|
||||
// are swallowed so one stale handle cannot abort the rest.
|
||||
@@ -154,6 +153,7 @@ export function apply(
|
||||
}
|
||||
|
||||
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
|
||||
export { registerBrowserTools } from "./browser-tools";
|
||||
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
|
||||
export { ObservationService } from "./observation";
|
||||
export { registerObservationRoutes } from "./observation-http";
|
||||
@@ -162,4 +162,3 @@ export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
|
||||
export { BskError, createBskRunner } from "./runner";
|
||||
export { SessionRegistry } from "./sessions";
|
||||
export type { PluginConfig, ToolDeps } from "./tools";
|
||||
export { registerTools } from "./tools";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Lazy tool-schema injection ("progressive disclosure", final stage): with
|
||||
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
|
||||
* `lazyTools` on (the default), the six browser tool schemas stay OUT of the
|
||||
* system prompt until the `browser-skill` skill has actually been invoked —
|
||||
* the skill catalog entry is the only advertisement. One successful
|
||||
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
|
||||
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
|
||||
/**
|
||||
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
|
||||
* the reveal already happened — the tool suite itself.
|
||||
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
|
||||
* @param registerSuite - registers the six browser tools and returns their disposer.
|
||||
*/
|
||||
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
|
||||
let suiteDisposer: (() => void) | undefined;
|
||||
|
||||
@@ -161,7 +161,7 @@ export class ObservationService {
|
||||
this.emit({ type: "upsert", session: { ...entry } });
|
||||
}
|
||||
|
||||
/** Register a fresh owned session (called from browser_session_start). */
|
||||
/** Register a fresh owned session (called from browser_session action=start). */
|
||||
addSession(sessionId: string, url?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
|
||||
@@ -176,7 +176,7 @@ export class ObservationService {
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Drop a session (called from browser_session_stop). */
|
||||
/** Drop a session (called from browser_session action=stop). */
|
||||
removeSession(sessionId: string): void {
|
||||
this.cancelCapture(sessionId);
|
||||
this.foregroundDepth.delete(sessionId);
|
||||
@@ -276,7 +276,7 @@ export class ObservationService {
|
||||
|
||||
/**
|
||||
* Stop one owned session and close its Agent Window (the overlay's stop
|
||||
* button — same end state as `browser_session_stop`). Never waits behind a
|
||||
* button — same end state as `browser_session` action=stop). Never waits behind a
|
||||
* hung in-flight command: tool children are killed first so the session's
|
||||
* keyed queue drains immediately, and no further captures queue up. A
|
||||
* session the daemon already forgot stops idempotently — the goal state
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
|
||||
export type ToolRegistrar = (definition: ToolDefinition) => void;
|
||||
|
||||
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
|
||||
|
||||
|
||||
@@ -23,10 +23,10 @@ export function registerPhaseOneInteractionTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_hover",
|
||||
name: "interact.hover",
|
||||
description:
|
||||
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
|
||||
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
|
||||
"Run browser_inspect action=observe or snapshot afterwards to discover newly visible refs.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -68,7 +68,7 @@ export function registerPhaseOneInteractionTools(
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
requirePositive(args.settleMs, "settleMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_hover");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=hover)");
|
||||
const cmdArgs = ["hover", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
@@ -102,7 +102,7 @@ export function registerPhaseOneInteractionTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_select",
|
||||
name: "interact.select",
|
||||
description:
|
||||
"Set a select element's option values. Pass one value for a normal select or multiple " +
|
||||
"values for a multi-select; values replace the current selection.",
|
||||
@@ -155,7 +155,7 @@ export function registerPhaseOneInteractionTools(
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
if (args.values.length === 0) throw new Error("values must contain at least one option");
|
||||
const sessionId = registry.resolve(args.session, "browser_select");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=select)");
|
||||
const cmdArgs = ["select", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
for (const value of args.values) cmdArgs.push("--value", value);
|
||||
|
||||
@@ -51,7 +51,7 @@ export function registerPhaseOneNavigationTools(
|
||||
const { registry } = deps;
|
||||
|
||||
const registerHistory = (direction: "back" | "forward") => {
|
||||
const toolName = `browser_navigate_${direction}` as const;
|
||||
const toolName = `page.${direction}` as const;
|
||||
const command = `navigate-${direction}`;
|
||||
register(
|
||||
defineTool({
|
||||
@@ -106,7 +106,7 @@ export function registerPhaseOneNavigationTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_reload",
|
||||
name: "page.reload",
|
||||
description:
|
||||
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
|
||||
"bypass the HTTP cache.",
|
||||
@@ -132,7 +132,7 @@ export function registerPhaseOneNavigationTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_reload");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=reload)");
|
||||
const cmdArgs = ["reload", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
|
||||
@@ -163,7 +163,7 @@ export function registerPhaseOneNavigationTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_wait_for_navigation",
|
||||
name: "page.wait",
|
||||
description:
|
||||
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
|
||||
"that may navigate when the action result itself does not wait for navigation.",
|
||||
@@ -195,7 +195,7 @@ export function registerPhaseOneNavigationTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=wait)");
|
||||
const timeoutMs = args.timeoutMs ?? 30_000;
|
||||
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
|
||||
@@ -143,7 +143,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_request_help",
|
||||
name: "assist.request-help",
|
||||
description:
|
||||
"Pause browser automation and ask the user to complete an in-page step such as login, " +
|
||||
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
|
||||
@@ -233,7 +233,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
|
||||
throw new Error("completionCriteria.stableForMs must be zero or greater");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_request_help");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=request-help)");
|
||||
const timeoutMs = args.timeoutMs ?? 300_000;
|
||||
const cmdArgs = [
|
||||
"request-help",
|
||||
@@ -285,10 +285,10 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_get_html",
|
||||
name: "inspect.html",
|
||||
description:
|
||||
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
|
||||
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
|
||||
"snapshot ref. Use only when browser_inspect observe/snapshot cannot answer the question.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
@@ -326,7 +326,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
|
||||
}
|
||||
requirePositive(args.maxBytes, "maxBytes");
|
||||
const sessionId = registry.resolve(args.session, "browser_get_html");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=html)");
|
||||
const cmdArgs = ["get-html", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
@@ -361,7 +361,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_console",
|
||||
name: "inspect.console",
|
||||
description:
|
||||
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
|
||||
"This is read-only and does not evaluate JavaScript.",
|
||||
@@ -448,7 +448,7 @@ export function registerPhaseOneSupportTools(
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_console");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=console)");
|
||||
const cmdArgs = ["console", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
if (args.includeStack === true) cmdArgs.push("--include-stack");
|
||||
@@ -493,7 +493,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_network",
|
||||
name: "inspect.network",
|
||||
description:
|
||||
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
|
||||
"and response headers and bodies are not captured.",
|
||||
@@ -552,7 +552,7 @@ export function registerPhaseOneSupportTools(
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_network");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=network)");
|
||||
const cmdArgs = ["network", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
|
||||
@@ -592,7 +592,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_window_resize",
|
||||
name: "assist.resize",
|
||||
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
|
||||
parameters: {
|
||||
width: {
|
||||
@@ -629,7 +629,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
|
||||
throw new Error("width and height must each be in the range 100..=7680");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_window_resize");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=resize)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
[
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { ToolDeps } from "./tools";
|
||||
const TAB_ID_REQUIRED = {
|
||||
type: "integer",
|
||||
required: true,
|
||||
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
|
||||
description: "Chrome tab id returned by browser_tabs list or create.",
|
||||
} as const;
|
||||
|
||||
interface RawTabInfo {
|
||||
@@ -29,7 +29,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_list",
|
||||
name: "tabs.list",
|
||||
description:
|
||||
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
|
||||
"hidden; use scope=user to find a user tab before borrowing it.",
|
||||
@@ -83,7 +83,7 @@ export function registerPhaseOneTabTools(
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_list");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=list)");
|
||||
const cmdArgs = ["tab", "list", "--session", sessionId];
|
||||
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
|
||||
@@ -118,7 +118,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_create",
|
||||
name: "tabs.create",
|
||||
description:
|
||||
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
|
||||
"set active=false to open it in the background.",
|
||||
@@ -148,7 +148,7 @@ export function registerPhaseOneTabTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url !== undefined) requireNonEmpty(args.url, "url");
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_create");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=create)");
|
||||
const cmdArgs = ["tab", "create", "--session", sessionId];
|
||||
if (args.url !== undefined) cmdArgs.push("--url", args.url);
|
||||
if (args.active === false) cmdArgs.push("--no-active");
|
||||
@@ -185,7 +185,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_close",
|
||||
name: "tabs.close",
|
||||
description: "Close a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -202,7 +202,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_close");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=close)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "close", String(args.tabId), "--session", sessionId],
|
||||
@@ -228,7 +228,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_select",
|
||||
name: "tabs.select",
|
||||
description: "Focus a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -249,7 +249,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_select");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=select)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "select", String(args.tabId), "--session", sessionId],
|
||||
@@ -275,10 +275,10 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_borrow",
|
||||
name: "tabs.borrow",
|
||||
description:
|
||||
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
|
||||
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
|
||||
"Return it with browser_tabs action=return; stopping the session also auto-returns it.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
@@ -302,7 +302,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=borrow)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "borrow", String(args.tabId), "--session", sessionId],
|
||||
@@ -339,7 +339,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_return",
|
||||
name: "tabs.return",
|
||||
description: "Return a borrowed tab to its original user window and tab-strip position.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -365,7 +365,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_return");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=return)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "return", String(args.tabId), "--session", sessionId],
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
*
|
||||
* Strict ownership boundary (the bsk daemon may be shared with other agents,
|
||||
* terminals, or dsh instances): the registry ONLY ever holds sessions created
|
||||
* by this plugin's browser_session_start. Tools cannot see or act on any
|
||||
* by this plugin's browser_session start action. Tools cannot see or act on any
|
||||
* other session — an explicit `session` argument naming a foreign id is an
|
||||
* error, the list tool shows owned sessions only, and stop/unload cleanup
|
||||
* can never touch a session this plugin did not create.
|
||||
@@ -48,7 +48,7 @@ export class SessionRegistry {
|
||||
if (this.sessions.size + this.pendingStarts >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
"stop one with browser_session action=stop before starting another",
|
||||
);
|
||||
}
|
||||
this.pendingStarts += 1;
|
||||
@@ -69,7 +69,7 @@ export class SessionRegistry {
|
||||
if (!this.sessions.has(session.sessionId) && this.sessions.size >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
"stop one with browser_session action=stop before starting another",
|
||||
);
|
||||
}
|
||||
this.sessions.set(session.sessionId, { ...session, owned: true });
|
||||
@@ -149,7 +149,7 @@ export class SessionRegistry {
|
||||
private foreignError(sessionId: string, toolName: string): Error {
|
||||
return new Error(
|
||||
`${toolName}: session "${sessionId}" does not belong to this plugin — only sessions ` +
|
||||
"created by browser_session_start are visible and operable here",
|
||||
"created by browser_session action=start are visible and operable here",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -168,7 +168,7 @@ export class SessionRegistry {
|
||||
const current = this.current();
|
||||
if (current === undefined) {
|
||||
throw new Error(
|
||||
`${toolName} needs a session but none is active — start one with browser_session_start`,
|
||||
`${toolName} needs a session but none is active — use browser_session action=start`,
|
||||
);
|
||||
}
|
||||
this.touch(current);
|
||||
@@ -184,10 +184,11 @@ export class SessionRegistry {
|
||||
explicit !== undefined && explicit.trim().length > 0 ? explicit : this.current();
|
||||
if (candidate === undefined) {
|
||||
throw new Error(
|
||||
"browser_session_stop needs a session but none is active — start one with browser_session_start",
|
||||
"browser_session action=stop needs a session but none is active — use action=start first",
|
||||
);
|
||||
}
|
||||
if (!this.isOwned(candidate)) throw this.foreignError(candidate, "browser_session_stop");
|
||||
if (!this.isOwned(candidate))
|
||||
throw this.foreignError(candidate, "browser_session(action=stop)");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
export const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
|
||||
"bsk session id to act on; must be one created by browser_session with action=start. " +
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
/**
|
||||
* Model-facing tool definitions. Each tool maps to one `bsk <cmd> --json`
|
||||
* invocation: spawn via the runner, parse the JSON envelope, and return a
|
||||
* canonical value whose `output.render` produces the model-facing text.
|
||||
* UI cards reuse the terminal style: the pending card is the command line,
|
||||
* the completed card carries the output.
|
||||
* Internal browser operation definitions shared by the six model-facing tools.
|
||||
* These definitions are never registered with `ctx.tools`: they provide the
|
||||
* action-level validation, execution, rendering, and UI presentation reused by
|
||||
* the domain tools without exposing one schema per operation to the model.
|
||||
*/
|
||||
|
||||
import { readFile, unlink } from "node:fs/promises";
|
||||
@@ -158,22 +157,15 @@ function abortError(): Error {
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Register the full browser tool suite; returns the combined unregister disposer. */
|
||||
export function registerTools(deps: ToolDeps): () => void {
|
||||
const { ctx, registry } = deps;
|
||||
// Track every registration without touching the cordis context itself
|
||||
// (contexts are fiber-owned; the lazy path registers from a listener fiber).
|
||||
const disposers: (() => void)[] = [];
|
||||
const register = (definition: ToolDefinition): (() => void) => {
|
||||
const dispose = ctx.tools.register(definition);
|
||||
// Test doubles occasionally return the registry map instead of a disposer.
|
||||
if (typeof dispose === "function") disposers.push(dispose);
|
||||
return dispose;
|
||||
};
|
||||
type DefinitionRegistrar = (definition: ToolDefinition) => void;
|
||||
|
||||
/** Define the private action handlers through an injected collector. */
|
||||
function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): void {
|
||||
const { registry } = deps;
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_start",
|
||||
name: "session.start",
|
||||
description:
|
||||
"Start a new browser session: opens an Agent Window in the connected browser and returns " +
|
||||
"its session id. The new session becomes the current session for subsequent browser_* calls. " +
|
||||
@@ -314,10 +306,10 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_stop",
|
||||
name: "session.stop",
|
||||
description:
|
||||
"Stop a browser session and close its Agent Window. Stops the given session, or the " +
|
||||
"current session when `session` is omitted. Only sessions created by browser_session_start " +
|
||||
"current session when `session` is omitted. Only plugin-created sessions " +
|
||||
"can be stopped — sessions owned by other programs sharing the bsk daemon are refused.",
|
||||
parameters: { session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -354,7 +346,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_list",
|
||||
name: "session.list",
|
||||
description:
|
||||
"List the browser sessions created by this plugin. Sessions owned by other programs on the " +
|
||||
"shared bsk daemon are not visible here. The session marked `current` is the one browser_* " +
|
||||
@@ -419,7 +411,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_navigate",
|
||||
name: "page.navigate",
|
||||
description:
|
||||
"Navigate the session's active tab to a URL and wait for a page lifecycle phase " +
|
||||
"(default: load). Returns the final URL after redirects.",
|
||||
@@ -461,7 +453,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url.trim().length === 0) throw new Error("url must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_navigate");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=navigate)");
|
||||
const cmdArgs = ["navigate", "--session", sessionId];
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
@@ -492,12 +484,12 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
const registerObservationTool = (kind: "snapshot" | "observe") => {
|
||||
const isSnapshot = kind === "snapshot";
|
||||
const name = isSnapshot ? "browser_snapshot" : "browser_observe";
|
||||
const name = isSnapshot ? "inspect.snapshot" : "inspect.observe";
|
||||
const description = isSnapshot
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
"probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -566,10 +558,10 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_click",
|
||||
name: "interact.click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
"last browser_inspect observation, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -607,7 +599,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
@@ -630,7 +622,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_fill",
|
||||
name: "interact.fill",
|
||||
description:
|
||||
"Fill an input / textarea / contenteditable element, clearing it first by default. " +
|
||||
"Target is a snapshot ref (@e3) or a CSS selector.",
|
||||
@@ -666,7 +658,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_fill");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=fill)");
|
||||
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
|
||||
if (args.noClear === true) cmdArgs.push("--no-clear");
|
||||
cmdArgs.push(args.target);
|
||||
@@ -687,7 +679,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_press",
|
||||
name: "interact.press",
|
||||
description:
|
||||
"Dispatch a keyboard key or combo (Enter, Escape, ArrowDown, Ctrl+A, …) to the session's " +
|
||||
"active tab, optionally focusing a target element first.",
|
||||
@@ -726,7 +718,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.key.trim().length === 0) throw new Error("key must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_press");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=press)");
|
||||
const cmdArgs = ["press", "--session", sessionId];
|
||||
if (args.target !== undefined) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
@@ -757,7 +749,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_screenshot",
|
||||
name: "inspect.screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
@@ -784,7 +776,11 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
attachmentId: { type: "string", required: true },
|
||||
mediaType: { type: "string", required: true, const: "image/png" },
|
||||
mediaType: {
|
||||
type: "string",
|
||||
required: true,
|
||||
enum: ["image/png", "image/jpeg", "image/webp", "image/gif"],
|
||||
},
|
||||
bytes: { type: "integer", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
@@ -818,7 +814,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_screenshot");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=screenshot)");
|
||||
const outPath = join(
|
||||
tmpdir(),
|
||||
`bsk-screenshot-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`,
|
||||
@@ -854,7 +850,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
? {
|
||||
image: {
|
||||
attachmentId: String(ref.attachmentId),
|
||||
mediaType: "image/png" as const,
|
||||
mediaType: ref.mediaType,
|
||||
bytes: ref.bytes,
|
||||
width: ref.width,
|
||||
height: ref.height,
|
||||
@@ -883,7 +879,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_emulate",
|
||||
name: "assist.emulate",
|
||||
description:
|
||||
"Apply (or clear) mobile device emulation on the session's active tab: viewport metrics, " +
|
||||
"user agent, and touch. Overrides are per-tab and not inherited by new tabs.",
|
||||
@@ -924,7 +920,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_emulate");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=emulate)");
|
||||
if (args.off === true) {
|
||||
if (args.device !== undefined || args.width !== undefined || args.height !== undefined) {
|
||||
throw new Error("off is mutually exclusive with device/width/height");
|
||||
@@ -973,7 +969,17 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
commandLine: (args) => cmdline(deps, args),
|
||||
presentTerminalResult,
|
||||
});
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the private action handlers without publishing them to the model. The
|
||||
* six browser tools dispatch through these handlers, preserving validation,
|
||||
* rendering, cancellation, ownership, UI presentation, and attachments.
|
||||
*/
|
||||
export function createBrowserOperationDefinitions(deps: ToolDeps): ToolDefinition[] {
|
||||
const definitions: ToolDefinition[] = [];
|
||||
defineBrowserOperations(deps, (definition) => {
|
||||
definitions.push(definition);
|
||||
});
|
||||
return definitions;
|
||||
}
|
||||
|
||||
+48
-22
@@ -1,17 +1,17 @@
|
||||
// @vitest-environment happy-dom
|
||||
// browser_screenshot keyed toolview: view-model derivation from frozen blocks,
|
||||
// browser_inspect keyed toolview: view-model derivation from frozen blocks,
|
||||
// image/path-only/running/error rendering, the registration key, and the
|
||||
// session-bound attachment loader.
|
||||
|
||||
import type { RunningToolCall, ToolResultNode } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
import { cleanup, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { apply } from "../../src/client/index";
|
||||
import {
|
||||
ScreenshotToolView,
|
||||
type ScreenshotToolViewProps,
|
||||
BrowserInspectToolView,
|
||||
type BrowserInspectToolViewProps,
|
||||
viewModelOf,
|
||||
} from "../../src/client/ScreenshotToolView";
|
||||
} from "../../src/client/BrowserInspectToolView";
|
||||
import { apply } from "../../src/client/index";
|
||||
|
||||
const ATTACHMENT = {
|
||||
attachmentId: "sha256:abc123",
|
||||
@@ -25,7 +25,7 @@ const ATTACHMENT = {
|
||||
function runningBlock(argsRaw: string): RunningToolCall {
|
||||
return {
|
||||
callId: "c1",
|
||||
name: "browser_screenshot",
|
||||
name: "browser_inspect",
|
||||
argsRaw,
|
||||
turn: 1,
|
||||
step: 1,
|
||||
@@ -41,7 +41,7 @@ function settledBlock(argsRaw: string, content: unknown[], isError = false): Too
|
||||
seq: 1,
|
||||
time: 0,
|
||||
callId: "c1",
|
||||
call: { name: "browser_screenshot", argsRaw },
|
||||
call: { name: "browser_inspect", argsRaw },
|
||||
callTime: 0,
|
||||
content,
|
||||
isError,
|
||||
@@ -58,14 +58,14 @@ afterEach(cleanup);
|
||||
|
||||
describe("viewModelOf", () => {
|
||||
it("derives the running model from the call frame", () => {
|
||||
const model = viewModelOf(runningBlock('{"session":"s1"}'));
|
||||
const model = viewModelOf(runningBlock('{"action":"screenshot","session":"s1"}'));
|
||||
expect(model.state).toBe("running");
|
||||
expect(model.command).toBe("bsk screenshot --session s1");
|
||||
expect(model.image).toBeNull();
|
||||
});
|
||||
|
||||
it("marks settled results carrying an image block", () => {
|
||||
const block = settledBlock("{}", [
|
||||
const block = settledBlock('{"action":"screenshot"}', [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
@@ -77,30 +77,56 @@ describe("viewModelOf", () => {
|
||||
});
|
||||
|
||||
it("keeps the path-only form image-free", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: PATH_TEXT }]));
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]),
|
||||
);
|
||||
expect(model.state).toBe("ok");
|
||||
expect(model.image).toBeNull();
|
||||
expect(model.summary).toBe(PATH_TEXT);
|
||||
});
|
||||
|
||||
it("marks error results", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: "Error: boom" }], true));
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"screenshot"}', [{ type: "text", text: "Error: boom" }], true),
|
||||
);
|
||||
expect(model.state).toBe("error");
|
||||
});
|
||||
|
||||
it("renders non-screenshot inspect actions as ordinary terminal calls", () => {
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"observe","session":"s1"}', [
|
||||
{ type: "text", text: "page observation" },
|
||||
]),
|
||||
);
|
||||
expect(model.title).toBe("Observe");
|
||||
expect(model.command).toBe("bsk observe --session s1");
|
||||
expect(model.image).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves action-specific arguments in diagnostic command cards", () => {
|
||||
const model = viewModelOf(
|
||||
runningBlock(
|
||||
'{"action":"console","session":"s1","tabId":7,"since":4,"limit":20,"maxTextChars":500,"includeStack":true}',
|
||||
),
|
||||
);
|
||||
expect(model.command).toBe(
|
||||
"bsk console --session s1 --tab-id 7 --since 4 --limit 20 --max-text-chars 500 --include-stack",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ScreenshotToolView", () => {
|
||||
const baseProps = { callId: "c1", toolName: "browser_screenshot", openFile: () => {} };
|
||||
describe("BrowserInspectToolView", () => {
|
||||
const baseProps = { callId: "c1", toolName: "browser_inspect", openFile: () => {} };
|
||||
const renderView = (
|
||||
block: ScreenshotToolViewProps["block"],
|
||||
loadImage: ScreenshotToolViewProps["loadImage"],
|
||||
block: BrowserInspectToolViewProps["block"],
|
||||
loadImage: BrowserInspectToolViewProps["loadImage"],
|
||||
) => {
|
||||
const props = { ...baseProps, block, loadImage } as unknown as ScreenshotToolViewProps;
|
||||
return render(<ScreenshotToolView {...props} />);
|
||||
const props = { ...baseProps, block, loadImage } as unknown as BrowserInspectToolViewProps;
|
||||
return render(<BrowserInspectToolView {...props} />);
|
||||
};
|
||||
|
||||
it("renders the screenshot image through the loader", async () => {
|
||||
const block = settledBlock("{}", [
|
||||
const block = settledBlock('{"action":"screenshot"}', [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
@@ -115,7 +141,7 @@ describe("ScreenshotToolView", () => {
|
||||
});
|
||||
|
||||
it("renders the path-only form without touching the loader", async () => {
|
||||
const block = settledBlock("{}", [{ type: "text", text: PATH_TEXT }]);
|
||||
const block = settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]);
|
||||
const loadImage = vi.fn(async () => "blob:unused");
|
||||
renderView(block, loadImage);
|
||||
screen.getByRole("button", { name: /screenshot/i }).click();
|
||||
@@ -125,14 +151,14 @@ describe("ScreenshotToolView", () => {
|
||||
|
||||
it("renders the running form without output", () => {
|
||||
const loadImage = vi.fn();
|
||||
renderView(runningBlock("{}"), loadImage);
|
||||
renderView(runningBlock('{"action":"screenshot"}'), loadImage);
|
||||
expect(screen.getByText("bsk screenshot --session (current)")).toBeTruthy();
|
||||
expect(loadImage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("client plugin registration", () => {
|
||||
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
|
||||
it("registers the browser_inspect toolview and the shell observation overlay", () => {
|
||||
const registrations: { name: string; key?: string; id?: string }[] = [];
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
@@ -150,7 +176,7 @@ describe("client plugin registration", () => {
|
||||
};
|
||||
apply(ctx as never);
|
||||
expect(registrations).toEqual([
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "tool.call.toolview", key: "browser_inspect" },
|
||||
{ name: "shell.overlay", id: "bsk-observation" },
|
||||
]);
|
||||
});
|
||||
@@ -73,15 +73,15 @@ describe("dispose cleanup ownership", () => {
|
||||
};
|
||||
apply(ctx as never, { maxSessions: 5, lazyTools: false }, { runnerFactory: () => runner });
|
||||
|
||||
const start = tools.get("browser_session_start");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
if (start === undefined || snapshot === undefined) throw new Error("tools not registered");
|
||||
await start.execute({}, makeExec());
|
||||
await start.execute({}, makeExec());
|
||||
const session = tools.get("browser_session");
|
||||
const inspect = tools.get("browser_inspect");
|
||||
if (session === undefined || inspect === undefined) throw new Error("tools not registered");
|
||||
await session.execute({ action: "start" }, makeExec());
|
||||
await session.execute({ action: "start" }, makeExec());
|
||||
// A foreign session cannot even be referenced any more.
|
||||
await expect(snapshot.execute({ session: "ext9" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
await expect(
|
||||
inspect.execute({ action: "snapshot", session: "ext9" }, makeExec()),
|
||||
).rejects.toThrow(/does not belong to this plugin/);
|
||||
|
||||
await Promise.all(disposers.map((dispose) => dispose()));
|
||||
|
||||
|
||||
@@ -211,19 +211,25 @@ describe("lazyTools wiring in apply()", () => {
|
||||
|
||||
it("lazyTools: false registers the suite at apply time", () => {
|
||||
const { tools } = applyHarness({ lazyTools: false });
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
expect(tools.has("browser_emulate")).toBe(true);
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_assist",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_page",
|
||||
"browser_session",
|
||||
"browser_tabs",
|
||||
]);
|
||||
});
|
||||
|
||||
it("lazyTools: true hides the suite until the skill fires", () => {
|
||||
const { tools, listeners } = applyHarness({ lazyTools: true });
|
||||
expect(tools.has("browser_session_start")).toBe(false);
|
||||
expect(tools.has("browser_session")).toBe(false);
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
expect(tools.has("browser_session")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// dsh 0.1 keys the client graph and Node import by `export const name`.
|
||||
// That string must stay identical to the published package name (and therefore
|
||||
// the client.cjs __ModuleLoader__ id, which tsdown reads from package.json).
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { name } from "../src/index";
|
||||
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8"),
|
||||
) as {
|
||||
name: string;
|
||||
dsh: { client: { external?: string[] } };
|
||||
};
|
||||
|
||||
describe("plugin identity", () => {
|
||||
it("uses the published package name as the Cordis plugin id", () => {
|
||||
expect(name).toBe("@wxg-prc-cpg/browser-skill-dsh-plugin");
|
||||
expect(name).toBe(pkg.name);
|
||||
});
|
||||
|
||||
it("declares client require()s so dsh arrives them before materialize", () => {
|
||||
expect(pkg.dsh.client.external).toEqual(
|
||||
expect.arrayContaining([
|
||||
"@deepseek-ai/dsh-client-ui-attachment",
|
||||
"@deepseek-ai/dsh-client-ui-primitives",
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -71,7 +71,9 @@ describe("SessionRegistry", () => {
|
||||
|
||||
it("resolve throws actionable guidance when no session exists", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
expect(() => registry.resolve(undefined, "browser_click")).toThrow(/browser_session_start/);
|
||||
expect(() => registry.resolve(undefined, "browser_interact(action=click)")).toThrow(
|
||||
/browser_session action=start/,
|
||||
);
|
||||
});
|
||||
|
||||
it("recency order survives remove of the current session", () => {
|
||||
|
||||
@@ -32,44 +32,25 @@ describe("registerBskSkill", () => {
|
||||
const content = String(skill.content);
|
||||
const mentionedTools = [...new Set(content.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? [])].sort();
|
||||
expect(mentionedTools).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
"browser_assist",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_page",
|
||||
"browser_session",
|
||||
"browser_tabs",
|
||||
]);
|
||||
expect(String(skill.description)).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/```(?:bash|sh|shell)\b/i);
|
||||
expect(content).not.toMatch(/--[a-z]/);
|
||||
expect(content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(content).toMatch(/All browser work\s+must use the injected tools directly/);
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||
expect(content.length).toBeGreaterThan(10_000);
|
||||
// Keep the lazily injected instructions inside a bounded prompt budget,
|
||||
// while the lower bound catches accidental truncation of the guidance.
|
||||
expect(content.length).toBeGreaterThan(3_000);
|
||||
expect(content.length).toBeLessThan(6_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// Source frontmatter is registration metadata and must not leak into the body.
|
||||
expect(content.startsWith("---")).toBe(false);
|
||||
@@ -144,7 +125,7 @@ describe("armAgentScopedBskSkill", () => {
|
||||
});
|
||||
|
||||
const after = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
||||
expect(after?.content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(after?.content).toMatch(/All browser work\s+must use the injected tools directly/);
|
||||
expect(after?.content).not.toMatch(/\bbsk\b/i);
|
||||
expect(after?.source).toBe("bundled");
|
||||
|
||||
|
||||
@@ -3,11 +3,12 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ToolDefinition, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { registerBrowserTools } from "../src/browser-tools";
|
||||
import { ObservationService } from "../src/observation";
|
||||
import { KeyedExecutor } from "../src/queue";
|
||||
import type { BskRunner, BskRunOptions, BskRunResult } from "../src/runner";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
import { type PluginConfig, registerTools } from "../src/tools";
|
||||
import type { PluginConfig } from "../src/tools";
|
||||
|
||||
const CONFIG: PluginConfig = {
|
||||
bskPath: "bsk",
|
||||
@@ -84,8 +85,66 @@ function fakeRunner(responses: Record<string, unknown>) {
|
||||
};
|
||||
}
|
||||
|
||||
const ACTION_ROUTES: Record<string, readonly [string, string]> = {
|
||||
"session.start": ["browser_session", "start"],
|
||||
"session.stop": ["browser_session", "stop"],
|
||||
"session.list": ["browser_session", "list"],
|
||||
"page.navigate": ["browser_page", "navigate"],
|
||||
"page.back": ["browser_page", "back"],
|
||||
"page.forward": ["browser_page", "forward"],
|
||||
"page.reload": ["browser_page", "reload"],
|
||||
"page.wait": ["browser_page", "wait"],
|
||||
"inspect.observe": ["browser_inspect", "observe"],
|
||||
"inspect.snapshot": ["browser_inspect", "snapshot"],
|
||||
"inspect.html": ["browser_inspect", "html"],
|
||||
"inspect.screenshot": ["browser_inspect", "screenshot"],
|
||||
"inspect.console": ["browser_inspect", "console"],
|
||||
"inspect.network": ["browser_inspect", "network"],
|
||||
"interact.click": ["browser_interact", "click"],
|
||||
"interact.hover": ["browser_interact", "hover"],
|
||||
"interact.fill": ["browser_interact", "fill"],
|
||||
"interact.select": ["browser_interact", "select"],
|
||||
"interact.press": ["browser_interact", "press"],
|
||||
"tabs.list": ["browser_tabs", "list"],
|
||||
"tabs.create": ["browser_tabs", "create"],
|
||||
"tabs.select": ["browser_tabs", "select"],
|
||||
"tabs.close": ["browser_tabs", "close"],
|
||||
"tabs.borrow": ["browser_tabs", "borrow"],
|
||||
"tabs.return": ["browser_tabs", "return"],
|
||||
"assist.resize": ["browser_assist", "resize"],
|
||||
"assist.emulate": ["browser_assist", "emulate"],
|
||||
"assist.request-help": ["browser_assist", "request-help"],
|
||||
};
|
||||
|
||||
class TestToolRegistry extends Map<string, ToolDefinition> {
|
||||
override get(name: string): ToolDefinition | undefined {
|
||||
const direct = super.get(name);
|
||||
if (direct !== undefined) return direct;
|
||||
const route = ACTION_ROUTES[name];
|
||||
if (route === undefined) return undefined;
|
||||
const [toolName, action] = route;
|
||||
const tool = super.get(toolName);
|
||||
if (tool === undefined) return undefined;
|
||||
const withAction = (args: unknown) => ({
|
||||
action,
|
||||
...(args as Record<string, unknown>),
|
||||
});
|
||||
return {
|
||||
...tool,
|
||||
output: {
|
||||
...tool.output,
|
||||
render: (args, value) => tool.output.render(withAction(args) as never, value),
|
||||
},
|
||||
execute: (args, exec) => tool.execute(withAction(args) as never, exec),
|
||||
isConcurrencySafe: (args) => tool.isConcurrencySafe?.(withAction(args) as never) === true,
|
||||
presentCall: (args) => tool.presentCall?.(withAction(args) as never),
|
||||
presentResult: (args, result) => tool.presentResult?.(withAction(args) as never, result),
|
||||
} as ToolDefinition;
|
||||
}
|
||||
}
|
||||
|
||||
function makeCtx(services: Record<string, unknown> = {}) {
|
||||
const tools = new Map<string, ToolDefinition>();
|
||||
const tools = new TestToolRegistry();
|
||||
const ctx = {
|
||||
tools: { register: (def: ToolDefinition) => tools.set(def.name, def) },
|
||||
get: (key: string) => services[key],
|
||||
@@ -118,7 +177,7 @@ function setup(
|
||||
const { runner, calls } = fakeRunner(responses);
|
||||
const registry = new SessionRegistry(maxSessions);
|
||||
const config = { ...CONFIG, maxSessions };
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner: runner as BskRunner,
|
||||
registry,
|
||||
@@ -130,56 +189,98 @@ function setup(
|
||||
}
|
||||
|
||||
async function startSession(tools: Map<string, ToolDefinition>, sessionId = "s1") {
|
||||
const tool = tools.get("browser_session_start");
|
||||
if (tool === undefined) throw new Error("browser_session_start not registered");
|
||||
const tool = tools.get("session.start");
|
||||
if (tool === undefined) throw new Error("session.start not registered");
|
||||
return tool.execute({}, makeExec()) as Promise<{ sessionId: string }>;
|
||||
}
|
||||
|
||||
const START_REPLY = (id: string) => ({ session_id: id, browser_instance_id: "chrome-1" });
|
||||
|
||||
const EXPECTED_ACTIONS = {
|
||||
browser_session: ["start", "stop", "list"],
|
||||
browser_page: ["navigate", "back", "forward", "reload", "wait"],
|
||||
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
|
||||
browser_interact: ["click", "hover", "fill", "select", "press"],
|
||||
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
|
||||
browser_assist: ["resize", "emulate", "request-help"],
|
||||
} as const;
|
||||
|
||||
describe("tool registration", () => {
|
||||
it("registers the full browser tool suite", () => {
|
||||
it("registers exactly six public schemas with the complete action contract", () => {
|
||||
const { tools } = setup({});
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
]);
|
||||
expect([...tools.keys()].sort()).toEqual(Object.keys(EXPECTED_ACTIONS).sort());
|
||||
for (const [name, actions] of Object.entries(EXPECTED_ACTIONS)) {
|
||||
const properties = tools.get(name)?.parameters.properties as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
expect(properties?.action).toMatchObject({ enum: [...actions] });
|
||||
}
|
||||
});
|
||||
|
||||
it("requires an action on every public tool", async () => {
|
||||
const { tools } = setup({});
|
||||
for (const name of Object.keys(EXPECTED_ACTIONS)) {
|
||||
await expect(tools.get(name)?.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("validates model args through defineTool (missing required param)", async () => {
|
||||
const { tools } = setup({});
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const navigate = tools.get("page.navigate");
|
||||
await expect(navigate?.execute({}, makeExec())).rejects.toThrow(/invalid arguments/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_start", () => {
|
||||
describe("action dispatch", () => {
|
||||
it("rejects actions outside the public contract", async () => {
|
||||
const { tools } = setup({});
|
||||
await expect(
|
||||
tools.get("browser_page")?.execute({ action: "unsupported" }, makeExec()),
|
||||
).rejects.toThrow(/invalid arguments|unsupported browser action/i);
|
||||
});
|
||||
|
||||
it("dispatches through the existing operation definitions", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
navigate: { tab_id: 7, url: "https://example.test/", reached: "load" },
|
||||
observe: SNAPSHOT_REPLY,
|
||||
fill: { tab_id: 7, value_length: 5 },
|
||||
"tab list": { tabs: [] },
|
||||
"window resize": { window_id: 9, width: 900, height: 700 },
|
||||
});
|
||||
await tools.get("browser_session")?.execute({ action: "start" }, makeExec());
|
||||
await tools
|
||||
.get("browser_page")
|
||||
?.execute({ action: "navigate", url: "https://example.test/" }, makeExec());
|
||||
await tools.get("browser_inspect")?.execute({ action: "observe" }, makeExec());
|
||||
await tools
|
||||
.get("browser_interact")
|
||||
?.execute({ action: "fill", target: "@e1", value: "hello" }, makeExec());
|
||||
await tools.get("browser_tabs")?.execute({ action: "list" }, makeExec());
|
||||
await tools
|
||||
.get("browser_assist")
|
||||
?.execute({ action: "resize", width: 900, height: 700 }, makeExec());
|
||||
|
||||
expect(registry.current()).toBe("s1");
|
||||
expect(calls.map(({ args }) => args)).toEqual([
|
||||
["session", "start"],
|
||||
["navigate", "--session", "s1", "https://example.test/"],
|
||||
["observe", "--session", "s1"],
|
||||
["fill", "--session", "s1", "--value", "hello", "@e1"],
|
||||
["tab", "list", "--session", "s1"],
|
||||
["window", "resize", "--session", "s1", "--width", "900", "--height", "700"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps action-specific validation in the private operation", async () => {
|
||||
const { tools } = setup({});
|
||||
await expect(
|
||||
tools.get("browser_page")?.execute({ action: "navigate" }, makeExec()),
|
||||
).rejects.toThrow(/url.*required|invalid arguments/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("session.start", () => {
|
||||
it("maps the start reply and tracks the session as current", async () => {
|
||||
const { tools, registry } = setup({ "session start": START_REPLY("s1") });
|
||||
const value = await startSession(tools);
|
||||
@@ -193,7 +294,7 @@ describe("browser_session_start", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
navigate: { tab_id: 7, url: "https://example.com", reached: "load" },
|
||||
});
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
const value = (await tool?.execute(
|
||||
{ width: 1280, height: 800, url: "https://example.com", device: "iphone-14", noFocus: true },
|
||||
makeExec(),
|
||||
@@ -218,7 +319,7 @@ describe("browser_session_start", () => {
|
||||
|
||||
it("requires width and height together", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
await expect(tool?.execute({ width: 1280 }, makeExec())).rejects.toThrow(/together/);
|
||||
});
|
||||
|
||||
@@ -228,7 +329,7 @@ describe("browser_session_start", () => {
|
||||
"session stop": {},
|
||||
});
|
||||
// navigate has no canned response -> non-zero exit -> error.
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
await expect(tool?.execute({ url: "https://example.com" }, makeExec())).rejects.toThrow(
|
||||
/navigate/,
|
||||
);
|
||||
@@ -246,7 +347,7 @@ describe("multi-session behavior", () => {
|
||||
it("operation tools default to the current session and echo it back", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const value = (await snapshot?.execute({}, makeExec())) as {
|
||||
session: string;
|
||||
refCount: number;
|
||||
@@ -262,7 +363,7 @@ describe("multi-session behavior", () => {
|
||||
});
|
||||
await startSession(tools, "s1");
|
||||
await startSession(tools, "s2");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const value = (await snapshot?.execute({ session: "s1" }, makeExec())) as {
|
||||
session: string;
|
||||
};
|
||||
@@ -275,7 +376,7 @@ describe("multi-session behavior", () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const callsBefore = calls.length;
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({ session: "other" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
@@ -284,8 +385,8 @@ describe("multi-session behavior", () => {
|
||||
|
||||
it("fails with guidance when no session exists", async () => {
|
||||
const { tools } = setup(responses);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/browser_session_start/);
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/browser_session action=start/);
|
||||
});
|
||||
|
||||
it("enforces the configured session cap before spawning", async () => {
|
||||
@@ -295,7 +396,7 @@ describe("multi-session behavior", () => {
|
||||
1,
|
||||
);
|
||||
await startSession(tools);
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/session limit/);
|
||||
// A rejected start must not spawn another bsk session (no leaked session).
|
||||
expect(calls.filter((c) => c.args.join(" ").startsWith("session start"))).toHaveLength(1);
|
||||
@@ -307,7 +408,7 @@ describe("multi-session behavior", () => {
|
||||
{},
|
||||
1,
|
||||
);
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
if (start === undefined) throw new Error("not registered");
|
||||
// Two truly concurrent starts against a cap of 1: exactly one may spawn.
|
||||
const outcomes = await Promise.allSettled([
|
||||
@@ -325,7 +426,7 @@ describe("multi-session behavior", () => {
|
||||
it("refuses to stop a session the plugin did not create", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
const stop = tools.get("session.stop");
|
||||
await expect(stop?.execute({ session: "ext7" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
@@ -334,14 +435,14 @@ describe("multi-session behavior", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_stop / list", () => {
|
||||
describe("session.stop / list", () => {
|
||||
it("stops the current session and clears the registry", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"session stop": {},
|
||||
});
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
const stop = tools.get("session.stop");
|
||||
const value = (await stop?.execute({}, makeExec())) as { stopped: string };
|
||||
expect(value.stopped).toBe("s1");
|
||||
expect(calls[1].args).toEqual(["session", "stop", "s1"]);
|
||||
@@ -354,7 +455,7 @@ describe("browser_session_stop / list", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
await startSession(tools);
|
||||
const list = tools.get("browser_session_list");
|
||||
const list = tools.get("session.list");
|
||||
const value = (await list?.execute({}, makeExec())) as {
|
||||
sessions: { sessionId: string; current: boolean }[];
|
||||
};
|
||||
@@ -377,10 +478,10 @@ describe("interaction tools", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("browser_click maps target, button, and click count", async () => {
|
||||
it("interact.click maps target, button, and click count", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
makeExec(),
|
||||
@@ -398,10 +499,10 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_fill passes value and --no-clear", async () => {
|
||||
it("interact.fill passes value and --no-clear", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const fill = tools.get("browser_fill");
|
||||
const fill = tools.get("interact.fill");
|
||||
await fill?.execute({ target: "e2", value: "hello", noClear: true }, makeExec());
|
||||
expect(calls[1].args).toEqual([
|
||||
"fill",
|
||||
@@ -414,20 +515,20 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_press picks --ref for snapshot refs and --selector otherwise", async () => {
|
||||
it("interact.press picks --ref for snapshot refs and --selector otherwise", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const press = tools.get("browser_press");
|
||||
const press = tools.get("interact.press");
|
||||
await press?.execute({ key: "Enter", target: "@e3" }, makeExec());
|
||||
expect(calls[1].args).toEqual(["press", "--session", "s1", "--ref", "@e3", "Enter"]);
|
||||
await press?.execute({ key: "Ctrl+A", target: "#field" }, makeExec());
|
||||
expect(calls[2].args).toEqual(["press", "--session", "s1", "--selector", "#field", "Ctrl+A"]);
|
||||
});
|
||||
|
||||
it("browser_navigate converts waitUntil and timeoutMs", async () => {
|
||||
it("page.navigate converts waitUntil and timeoutMs", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const navigate = tools.get("page.navigate");
|
||||
const value = (await navigate?.execute(
|
||||
{ url: "https://a.test", waitUntil: "networkidle", timeoutMs: 1500 },
|
||||
makeExec(),
|
||||
@@ -445,16 +546,16 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_emulate rejects off combined with overrides", async () => {
|
||||
it("assist.emulate rejects off combined with overrides", async () => {
|
||||
const { tools } = setup(responses);
|
||||
await startSession(tools);
|
||||
const emulate = tools.get("browser_emulate");
|
||||
const emulate = tools.get("assist.emulate");
|
||||
await expect(emulate?.execute({ off: true, device: "iphone-14" }, makeExec())).rejects.toThrow(
|
||||
/mutually exclusive/,
|
||||
);
|
||||
});
|
||||
|
||||
it("browser_hover and browser_select map their structured arguments", async () => {
|
||||
it("interact.hover and interact.select map their structured arguments", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
hover: { tab_id: 7, used_ref: "@e3", x: 12, y: 24 },
|
||||
@@ -468,7 +569,7 @@ describe("interaction tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
const hovered = (await tools
|
||||
.get("browser_hover")
|
||||
.get("interact.hover")
|
||||
?.execute(
|
||||
{ target: "@e3", modifiers: ["shift"], settleMs: 250, timeoutMs: 2000 },
|
||||
makeExec(),
|
||||
@@ -488,7 +589,7 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
|
||||
const selected = (await tools
|
||||
.get("browser_select")
|
||||
.get("interact.select")
|
||||
?.execute({ target: "@e4", values: ["a", "b"], tabId: 7 }, makeExec())) as {
|
||||
selectedValues: string[];
|
||||
selectedLabels: string[];
|
||||
@@ -546,25 +647,25 @@ describe("phase-one tab tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
|
||||
const listed = (await tools
|
||||
.get("browser_tab_list")
|
||||
?.execute({ scope: "agent" }, makeExec())) as { tabs: { tabId: number }[] };
|
||||
const listed = (await tools.get("tabs.list")?.execute({ scope: "agent" }, makeExec())) as {
|
||||
tabs: { tabId: number }[];
|
||||
};
|
||||
expect(listed.tabs[0].tabId).toBe(7);
|
||||
const created = (await tools
|
||||
.get("browser_tab_create")
|
||||
.get("tabs.create")
|
||||
?.execute({ url: "https://new.test", active: false, index: 1 }, makeExec())) as {
|
||||
tabId: number;
|
||||
};
|
||||
expect(created.tabId).toBe(8);
|
||||
await tools.get("browser_tab_close")?.execute({ tabId: 8 }, makeExec());
|
||||
await tools.get("browser_tab_select")?.execute({ tabId: 7 }, makeExec());
|
||||
const borrowed = (await tools
|
||||
.get("browser_tab_borrow")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { originalWindowId: number };
|
||||
await tools.get("tabs.close")?.execute({ tabId: 8 }, makeExec());
|
||||
await tools.get("tabs.select")?.execute({ tabId: 7 }, makeExec());
|
||||
const borrowed = (await tools.get("tabs.borrow")?.execute({ tabId: 20 }, makeExec())) as {
|
||||
originalWindowId: number;
|
||||
};
|
||||
expect(borrowed.originalWindowId).toBe(9);
|
||||
const returned = (await tools
|
||||
.get("browser_tab_return")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { fallback: boolean };
|
||||
const returned = (await tools.get("tabs.return")?.execute({ tabId: 20 }, makeExec())) as {
|
||||
fallback: boolean;
|
||||
};
|
||||
expect(returned.fallback).toBe(false);
|
||||
|
||||
expect(calls.slice(1).map((call) => call.args)).toEqual([
|
||||
@@ -614,15 +715,14 @@ describe("phase-one navigation tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
await tools
|
||||
.get("browser_navigate_back")
|
||||
.get("page.back")
|
||||
?.execute({ tabId: 7, waitUntil: "commit", timeoutMs: 1000 }, makeExec());
|
||||
await tools.get("browser_navigate_forward")?.execute({}, makeExec());
|
||||
await tools
|
||||
.get("browser_reload")
|
||||
?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
|
||||
const waited = (await tools
|
||||
.get("browser_wait_for_navigation")
|
||||
?.execute({ timeoutMs: 2500 }, makeExec())) as { reached: string; errorText: string };
|
||||
await tools.get("page.forward")?.execute({}, makeExec());
|
||||
await tools.get("page.reload")?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
|
||||
const waited = (await tools.get("page.wait")?.execute({ timeoutMs: 2500 }, makeExec())) as {
|
||||
reached: string;
|
||||
errorText: string;
|
||||
};
|
||||
expect(waited).toMatchObject({ reached: "timeout", errorText: "no navigation" });
|
||||
expect(calls[1].args).toEqual([
|
||||
"navigate-back",
|
||||
@@ -668,7 +768,7 @@ describe("phase-one support tools", () => {
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const value = (await tools.get("browser_request_help")?.execute(
|
||||
const value = (await tools.get("assist.request-help")?.execute(
|
||||
{
|
||||
prompt: "Please sign in",
|
||||
title: "Login required",
|
||||
@@ -736,17 +836,17 @@ describe("phase-one support tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
const html = (await tools
|
||||
.get("browser_get_html")
|
||||
.get("inspect.html")
|
||||
?.execute({ ref: "@e2", maxBytes: 2048 }, makeExec())) as { html: string };
|
||||
expect(html.html).toBe("<main>Hi</main>");
|
||||
const consoleValue = (await tools
|
||||
.get("browser_console")
|
||||
.get("inspect.console")
|
||||
?.execute({ since: 1, limit: 10, includeStack: true }, makeExec())) as {
|
||||
entries: { stackTrace: { functionName: string }[] }[];
|
||||
};
|
||||
expect(consoleValue.entries[0].stackTrace[0].functionName).toBe("run");
|
||||
const networkValue = (await tools
|
||||
.get("browser_network")
|
||||
.get("inspect.network")
|
||||
?.execute({ tabId: 7, maxTextChars: 512 }, makeExec())) as {
|
||||
entries: { mimeType: string }[];
|
||||
};
|
||||
@@ -764,7 +864,7 @@ describe("phase-one support tools", () => {
|
||||
"window resize": { window_id: 2, width: 1280, height: 800 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const resize = tools.get("browser_window_resize");
|
||||
const resize = tools.get("assist.resize");
|
||||
const value = (await resize?.execute({ width: 1280, height: 800 }, makeExec())) as {
|
||||
windowId: number;
|
||||
};
|
||||
@@ -785,7 +885,7 @@ describe("phase-one support tools", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_screenshot", () => {
|
||||
describe("inspect.screenshot", () => {
|
||||
function pngFile(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bsk-test-"));
|
||||
const path = join(dir, "shot.png");
|
||||
@@ -794,6 +894,15 @@ describe("browser_screenshot", () => {
|
||||
return path;
|
||||
}
|
||||
|
||||
function jpegFile(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bsk-test-"));
|
||||
const path = join(dir, "shot.png");
|
||||
// A JPEG SOI marker (FF D8 FF) even though the file is named ".png" —
|
||||
// mirrors captureVisibleTab returning JPEG on some Chromium builds.
|
||||
writeFileSync(path, Buffer.from([0xff, 0xd8, 0xff, 0xdb, 0, 1, 2, 3, 4]));
|
||||
return path;
|
||||
}
|
||||
|
||||
it("returns the PNG path when no attachment store is mounted", async () => {
|
||||
const path = pngFile();
|
||||
const { tools, calls } = setup({
|
||||
@@ -801,7 +910,7 @@ describe("browser_screenshot", () => {
|
||||
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 8 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({ ref: "@e1" }, makeExec())) as {
|
||||
path: string;
|
||||
image?: unknown;
|
||||
@@ -850,7 +959,7 @@ describe("browser_screenshot", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as {
|
||||
image?: { attachmentId: string; mediaType: string };
|
||||
};
|
||||
@@ -859,6 +968,50 @@ describe("browser_screenshot", () => {
|
||||
expect(rendered.map((block) => block.type)).toEqual(["text", "image"]);
|
||||
});
|
||||
|
||||
it("declares the real media type when capture bytes are JPEG, not a hard-coded PNG", async () => {
|
||||
const path = jpegFile();
|
||||
const services = {
|
||||
attachments: {
|
||||
imageLimits: {
|
||||
mediaTypes: ["image/png", "image/jpeg", "image/webp", "image/gif"],
|
||||
maxImageBytes: 10_000_000,
|
||||
maxMessageImageBytes: 10_000_000,
|
||||
},
|
||||
saveImage: async (input: { data: Uint8Array; mediaType: string; name?: string }) => ({
|
||||
attachmentId: "att-jpeg",
|
||||
mediaType: input.mediaType,
|
||||
bytes: input.data.byteLength,
|
||||
width: 800,
|
||||
height: 600,
|
||||
name: input.name,
|
||||
}),
|
||||
},
|
||||
llm: {
|
||||
resolveModelInfo: async () => ({ inputModalities: ["text", "image"] }),
|
||||
},
|
||||
};
|
||||
const { tools } = setup(
|
||||
{
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 9 },
|
||||
},
|
||||
services,
|
||||
);
|
||||
await startSession(tools);
|
||||
const exec = makeExec({
|
||||
agent: {
|
||||
session: { requestHeader: () => ({ config: { provider: "deepseek", model: "vl" } }) },
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as {
|
||||
image?: { attachmentId: string; mediaType: string };
|
||||
};
|
||||
// The reference media type must reflect the sniffed JPEG bytes, not "image/png".
|
||||
expect(value.image).toMatchObject({ attachmentId: "att-jpeg", mediaType: "image/jpeg" });
|
||||
});
|
||||
|
||||
it("stays path-only when the model route is text-only", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
@@ -884,7 +1037,7 @@ describe("browser_screenshot", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
expect(value.path).toBe(path);
|
||||
@@ -897,14 +1050,14 @@ describe("error and cancellation semantics", () => {
|
||||
"session start": START_REPLY("s1"),
|
||||
});
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/no canned response/);
|
||||
});
|
||||
|
||||
it("turns an aborted child into an AbortError", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const exec = makeExec({ signal: controller.signal });
|
||||
@@ -921,7 +1074,7 @@ describe("error and cancellation semantics", () => {
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
@@ -929,7 +1082,7 @@ describe("error and cancellation semantics", () => {
|
||||
observation: disabledObservation({ ctx, runner, registry }),
|
||||
queue: new KeyedExecutor(),
|
||||
});
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/BrowserSkill must be installed/);
|
||||
});
|
||||
});
|
||||
@@ -937,7 +1090,7 @@ describe("error and cancellation semantics", () => {
|
||||
describe("presentation", () => {
|
||||
it("presents calls as terminal cards with the bsk command line", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const view = click?.presentCall?.({ target: "@e1" }) as {
|
||||
card: string;
|
||||
title: string;
|
||||
@@ -948,7 +1101,7 @@ describe("presentation", () => {
|
||||
|
||||
it("presents results as terminal cards carrying the rendered text", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const view = click?.presentResult?.({ target: "@e1" }, {
|
||||
content: [{ type: "text", text: "clicked" }],
|
||||
isError: false,
|
||||
@@ -958,7 +1111,7 @@ describe("presentation", () => {
|
||||
|
||||
it("projects the text block of multi-block results (screenshot text + image)", () => {
|
||||
const { tools } = setup({});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const view = screenshot?.presentResult?.({}, {
|
||||
content: [
|
||||
{ type: "text", text: "[session s1] screenshot of tab 7" },
|
||||
@@ -1017,7 +1170,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
function setupWithRunner(runner: BskRunner, services: Record<string, unknown> = {}) {
|
||||
const { ctx, tools } = makeCtx(services);
|
||||
const registry = new SessionRegistry(5);
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
@@ -1052,7 +1205,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeDefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
@@ -1064,7 +1217,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
const { runner, calls } = outWritingRunner();
|
||||
const { tools } = setupWithRunner(runner);
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, makeExec())) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
@@ -1139,11 +1292,18 @@ describe("observation action instrumentation timing", () => {
|
||||
queue,
|
||||
options: { enabled: true, thumbnailIntervalMs: 1500, idleIntervalMs: 8000 },
|
||||
});
|
||||
registerTools({ ctx: ctx as never, runner, registry, config: CONFIG, observation, queue });
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
config: CONFIG,
|
||||
observation,
|
||||
queue,
|
||||
});
|
||||
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const navigate = tools.get("page.navigate");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const a = navigate?.execute({ url: "http://x" }, makeExec()) as Promise<unknown>;
|
||||
// A starts immediately (empty queue) and blocks on the gate.
|
||||
await vi.waitFor(() =>
|
||||
|
||||
@@ -6,27 +6,27 @@
|
||||
"disconnected": "Disconnected",
|
||||
"connecting": "Connecting…",
|
||||
"connected": "Connected",
|
||||
"version_skew": "Protocol mismatch",
|
||||
"version_skew": "Connected",
|
||||
"disabled": "Connection off"
|
||||
},
|
||||
"stateDetail": {
|
||||
"disconnected": "Open BrowserSkill to connect.",
|
||||
"connecting": "Connecting to BrowserSkill…",
|
||||
"connected": "Ready for automation tasks.",
|
||||
"version_skew": "Some features may be unavailable."
|
||||
"version_skew": "Protocol versions differ. Please upgrade."
|
||||
},
|
||||
"stateBadge": {
|
||||
"disconnected": "Standby",
|
||||
"connecting": "Igniting",
|
||||
"connected": "Ready",
|
||||
"version_skew": "Action needed",
|
||||
"version_skew": "Upgradable",
|
||||
"disabled": "Off"
|
||||
},
|
||||
"connectionToggleTitle": "BrowserSkill connection",
|
||||
"controlHintsToggleTitle": "Control hints",
|
||||
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
||||
"controlHintsInfoLabel": "About control hints",
|
||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, CLI protocol v{{cliProtocol}}. Protocol versions differ — please upgrade.",
|
||||
"upgradeAvailable": "Upgradable",
|
||||
"launcher": {
|
||||
"title": "Quick actions"
|
||||
|
||||
@@ -6,27 +6,27 @@
|
||||
"disconnected": "未连接",
|
||||
"connecting": "连接中…",
|
||||
"connected": "已连接",
|
||||
"version_skew": "协议不一致",
|
||||
"version_skew": "已连接",
|
||||
"disabled": "连接已关闭"
|
||||
},
|
||||
"stateDetail": {
|
||||
"disconnected": "请先打开 BrowserSkill。",
|
||||
"connecting": "正在连接 BrowserSkill…",
|
||||
"connected": "可接收自动化任务。",
|
||||
"version_skew": "部分能力可能不可用。"
|
||||
"version_skew": "协议版本不同,请及时升级。"
|
||||
},
|
||||
"stateBadge": {
|
||||
"disconnected": "Standby",
|
||||
"connecting": "Igniting",
|
||||
"connected": "Ready",
|
||||
"version_skew": "Action needed",
|
||||
"version_skew": "可升级",
|
||||
"disabled": "Off"
|
||||
},
|
||||
"connectionToggleTitle": "BrowserSkill 连接",
|
||||
"controlHintsToggleTitle": "控制提示",
|
||||
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
||||
"controlHintsInfoLabel": "控制提示说明",
|
||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},CLI 协议 v{{cliProtocol}}。协议版本不同,请及时升级。",
|
||||
"upgradeAvailable": "可升级",
|
||||
"launcher": {
|
||||
"title": "快捷功能"
|
||||
|
||||
@@ -298,6 +298,34 @@ describe("renderVom single-layer page", () => {
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("renders through deeply nested transparent wrappers without exhausting the call stack", () => {
|
||||
const wrapperCount = 10_000;
|
||||
const nodes = [node({ id: 1, role: "RootWebArea", name: "Doc" })];
|
||||
for (let index = 0; index < wrapperCount; index += 1) {
|
||||
nodes.push(
|
||||
node({
|
||||
id: index + 2,
|
||||
parentId: index + 1,
|
||||
role: "generic",
|
||||
}),
|
||||
);
|
||||
}
|
||||
nodes.push(
|
||||
node({
|
||||
id: wrapperCount + 2,
|
||||
parentId: wrapperCount + 1,
|
||||
role: "button",
|
||||
name: "Deep action",
|
||||
tag: "button",
|
||||
}),
|
||||
);
|
||||
|
||||
const out = renderVom(scene(nodes));
|
||||
|
||||
expect(out.text).toContain('@e1 button "Deep action"');
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: wrapperCount + 2 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
@@ -785,6 +813,55 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("北京市小客车指标调控管理信息系统]");
|
||||
});
|
||||
|
||||
it("derives duplicate-label context from indexed DOM ancestry without semantic proximity", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc", contextScopeId: "root" }),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "View",
|
||||
tag: "button",
|
||||
domParentId: 2,
|
||||
domAncestorIds: [2, 1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "View",
|
||||
tag: "button",
|
||||
domParentId: 4,
|
||||
domAncestorIds: [4, 1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "group",
|
||||
name: "Project Alpha",
|
||||
domParentId: 1,
|
||||
domAncestorIds: [1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 1,
|
||||
role: "group",
|
||||
name: "Project Beta",
|
||||
domParentId: 1,
|
||||
domAncestorIds: [1],
|
||||
contextScopeId: "root",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]');
|
||||
expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]');
|
||||
});
|
||||
|
||||
it("does not add low-value root context to duplicated brand links", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
@@ -833,6 +910,34 @@ describe("renderVom single-layer page", () => {
|
||||
});
|
||||
|
||||
describe("renderVom double-layer page", () => {
|
||||
it("collects blocking-layer descendants even when children precede parents in source order", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "button",
|
||||
name: "Close",
|
||||
tag: "button",
|
||||
paintOrder: 0,
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "Modal",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Close"');
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("renders a blocking modal first and folds the base page into an occluded summary", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
+134
-66
@@ -421,6 +421,41 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
||||
return children;
|
||||
}
|
||||
|
||||
const CONTEXT_LABEL_ROLES = new Set([
|
||||
"heading",
|
||||
"article",
|
||||
"section",
|
||||
"group",
|
||||
"region",
|
||||
"row",
|
||||
"form",
|
||||
]);
|
||||
|
||||
interface DomContextIndex {
|
||||
nonBoundaryByScopeParent: Map<string | undefined, Map<number, VomNode[]>>;
|
||||
sourceOrderById: Map<number, number>;
|
||||
}
|
||||
|
||||
function buildDomContextIndex(nodes: VomNode[]): DomContextIndex {
|
||||
const nonBoundaryByScopeParent = new Map<string | undefined, Map<number, VomNode[]>>();
|
||||
const sourceOrderById = new Map<number, number>();
|
||||
for (const [sourceOrder, node] of nodes.entries()) {
|
||||
sourceOrderById.set(node.id, sourceOrder);
|
||||
if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node)) || isContextBoundary(node)) continue;
|
||||
const parentId = node.domParentId;
|
||||
if (parentId === undefined || parentId === null) continue;
|
||||
let byParent = nonBoundaryByScopeParent.get(node.contextScopeId);
|
||||
if (!byParent) {
|
||||
byParent = new Map();
|
||||
nonBoundaryByScopeParent.set(node.contextScopeId, byParent);
|
||||
}
|
||||
const siblings = byParent.get(parentId) ?? [];
|
||||
siblings.push(node);
|
||||
byParent.set(parentId, siblings);
|
||||
}
|
||||
return { nonBoundaryByScopeParent, sourceOrderById };
|
||||
}
|
||||
|
||||
function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const node of nodes) {
|
||||
@@ -440,13 +475,7 @@ function needsHandleContext(nodeName: string, duplicateRefNames: Set<string>): b
|
||||
}
|
||||
|
||||
function contextLabel(node: VomNode, targetName: string): string | undefined {
|
||||
if (
|
||||
!["heading", "article", "section", "group", "region", "row", "form"].includes(
|
||||
normalizedRole(node),
|
||||
)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node))) return undefined;
|
||||
return weakContextText(
|
||||
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
|
||||
targetName,
|
||||
@@ -607,19 +636,45 @@ function collectDomContext(
|
||||
): ContextCandidate[] {
|
||||
const ancestorIds = node.domAncestorIds ?? [];
|
||||
if (ancestorIds.length === 0) return [];
|
||||
const ancestors = new Set(ancestorIds);
|
||||
const contexts: ContextCandidate[] = [];
|
||||
const ancestorRank = new Map<number, number>();
|
||||
for (const [rank, ancestorId] of ancestorIds.entries()) {
|
||||
if (!ancestorRank.has(ancestorId)) ancestorRank.set(ancestorId, rank);
|
||||
}
|
||||
|
||||
for (const candidate of state.nodesById.values()) {
|
||||
const candidates = new Map<number, VomNode>();
|
||||
const nonBoundaryByParent = state.domContextIndex.nonBoundaryByScopeParent.get(
|
||||
node.contextScopeId,
|
||||
);
|
||||
for (const ancestorId of ancestorRank.keys()) {
|
||||
const boundary = state.nodesById.get(ancestorId);
|
||||
const boundaryParentId = boundary?.domParentId;
|
||||
if (
|
||||
boundary &&
|
||||
sharesContextScope(node, boundary) &&
|
||||
isContextBoundary(boundary) &&
|
||||
boundaryParentId !== undefined &&
|
||||
boundaryParentId !== null &&
|
||||
ancestorRank.has(boundaryParentId)
|
||||
) {
|
||||
candidates.set(boundary.id, boundary);
|
||||
}
|
||||
for (const candidate of nonBoundaryByParent?.get(ancestorId) ?? []) {
|
||||
candidates.set(candidate.id, candidate);
|
||||
}
|
||||
}
|
||||
|
||||
const sourceOrderById = state.domContextIndex.sourceOrderById;
|
||||
const orderedCandidates = [...candidates.values()].sort(
|
||||
(a, b) => (sourceOrderById.get(a.id) ?? 0) - (sourceOrderById.get(b.id) ?? 0),
|
||||
);
|
||||
for (const candidate of orderedCandidates) {
|
||||
if (candidate.id === node.id) continue;
|
||||
if (!sharesContextScope(node, candidate)) continue;
|
||||
if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue;
|
||||
const parentId = candidate.domParentId;
|
||||
if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue;
|
||||
if (parentId === undefined || parentId === null) continue;
|
||||
const label = contextLabel(candidate, nodeName);
|
||||
if (!label) continue;
|
||||
const rank = ancestorIds.indexOf(parentId);
|
||||
contexts.push({ text: label, source: "dom", order: rank < 0 ? contexts.length : rank });
|
||||
contexts.push({ text: label, source: "dom", order: ancestorRank.get(parentId) ?? 0 });
|
||||
}
|
||||
|
||||
return contexts;
|
||||
@@ -734,6 +789,7 @@ interface RenderState {
|
||||
children: Map<number | null, VomNode[]>;
|
||||
parentMap: Map<number, number | null>;
|
||||
nodesById: Map<number, VomNode>;
|
||||
domContextIndex: DomContextIndex;
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
@@ -768,20 +824,25 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
|
||||
}
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
parentId: number | null,
|
||||
function pushRenderChildren(
|
||||
stack: Array<{ node: VomNode; depth: number }>,
|
||||
children: readonly VomNode[],
|
||||
depth: number,
|
||||
state: RenderState,
|
||||
): void {
|
||||
if (state.stopped) return;
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], depth });
|
||||
}
|
||||
}
|
||||
|
||||
const nodes = children.get(parentId) ?? [];
|
||||
for (const node of nodes) {
|
||||
if (state.stopped) return;
|
||||
function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
|
||||
const stack: Array<{ node: VomNode; depth: number }> = [];
|
||||
pushRenderChildren(stack, children.get(null) ?? [], 1);
|
||||
|
||||
while (stack.length > 0 && !state.stopped) {
|
||||
const { node, depth } = stack.pop() as { node: VomNode; depth: number };
|
||||
|
||||
if (!shouldRender(node)) {
|
||||
renderTree(children, node.id, depth, state);
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -798,7 +859,7 @@ function renderTree(
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
break;
|
||||
}
|
||||
|
||||
state.lines.push(line);
|
||||
@@ -823,7 +884,7 @@ function renderTree(
|
||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||
continue;
|
||||
}
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -848,12 +909,13 @@ function renderNodes(
|
||||
children,
|
||||
parentMap: buildParentMap(nodes),
|
||||
nodesById: new Map(nodes.map((node) => [node.id, node])),
|
||||
domContextIndex: buildDomContextIndex(nodes),
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
};
|
||||
|
||||
renderTree(children, null, 1, state);
|
||||
renderTree(children, state);
|
||||
|
||||
return state;
|
||||
}
|
||||
@@ -922,23 +984,33 @@ function paintLineageByFrame(
|
||||
return lineage;
|
||||
}
|
||||
|
||||
function cachedPaintLineageByFrame(
|
||||
node: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
cache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): Map<string | undefined, VomNode> {
|
||||
const cached = cache.get(node.id);
|
||||
if (cached) return cached;
|
||||
const lineage = paintLineageByFrame(node, parentMap, nodesById);
|
||||
cache.set(node.id, lineage);
|
||||
return lineage;
|
||||
}
|
||||
|
||||
/** Resolve both nodes to the nearest document where their paint order is comparable. */
|
||||
function comparablePaintNodes(
|
||||
target: VomNode,
|
||||
blocker: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
lineageCache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): { target: VomNode; blocker: VomNode } | null {
|
||||
if (target.frameId === blocker.frameId) return { target, blocker };
|
||||
const targetLineage = paintLineageByFrame(target, parentMap, nodesById);
|
||||
let blockerInFrame: VomNode | undefined = blocker;
|
||||
let guard = 0;
|
||||
while (blockerInFrame && guard <= parentMap.size) {
|
||||
const targetInFrame = targetLineage.get(blockerInFrame.frameId);
|
||||
const targetLineage = cachedPaintLineageByFrame(target, parentMap, nodesById, lineageCache);
|
||||
const blockerLineage = cachedPaintLineageByFrame(blocker, parentMap, nodesById, lineageCache);
|
||||
for (const [frameId, blockerInFrame] of blockerLineage) {
|
||||
const targetInFrame = targetLineage.get(frameId);
|
||||
if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame };
|
||||
const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null;
|
||||
blockerInFrame = parentId === null ? undefined : nodesById.get(parentId);
|
||||
guard += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -949,12 +1021,13 @@ function isBlockedByRegion(
|
||||
parentMap: Map<number, number | null>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
viewportCoverage: number,
|
||||
lineageCache: Map<number, Map<string | undefined, VomNode>>,
|
||||
): boolean {
|
||||
if (target.id === blocker.id) return false;
|
||||
if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
|
||||
if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
|
||||
if (!target.rect || !blocker.rect) return false;
|
||||
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById);
|
||||
const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById, lineageCache);
|
||||
if (!paintNodes) return false;
|
||||
|
||||
const points = interactionPoints(target.rect);
|
||||
@@ -979,17 +1052,22 @@ function isBlockedByRegion(
|
||||
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
|
||||
}
|
||||
|
||||
function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<number> {
|
||||
function collectDescendants(nodes: VomNode[], roots: Set<number>): Set<number> {
|
||||
const children = new Map<number, number[]>();
|
||||
for (const node of nodes) {
|
||||
if (node.parentId === null) continue;
|
||||
const siblings = children.get(node.parentId) ?? [];
|
||||
siblings.push(node.id);
|
||||
children.set(node.parentId, siblings);
|
||||
}
|
||||
|
||||
const included = new Set(roots);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
const queue = [...roots];
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
for (const childId of children.get(queue[index]) ?? []) {
|
||||
if (included.has(childId)) continue;
|
||||
included.add(childId);
|
||||
queue.push(childId);
|
||||
}
|
||||
}
|
||||
return included;
|
||||
@@ -998,6 +1076,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<numb
|
||||
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
const parentMap = buildParentMap(nodes);
|
||||
const nodesById = new Map(nodes.map((node) => [node.id, node]));
|
||||
const lineageCache = new Map<number, Map<string | undefined, VomNode>>();
|
||||
const candidates = nodes
|
||||
.filter(isPositionedRegionCandidate)
|
||||
.map((node) => {
|
||||
@@ -1014,40 +1093,29 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
for (const target of nodes) {
|
||||
if (!isVomReferenceNode(target)) continue;
|
||||
const blocked = candidates.some((candidate) =>
|
||||
isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage),
|
||||
isBlockedByRegion(
|
||||
target,
|
||||
candidate.node,
|
||||
parentMap,
|
||||
nodesById,
|
||||
candidate.viewportCoverage,
|
||||
lineageCache,
|
||||
),
|
||||
);
|
||||
if (blocked) blockedRoots.add(target.id);
|
||||
}
|
||||
|
||||
if (blockedRoots.size === 0) return nodes;
|
||||
const blocked = collectDescendantsOfIds(nodes, blockedRoots);
|
||||
const blocked = collectDescendants(nodes, blockedRoots);
|
||||
return nodes.filter((node) => !blocked.has(node.id));
|
||||
}
|
||||
|
||||
function collectDescendantsOfMembers(nodes: VomNode[], members: Set<number>): Set<number> {
|
||||
const included = new Set(members);
|
||||
let changed = true;
|
||||
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return included;
|
||||
}
|
||||
|
||||
function countRenderable(nodes: VomNode[]): number {
|
||||
return nodes.filter(shouldRender).length;
|
||||
}
|
||||
|
||||
function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult {
|
||||
const included = collectDescendantsOfMembers(scene.nodes, layer.members);
|
||||
const included = collectDescendants(scene.nodes, layer.members);
|
||||
const visibleNodes = scene.nodes.filter((node) => included.has(node.id));
|
||||
const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id)));
|
||||
const header = [
|
||||
|
||||
+106
-282
@@ -1,327 +1,151 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## Recover without wandering
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
Reference in New Issue
Block a user