mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
test: add extensible browser evaluation harness
This commit is contained in:
drakezhang
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "diagnostics",
|
||||
"title": "Console and network inspection",
|
||||
"order": 50,
|
||||
"suite": "core",
|
||||
"tags": ["console", "network", "diagnostics"],
|
||||
"fixture": { "startPath": "/diagnostics" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Use browser diagnostics to inspect the page console and recent network activity. Report the console marker and the token returned by the ping request, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。使用浏览器诊断能力检查页面 console 和最近的 network 活动，报告 console marker 以及 ping 请求返回的 token，最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"inspect.console",
|
||||
"inspect.network"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "diagnostic ping completed",
|
||||
"type": "network.ping",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported console marker", "includes": "CONSOLE-61" },
|
||||
{ "label": "reported network token", "includes": "NETWORK-73" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "console inspection was executed", "key": "consoleRead" },
|
||||
{ "label": "network inspection was executed", "key": "networkRead" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "console", "evidence": "consoleRead" },
|
||||
{ "action": "network", "evidence": "networkRead" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "form-controls",
|
||||
"title": "Form, select, keyboard, and screenshot",
|
||||
"order": 20,
|
||||
"suite": "core",
|
||||
"tags": ["form", "keyboard", "screenshot", "select"],
|
||||
"fixture": { "startPath": "/form" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Fill Text input with \"agent-parity\" and Textarea with \"BrowserSkill works\". Select the visible option \"Two\" from Dropdown. Capture a screenshot of the form, focus Submit, and press Enter to submit. Confirm that the result says \"Received!\", then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把 Text input 填成“agent-parity”，Textarea 填成“BrowserSkill works”，Dropdown 选择可见选项“Two”。截取表单截图，然后聚焦 Submit 并按 Enter 提交。确认结果页出现“Received!”，最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.fill",
|
||||
"interact.select",
|
||||
"interact.press",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Enter was pressed on Submit",
|
||||
"type": "form.enter_pressed",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "form values were submitted",
|
||||
"type": "form.submitted",
|
||||
"where": {
|
||||
"data.text": "agent-parity",
|
||||
"data.notes": "BrowserSkill works",
|
||||
"data.choice": "two"
|
||||
},
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "result page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/result" },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "confirmed result", "includes": "Received!" }],
|
||||
"adapter": [
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#text-input",
|
||||
"value": "agent-parity"
|
||||
},
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#notes",
|
||||
"value": "BrowserSkill works"
|
||||
},
|
||||
{ "action": "select", "selector": "#choice", "values": ["two"] },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "form.png",
|
||||
"evidence": "screenshotCreated"
|
||||
},
|
||||
{ "action": "press", "key": "Enter", "selector": "#submit" },
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "page.shown",
|
||||
"where": { "path": "/result" }
|
||||
},
|
||||
{ "action": "wait", "duration": "200ms" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "hover-inspect",
|
||||
"title": "Hover, local HTML, and screenshot",
|
||||
"order": 30,
|
||||
"suite": "core",
|
||||
"tags": ["hover", "html", "screenshot"],
|
||||
"fixture": { "startPath": "/hover-inspect" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Hover over Account, then choose Profile beta from the revealed menu. Inspect the local HTML around the Inspection card and capture a screenshot. Report both marker codes you find, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把鼠标悬停在“Account”上，然后点击展开菜单中的“Profile beta”。检查 Inspection card 附近的局部 HTML，并截取页面截图。报告找到的两个标记代码，最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.hover",
|
||||
"interact.click",
|
||||
"inspect.html",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Account menu was hovered",
|
||||
"type": "hover.menu_opened",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "Profile beta was selected",
|
||||
"type": "hover.profile_clicked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported hover marker", "includes": "HOVER-17" },
|
||||
{ "label": "reported HTML marker", "includes": "INSPECT-27" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "HTML inspection was executed", "key": "htmlInspected" },
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "hover", "selector": "#account-menu" },
|
||||
{ "action": "click", "selector": "#profile-link" },
|
||||
{ "action": "html", "evidence": "htmlInspected" },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "hover-inspect.png",
|
||||
"evidence": "screenshotCreated"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "mobile-emulation",
|
||||
"title": "Mobile emulation",
|
||||
"order": 60,
|
||||
"suite": "core",
|
||||
"tags": ["emulation", "mobile", "viewport"],
|
||||
"fixture": { "startPath": "/responsive" },
|
||||
"prompts": {
|
||||
"en": "Open {url}, emulate an iPhone 14, and report the viewport marker shown on the page. Close the browser session when finished.",
|
||||
"zh-CN": "打开 {url}，模拟 iPhone 14，并报告页面显示的 viewport marker。完成后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": ["session.start", "session.stop", "page.navigate", "assist.emulate"],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "iPhone-sized viewport was observed",
|
||||
"type": "viewport.sample",
|
||||
"where": { "data.width": 390, "data.height": 844 },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported viewport marker", "includes": "VIEWPORT-84" }],
|
||||
"adapter": [
|
||||
{ "label": "device emulation was executed", "key": "emulatedDevice" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "resize", "width": 900, "height": 700 },
|
||||
{
|
||||
"action": "emulate",
|
||||
"device": "iphone-14",
|
||||
"evidence": "emulatedDevice"
|
||||
},
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "html" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "navigation-history",
|
||||
"title": "Navigation, history, and reload",
|
||||
"order": 10,
|
||||
"suite": "core",
|
||||
"tags": ["navigation", "history", "reload"],
|
||||
"fixture": { "startPath": "/navigation/start" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Read the main heading, follow the Details link, and report the code shown there. Then go back, go forward, reload the page, and close the browser session. Use only the browser capability for page operations.",
|
||||
"zh-CN": "打开 {url}。读取页面主标题，点击“详情”，并报告详情页上的代码。然后依次后退、前进、刷新页面，最后关闭浏览器会话。页面操作只使用浏览器能力。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"page.back",
|
||||
"page.forward",
|
||||
"page.reload",
|
||||
"interact.click"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "details link was clicked",
|
||||
"type": "navigation.detail.clicked",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "back returned to the start page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/start" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "forward returned to the detail page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/detail" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "detail page was reloaded",
|
||||
"type": "page.shown",
|
||||
"where": {
|
||||
"data.path": "/navigation/detail",
|
||||
"data.navigationType": "reload"
|
||||
},
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported detail code", "includes": "NAV-42" }],
|
||||
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "observe" },
|
||||
{ "action": "snapshot" },
|
||||
{ "action": "click", "selector": "#details-link" },
|
||||
{ "action": "html" },
|
||||
{ "action": "back" },
|
||||
{ "action": "forward" },
|
||||
{ "action": "reload" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "tabs",
|
||||
"title": "Create, select, and close a tab",
|
||||
"order": 40,
|
||||
"suite": "core",
|
||||
"tags": ["tabs", "ownership", "lifecycle"],
|
||||
"fixture": { "startPath": "/tabs" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Create a new browser tab at {baseUrl}/tabs/child?run={runId}, read its token, click \"Mark child visited\", close the child tab, return to the original tab, and close the browser session.",
|
||||
"zh-CN": "打开 {url}。新建一个浏览器标签页并访问 {baseUrl}/tabs/child?run={runId}，读取其中的 token，点击“Mark child visited”，关闭子标签页，回到原标签页，最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.click",
|
||||
"tabs.create",
|
||||
"tabs.select",
|
||||
"tabs.close"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "child tab page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/tabs/child" },
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "child tab action ran",
|
||||
"type": "tabs.child_marked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported child token", "includes": "TAB-29" }],
|
||||
"adapter": [
|
||||
{ "label": "child tab was closed", "key": "childTabClosed" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate", "saveAs": "original" },
|
||||
{ "action": "tab-list" },
|
||||
{
|
||||
"action": "tab-create",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"saveAs": "child"
|
||||
},
|
||||
{
|
||||
"action": "navigate",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{
|
||||
"action": "click",
|
||||
"selector": "#mark-child",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{ "action": "html", "tabId": "{child.tab_id}" },
|
||||
{
|
||||
"action": "tab-close",
|
||||
"tabId": "{child.tab_id}",
|
||||
"evidence": "childTabClosed"
|
||||
},
|
||||
{ "action": "tab-select", "tabId": "{original.tab_id}" }
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user