agent-radar
7be535059e
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s
896 lines
33 KiB
JavaScript
896 lines
33 KiB
JavaScript
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import { execFile, execFileSync } from 'node:child_process';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join, relative, resolve } from 'node:path';
|
|
|
|
import { AiSdkBackend } from '../../packages/runtime/dist/ai-sdk-backend.js';
|
|
import { buildComputerUseTools } from '../../packages/runtime/dist/computer-use-tools.js';
|
|
import { getAIModel } from '../../packages/runtime/dist/model-factory.js';
|
|
import { createMakaCuBackend } from '../../packages/computer-use/dist/index.js';
|
|
import { createDirectRuntimeTurnLedger } from './direct-runtime-ledger.mjs';
|
|
import { requireComputerUseLabRoot } from './lab-root.mjs';
|
|
import { sanitizeCuDirectReport } from './report-sanitize.mjs';
|
|
|
|
const repoRoot = new URL('../..', import.meta.url).pathname;
|
|
const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/maka-cu');
|
|
const labRoot = requireComputerUseLabRoot();
|
|
const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
|
|
const statePath = join(labRoot, 'test-app/runtime/state.json');
|
|
const fixturePID = Number(process.env.MAKA_CU_AX_MODEL_FIXTURE_PID);
|
|
const inputAgeProbe = process.env.MAKA_CU_AX_MODEL_INPUT_AGE_PROBE;
|
|
const temporaryDirectory = process.env.MAKA_CU_AX_MODEL_TEMP_DIR;
|
|
const reportPath = process.env.MAKA_CU_AX_MODEL_REPORT;
|
|
const provider = process.env.MAKA_CU_MODEL_PROVIDER ?? 'openai';
|
|
const baseUrl =
|
|
process.env.MAKA_CU_MODEL_BASE_URL ??
|
|
(provider === 'anthropic' ? 'http://127.0.0.1:8537' : 'http://127.0.0.1:8538/v1');
|
|
const modelId =
|
|
process.env.MAKA_CU_MODEL_ID ?? (provider === 'anthropic' ? 'claude-sonnet-4-6' : 'gpt-5.6-sol');
|
|
const apiKey =
|
|
process.env.MAKA_CU_MODEL_API_KEY ?? (provider === 'anthropic' ? 'coproxy' : 'bridge-managed');
|
|
const scenario = process.env.MAKA_CU_AX_MODEL_SCENARIO ?? 'set-value';
|
|
const targetValue = 'model-real-ax';
|
|
// The digest the manifest pins, rather than a copy of it: a hardcoded digest
|
|
// here goes stale the first time the executor is rebuilt, and the failure it
|
|
// produces is "refusing to spawn an unrecognized binary", which reads like a
|
|
// tampering alarm rather than like a stale constant.
|
|
const pinnedExecutorSha256 = JSON.parse(
|
|
await readFile(join(repoRoot, 'apps', 'desktop', 'bundled-tools.json'), 'utf8'),
|
|
).makaCu.binarySha256;
|
|
const expectedBinarySha256 = process.env.MAKA_CU_AX_MODEL_EXPECTED_SHA256 ?? pinnedExecutorSha256;
|
|
const resolvedTemporaryDirectory = resolve(temporaryDirectory ?? '');
|
|
const relativeTemporaryDirectory = relative(resolve(tmpdir()), resolvedTemporaryDirectory);
|
|
|
|
if (
|
|
!Number.isInteger(fixturePID) ||
|
|
fixturePID <= 0 ||
|
|
!inputAgeProbe ||
|
|
!temporaryDirectory ||
|
|
!reportPath ||
|
|
relativeTemporaryDirectory.startsWith('..') ||
|
|
relativeTemporaryDirectory === ''
|
|
) {
|
|
throw new Error('real AX model E2E requires launcher-owned inputs');
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const traces = [];
|
|
const actions = [];
|
|
const messages = [];
|
|
const telemetry = [];
|
|
const semanticOutcomes = [];
|
|
let freshAxValue;
|
|
let injectedIntervention = false;
|
|
let restartRequested = false;
|
|
let staleRecoveryObserved = false;
|
|
let ambiguousRecoveryObserved = false;
|
|
let semanticStructureChangeRejected = false;
|
|
let ambiguityFixtureMutated = false;
|
|
let nextId = 0;
|
|
let now = Date.now();
|
|
let totalActionAttempts = 0;
|
|
const actionAttemptCounts = new Map();
|
|
const fixtureInstances = new Map();
|
|
const observationTargets = new Map();
|
|
const generatedAt = new Date().toISOString();
|
|
const gitRevision = execFileSync('git', ['rev-parse', 'HEAD'], {
|
|
cwd: repoRoot,
|
|
encoding: 'utf8',
|
|
}).trim();
|
|
const reportLineage = {
|
|
runId: randomUUID(),
|
|
gitRevision,
|
|
generatedAt,
|
|
contentLineage: {
|
|
generator: 'scripts/computer-use/real-ax-harness.mjs',
|
|
gitRevision,
|
|
generatedAt,
|
|
},
|
|
};
|
|
|
|
function registerFixtureInstance(app, windowId) {
|
|
const windowIds = fixtureInstances.get(app.pid) ?? new Set();
|
|
windowIds.add(windowId);
|
|
fixtureInstances.set(app.pid, windowIds);
|
|
}
|
|
|
|
function parseObservation(text) {
|
|
if (typeof text !== 'string') return undefined;
|
|
const candidates = [text];
|
|
for (const markerText of ['Fresh observation:\n', 'Fresh observation: ']) {
|
|
const marker = text.lastIndexOf(markerText);
|
|
if (marker >= 0) candidates.push(text.slice(marker + markerText.length));
|
|
}
|
|
for (const candidate of candidates.reverse()) {
|
|
try {
|
|
const value = JSON.parse(candidate);
|
|
if (
|
|
value &&
|
|
typeof value === 'object' &&
|
|
typeof value.observation_id === 'string' &&
|
|
Number.isInteger(value.pid) &&
|
|
Number.isInteger(value.window_id)
|
|
) {
|
|
return value;
|
|
}
|
|
} catch {
|
|
// Try the next model-visible JSON projection.
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function actionBudgetFor(currentScenario) {
|
|
const mutation =
|
|
currentScenario === 'ax-click' || currentScenario === 'ambiguity'
|
|
? 'click_element'
|
|
: 'set_value';
|
|
return {
|
|
total: 8,
|
|
counts: {
|
|
list_apps: 2,
|
|
observe: 4,
|
|
wait: 2,
|
|
[mutation]:
|
|
currentScenario === 'intervention-recovery' || currentScenario === 'restart-recovery'
|
|
? 2
|
|
: 1,
|
|
...(currentScenario === 'ax-multi-step' ? { set_value: 1, click_element: 1 } : {}),
|
|
},
|
|
};
|
|
}
|
|
|
|
function qualificationScenarioFor(currentScenario) {
|
|
return {
|
|
expectedFailures:
|
|
currentScenario === 'restart-recovery'
|
|
? [{ action: 'set_value', error: 'target_missing' }]
|
|
: currentScenario === 'ambiguity'
|
|
? [
|
|
{ action: 'click_element', error: 'stale_frame' },
|
|
{ action: 'click_element', error: 'target_changed' },
|
|
]
|
|
: [],
|
|
};
|
|
}
|
|
|
|
const qualificationScenario = qualificationScenarioFor(scenario);
|
|
|
|
async function physicalInputRecentlyActive() {
|
|
const output = await new Promise((resolvePromise, reject) => {
|
|
execFile(
|
|
inputAgeProbe,
|
|
[],
|
|
{
|
|
encoding: 'utf8',
|
|
timeout: 1_000,
|
|
},
|
|
(error, stdout, stderr) => {
|
|
if (error) reject(new Error(stderr || error.message));
|
|
else resolvePromise(stdout.trim());
|
|
},
|
|
);
|
|
});
|
|
const ageSeconds = Number(output);
|
|
if (!Number.isFinite(ageSeconds)) {
|
|
throw new Error(`invalid physical input age: ${output}`);
|
|
}
|
|
return ageSeconds < 1;
|
|
}
|
|
|
|
const backend = createMakaCuBackend({
|
|
binaryPath,
|
|
expectedBinarySha256,
|
|
timeoutMs: 10_000,
|
|
physicalInputRecentlyActive,
|
|
allowCompatibilityInputDispatch: false,
|
|
onTrace(event) {
|
|
traces.push(event);
|
|
},
|
|
});
|
|
|
|
const appSummaries = await backend.listApps(new AbortController().signal);
|
|
const fixtureMatches = appSummaries.filter(
|
|
(app) =>
|
|
app.pid === fixturePID &&
|
|
(app.name === 'Codex CUA Lab' ||
|
|
app.windows?.some((window) => window.title === 'Codex CUA Lab')),
|
|
);
|
|
if (fixtureMatches.length !== 1) {
|
|
throw new Error(`expected one exact fixture app, got ${fixtureMatches.length}`);
|
|
}
|
|
let fixture = fixtureMatches[0];
|
|
const windows = fixture.windows?.filter((window) => window.title === 'Codex CUA Lab') ?? [];
|
|
if (windows.length !== 1) {
|
|
throw new Error(`expected one exact fixture window, got ${windows.length}`);
|
|
}
|
|
let fixtureWindowId = windows[0].windowId;
|
|
registerFixtureInstance(fixture, fixtureWindowId);
|
|
|
|
const instrumentedBackend = {
|
|
...backend,
|
|
async observeApp(input, signal, context) {
|
|
const observation = await backend.observeApp(input, signal, context);
|
|
if (
|
|
scenario === 'ambiguity' &&
|
|
context.sessionId === 'real-ax-model-e2e' &&
|
|
!ambiguityFixtureMutated
|
|
) {
|
|
ambiguityFixtureMutated = true;
|
|
const setupContext = {
|
|
sessionId: 'real-ax-model-ambiguity-setup',
|
|
turnId: 'setup',
|
|
toolCallId: 'fixture-mutate-ambiguity',
|
|
};
|
|
const setupObservation = await backend.observeApp(
|
|
{
|
|
app: fixture.appId,
|
|
windowId: fixtureWindowId,
|
|
includeScreenshot: true,
|
|
},
|
|
signal,
|
|
setupContext,
|
|
);
|
|
const mutation = setupObservation.elements.find(
|
|
(element) => element.label === 'CUA Lab Duplicate Stale Target',
|
|
);
|
|
if (!mutation) throw new Error('ambiguity mutation control is missing');
|
|
const setupResult = await backend.runSemantic(
|
|
{
|
|
type: 'click_element',
|
|
observationId: setupObservation.observationId,
|
|
elementId: mutation.elementId,
|
|
elementIdentity: mutation.identity,
|
|
},
|
|
signal,
|
|
setupContext,
|
|
);
|
|
if (!setupResult.outcome.ok) {
|
|
throw new Error(`ambiguity fixture mutation failed: ${setupResult.outcome.message}`);
|
|
}
|
|
}
|
|
if (scenario === 'restart-recovery' && !restartRequested) {
|
|
restartRequested = true;
|
|
await writeFile(
|
|
join(resolvedTemporaryDirectory, 'restart-request.json'),
|
|
`${JSON.stringify({ oldPID: observation.pid })}\n`,
|
|
{ flag: 'wx', mode: 0o600 },
|
|
);
|
|
const completePath = join(resolvedTemporaryDirectory, 'restart-complete.json');
|
|
const deadline = Date.now() + 15_000;
|
|
let completed;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
completed = JSON.parse(await readFile(completePath, 'utf8'));
|
|
break;
|
|
} catch (error) {
|
|
if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
|
|
}
|
|
await new Promise((resolvePromise) => setTimeout(resolvePromise, 50));
|
|
}
|
|
if (!completed) throw new Error('restart completion timeout');
|
|
const apps = await backend.listApps(signal);
|
|
const currentMatches = apps.filter(
|
|
(app) =>
|
|
app.pid === completed.newPID &&
|
|
(app.name === 'Codex CUA Lab' ||
|
|
app.windows?.some((window) => window.title === 'Codex CUA Lab')),
|
|
);
|
|
if (currentMatches.length !== 1) {
|
|
throw new Error(`expected one restarted fixture, got ${currentMatches.length}`);
|
|
}
|
|
fixture = currentMatches[0];
|
|
const currentWindows =
|
|
fixture.windows?.filter((window) => window.title === 'Codex CUA Lab') ?? [];
|
|
if (currentWindows.length !== 1) {
|
|
throw new Error(`expected one restarted fixture window, got ${currentWindows.length}`);
|
|
}
|
|
fixtureWindowId = currentWindows[0].windowId;
|
|
registerFixtureInstance(fixture, fixtureWindowId);
|
|
}
|
|
return observation;
|
|
},
|
|
async runSemantic(action, signal, context) {
|
|
if (
|
|
scenario === 'intervention-recovery' &&
|
|
action.type === 'set_value' &&
|
|
!injectedIntervention
|
|
) {
|
|
injectedIntervention = true;
|
|
return {
|
|
outcome: {
|
|
ok: false,
|
|
error: 'user_intervened',
|
|
message: 'synthetic intervention gate requires a fresh observation',
|
|
},
|
|
};
|
|
}
|
|
const result = await backend.runSemantic(action, signal, context);
|
|
semanticOutcomes.push({
|
|
actionType: action.type,
|
|
ok: result.outcome.ok,
|
|
...(!result.outcome.ok
|
|
? {
|
|
error: result.outcome.error,
|
|
ambiguous: /ambiguous/i.test(result.outcome.message),
|
|
}
|
|
: {}),
|
|
});
|
|
if (
|
|
scenario === 'restart-recovery' &&
|
|
!result.outcome.ok &&
|
|
result.outcome.error === 'target_missing'
|
|
) {
|
|
staleRecoveryObserved = true;
|
|
}
|
|
if (
|
|
scenario === 'ambiguity' &&
|
|
action.type === 'click_element' &&
|
|
!result.outcome.ok &&
|
|
result.outcome.error === 'stale_frame' &&
|
|
/ambiguous/.test(result.outcome.message)
|
|
) {
|
|
ambiguousRecoveryObserved = true;
|
|
}
|
|
if (
|
|
scenario === 'ambiguity' &&
|
|
action.type === 'click_element' &&
|
|
!result.outcome.ok &&
|
|
result.outcome.error === 'target_changed'
|
|
) {
|
|
semanticStructureChangeRejected = true;
|
|
}
|
|
if (action.type === 'set_value' && result.outcome.ok) {
|
|
const field = result.observation?.elements.find(
|
|
(element) => element.label === 'CUA Lab Set Value Field',
|
|
);
|
|
freshAxValue = result.outcome.evidence?.readbackValue ?? field?.value;
|
|
}
|
|
return result;
|
|
},
|
|
};
|
|
const [rawComputerTool] = buildComputerUseTools({ backend: instrumentedBackend });
|
|
const computerTool = {
|
|
...rawComputerTool,
|
|
async impl(args, context) {
|
|
const action = typeof args?.action === 'string' ? args.action : 'unknown';
|
|
const allowed =
|
|
scenario === 'observe-only'
|
|
? new Set(['list_apps', 'observe', 'wait'])
|
|
: scenario === 'ax-click' || scenario === 'ambiguity'
|
|
? new Set(['list_apps', 'observe', 'click_element', 'wait'])
|
|
: scenario === 'ax-multi-step'
|
|
? new Set(['list_apps', 'observe', 'set_value', 'click_element', 'wait'])
|
|
: new Set(['list_apps', 'observe', 'set_value', 'wait']);
|
|
const budget = actionBudgetFor(scenario);
|
|
const nextTotal = totalActionAttempts + 1;
|
|
const nextActionCount = (actionAttemptCounts.get(action) ?? 0) + 1;
|
|
totalActionAttempts = nextTotal;
|
|
actionAttemptCounts.set(action, nextActionCount);
|
|
const sourceTarget = observationTargets.get(args?.observation_id);
|
|
const ownedWindows = fixtureInstances.get(sourceTarget?.pid);
|
|
const attempt = {
|
|
action: { type: action },
|
|
toolCallId: context.toolCallId,
|
|
sourceObservationId: args?.observation_id,
|
|
targetPid: sourceTarget?.pid,
|
|
targetWindowId: sourceTarget?.windowId,
|
|
targetOwned: Boolean(ownedWindows?.has(sourceTarget?.windowId)),
|
|
success: false,
|
|
durationMs: 0,
|
|
};
|
|
actions.push(attempt);
|
|
const rejectAttempt = (resultCode, message) => {
|
|
attempt.text = `maka_computer.${action} failed: ${resultCode}`;
|
|
throw new Error(message);
|
|
};
|
|
if (!allowed.has(action)) {
|
|
rejectAttempt(
|
|
'unsupported_action_policy',
|
|
`model action '${action}' is outside the AX-only scenario budget`,
|
|
);
|
|
}
|
|
if (nextTotal > budget.total || nextActionCount > (budget.counts[action] ?? 0)) {
|
|
rejectAttempt('action_budget_exceeded', 'AX-only scenario action budget exceeded');
|
|
}
|
|
if (action === 'observe') {
|
|
// The `computer` tool contract is "observe requires app OR window_id", so a
|
|
// model that supplies only one of them is behaving correctly. Requiring both
|
|
// would fail every compliant model before it ever reaches a dispatch. Assert
|
|
// the real invariant instead: whatever the model did supply must identify the
|
|
// fixture, and must not point at any other target.
|
|
const appProvided = args.app !== undefined;
|
|
const windowProvided = args.window_id !== undefined;
|
|
const appMatches = args.app === fixture.appId;
|
|
const windowMatches = args.window_id === fixtureWindowId;
|
|
const identifiesFixture = (appProvided && appMatches) || (windowProvided && windowMatches);
|
|
const contradictsFixture = (appProvided && !appMatches) || (windowProvided && !windowMatches);
|
|
if (!identifiesFixture || contradictsFixture) {
|
|
rejectAttempt(
|
|
'target_mismatch',
|
|
'model observe target does not match the exact fixture identity',
|
|
);
|
|
}
|
|
}
|
|
const actionStartedAt = Date.now();
|
|
let result;
|
|
try {
|
|
result = await rawComputerTool.impl(args, context);
|
|
} catch (error) {
|
|
const resultCode = typeof error?.code === 'string' ? error.code : 'tool_error';
|
|
attempt.durationMs = Date.now() - actionStartedAt;
|
|
attempt.text = `maka_computer.${action} failed: ${resultCode}`;
|
|
throw error;
|
|
}
|
|
const observation = parseObservation(result?.modelText) ?? parseObservation(result?.text);
|
|
const dispatch = traces.findLast(
|
|
(trace) => trace.type === 'dispatch' && trace.toolCallId === context.toolCallId,
|
|
);
|
|
const resultCode =
|
|
result?.error ??
|
|
(typeof result?.text === 'string'
|
|
? result.text.match(/\bfailed:\s*([a-z][a-z0-9_]{1,63})\b/i)?.[1]
|
|
: undefined);
|
|
if (observation) {
|
|
observationTargets.set(observation.observation_id, {
|
|
pid: observation.pid,
|
|
windowId: observation.window_id,
|
|
});
|
|
}
|
|
const targetPid = observation?.pid ?? dispatch?.pid ?? sourceTarget?.pid;
|
|
const targetWindowId = observation?.window_id ?? dispatch?.windowId ?? sourceTarget?.windowId;
|
|
const resultOwnedWindows = fixtureInstances.get(targetPid);
|
|
Object.assign(attempt, {
|
|
resultObservationId: observation?.observation_id,
|
|
targetPid,
|
|
targetWindowId,
|
|
targetOwned: Boolean(resultOwnedWindows?.has(targetWindowId)),
|
|
success: !resultCode,
|
|
expectedFailure: qualificationScenario.expectedFailures.some(
|
|
(expected) => expected.action === action && expected.error === resultCode,
|
|
),
|
|
durationMs: Date.now() - actionStartedAt,
|
|
text:
|
|
result?.text ?? (resultCode ? `maka_computer.${action} failed: ${resultCode}` : undefined),
|
|
});
|
|
return result;
|
|
},
|
|
};
|
|
|
|
const connection = {
|
|
slug: provider === 'anthropic' ? 'coproxy' : 'azure-bridge',
|
|
name: provider === 'anthropic' ? 'Anthropic Coproxy' : 'Azure Bridge',
|
|
providerType: provider,
|
|
baseUrl,
|
|
defaultModel: modelId,
|
|
enabled: true,
|
|
createdAt: 1,
|
|
updatedAt: 1,
|
|
};
|
|
const task =
|
|
scenario === 'observe-only'
|
|
? 'Use Maka Computer to inspect "Codex CUA Lab". Start with list_apps, observe ' +
|
|
'the exact app/window, report that "CUA Lab Set Value Field" is visible, ' +
|
|
'do not mutate anything, then finish.'
|
|
: scenario === 'ax-click'
|
|
? 'Use Maka Computer to click the element labeled "CUA Lab Primary Button" exactly ' +
|
|
'once in "Codex CUA Lab". Start with list_apps and observe, use click_element ' +
|
|
'with IDs from that observation, verify the fresh state, then finish. Never use coordinates.'
|
|
: scenario === 'ax-multi-step'
|
|
? `Use Maka Computer in "Codex CUA Lab" to complete two semantic actions in order. ` +
|
|
`First set "CUA Lab Set Value Field" to "${targetValue}" with set_value. ` +
|
|
'Then use the fresh observation returned by that action to click ' +
|
|
'"CUA Lab Primary Button" exactly once with click_element. Verify both fresh ' +
|
|
'results, then finish. Never use coordinates, scroll, drag, type, or key actions.'
|
|
: scenario === 'ambiguity'
|
|
? 'Use Maka Computer to click the observed element labeled "CUA Lab Stale Target" ' +
|
|
'exactly once in "Codex CUA Lab". Start with list_apps and observe, then use ' +
|
|
'click_element with the exact observation_id and element_id. If the tool rejects ' +
|
|
'the target, report the failure and stop rather than guessing or using coordinates.'
|
|
: `Use Maka Computer to set "CUA Lab Set Value Field" in "Codex CUA Lab" ` +
|
|
`to "${targetValue}". Start with list_apps, observe the exact app/window, ` +
|
|
'use set_value with the observation_id and element_id from that observation, ' +
|
|
'verify the fresh observation value, then finish. If user_intervened is returned, ' +
|
|
'observe again before retrying. If target_missing or stale_frame is returned, ' +
|
|
'list apps and observe the current process again before retrying. Never use ' +
|
|
'coordinate, click, scroll, drag, type, or key actions.';
|
|
const turnId = 'turn-real-ax-model';
|
|
const durableTurn = createDirectRuntimeTurnLedger({
|
|
sessionId: 'real-ax-model-e2e',
|
|
turnId,
|
|
text: task,
|
|
newId: () => `runtime-event-${++nextId}`,
|
|
now: () => ++now,
|
|
});
|
|
const runtime = new AiSdkBackend({
|
|
sessionId: 'real-ax-model-e2e',
|
|
header: {
|
|
id: 'real-ax-model-e2e',
|
|
workspaceRoot: repoRoot,
|
|
cwd: repoRoot,
|
|
createdAt: now,
|
|
lastUsedAt: now,
|
|
name: 'Real AX Model E2E',
|
|
isFlagged: false,
|
|
labels: [],
|
|
isArchived: false,
|
|
status: 'active',
|
|
statusUpdatedAt: now,
|
|
hasUnread: false,
|
|
backend: 'ai-sdk',
|
|
llmConnectionSlug: connection.slug,
|
|
connectionLocked: true,
|
|
model: modelId,
|
|
permissionMode: 'bypass',
|
|
schemaVersion: 1,
|
|
},
|
|
appendMessage: async (message) => {
|
|
messages.push(message);
|
|
},
|
|
connection,
|
|
apiKey,
|
|
modelId,
|
|
readExecutionBoundary: async () => ({ kind: 'bypass', revision: 0 }),
|
|
modelFactory: (input) => getAIModel(input),
|
|
tools: [computerTool],
|
|
maxSteps: 8,
|
|
loadTurnRuntimeEvents: durableTurn.loadTurnRuntimeEvents,
|
|
newId: () => `id-${++nextId}`,
|
|
now: () => ++now,
|
|
recordToolInvocation: (record) => {
|
|
telemetry.push({
|
|
toolName: record.toolName,
|
|
status: record.status,
|
|
errorClass: record.errorClass,
|
|
argsSummary: record.argsSummary,
|
|
});
|
|
},
|
|
});
|
|
|
|
const events = [];
|
|
let terminalEvent;
|
|
try {
|
|
for await (const event of runtime.send({
|
|
turnId,
|
|
text: task,
|
|
context: [],
|
|
headAnchorRuntimeEvent: durableTurn.anchor,
|
|
})) {
|
|
durableTurn.record(event);
|
|
events.push(event.type);
|
|
if (event.type === 'complete' || event.type === 'abort' || event.type === 'error') {
|
|
terminalEvent = {
|
|
type: event.type,
|
|
...(typeof event.stopReason === 'string' ? { stopReason: event.stopReason } : {}),
|
|
};
|
|
}
|
|
}
|
|
|
|
const fixtureState = JSON.parse(await readFile(statePath, 'utf8'));
|
|
const setValueDispatches = traces.filter(
|
|
(trace) =>
|
|
trace.type === 'dispatch' && trace.actionType === 'set_value' && trace.address === 'ax',
|
|
);
|
|
const clickDispatches = traces.filter(
|
|
(trace) =>
|
|
trace.type === 'dispatch' && trace.actionType === 'click_element' && trace.address === 'ax',
|
|
);
|
|
const modelClickDispatches = clickDispatches.filter(
|
|
(trace) => trace.toolCallId !== 'fixture-mutate-ambiguity',
|
|
);
|
|
const setupClickDispatches = clickDispatches.filter(
|
|
(trace) => trace.toolCallId === 'fixture-mutate-ambiguity',
|
|
);
|
|
const forbiddenDispatches = traces.filter(
|
|
(trace) => trace.type === 'dispatch' && trace.address === 'px',
|
|
);
|
|
if (
|
|
!['observe-only', 'ax-click', 'ambiguity'].includes(scenario) &&
|
|
freshAxValue !== targetValue
|
|
) {
|
|
throw new Error(`fresh AX value mismatch: ${String(freshAxValue)}`);
|
|
}
|
|
const expectedSetValueDispatches =
|
|
scenario === 'observe-only' || scenario === 'ax-click' || scenario === 'ambiguity' ? 0 : 1;
|
|
if (setValueDispatches.length !== expectedSetValueDispatches) {
|
|
throw new Error(
|
|
`expected ${expectedSetValueDispatches} AX set_value dispatches, got ${setValueDispatches.length}`,
|
|
);
|
|
}
|
|
const expectedClickDispatches = scenario === 'ax-click' || scenario === 'ax-multi-step' ? 1 : 0;
|
|
if (modelClickDispatches.length !== expectedClickDispatches) {
|
|
throw new Error(
|
|
`expected ${expectedClickDispatches} model AX click dispatches, got ${modelClickDispatches.length}`,
|
|
);
|
|
}
|
|
if (scenario === 'ambiguity' && setupClickDispatches.length !== 1) {
|
|
throw new Error(`expected one ambiguity setup AX dispatch, got ${setupClickDispatches.length}`);
|
|
}
|
|
if (forbiddenDispatches.length !== 0) {
|
|
throw new Error(`forbidden pixel dispatch count: ${forbiddenDispatches.length}`);
|
|
}
|
|
if (scenario === 'restart-recovery' && !staleRecoveryObserved) {
|
|
throw new Error('model restart scenario did not observe target_missing');
|
|
}
|
|
if (scenario === 'restart-recovery') {
|
|
const staleIndex = actions.findIndex(
|
|
(record) =>
|
|
record.action.type === 'set_value' &&
|
|
record.success === false &&
|
|
record.expectedFailure === true,
|
|
);
|
|
const freshIndex = actions.findIndex(
|
|
(record, index) =>
|
|
index > staleIndex &&
|
|
record.action.type === 'observe' &&
|
|
record.success === true &&
|
|
typeof record.resultObservationId === 'string' &&
|
|
record.targetPid === fixture.pid,
|
|
);
|
|
const retry = actions.find(
|
|
(record, index) =>
|
|
index > freshIndex &&
|
|
record.action.type === 'set_value' &&
|
|
record.success === true &&
|
|
record.sourceObservationId === actions[freshIndex]?.resultObservationId &&
|
|
record.targetPid === fixture.pid,
|
|
);
|
|
const retryDispatch =
|
|
retry &&
|
|
setValueDispatches.some(
|
|
(trace) =>
|
|
trace.toolCallId === retry.toolCallId &&
|
|
trace.pid === retry.targetPid &&
|
|
trace.windowId === retry.targetWindowId,
|
|
);
|
|
if (staleIndex < 0 || freshIndex < 0 || !retry || !retryDispatch) {
|
|
throw new Error(
|
|
'restart recovery requires stale target_missing, fresh observation, and successful AX retry',
|
|
);
|
|
}
|
|
}
|
|
if (
|
|
scenario === 'ambiguity' &&
|
|
(!actions.some((record) => record.action.type === 'click_element') ||
|
|
(!ambiguousRecoveryObserved && !semanticStructureChangeRejected))
|
|
) {
|
|
throw new Error(
|
|
`model ambiguity scenario did not attempt and observe fail-closed rejection: ${JSON.stringify(
|
|
{
|
|
actionTypes: actions.map((record) => record.action.type),
|
|
semanticOutcomes,
|
|
setupClickDispatches: setupClickDispatches.length,
|
|
modelClickDispatches: modelClickDispatches.length,
|
|
},
|
|
)}`,
|
|
);
|
|
}
|
|
if (
|
|
(scenario === 'ax-click' || scenario === 'ax-multi-step') &&
|
|
fixtureState.controls.buttonClickCount !== 1
|
|
) {
|
|
throw new Error(`primary button count mismatch: ${fixtureState.controls.buttonClickCount}`);
|
|
}
|
|
if (
|
|
scenario === 'ambiguity' &&
|
|
(fixtureState.hierarchy.mode !== 'ambiguous' ||
|
|
fixtureState.hierarchy.staleTargetClickCount !== 0 ||
|
|
fixtureState.hierarchy.wrongTargetClickCount !== 0)
|
|
) {
|
|
throw new Error(`ambiguous stale target mutated: ${JSON.stringify(fixtureState.hierarchy)}`);
|
|
}
|
|
if (
|
|
fixtureState.synthetic !== true ||
|
|
fixtureState.appPath !== expectedAppPath ||
|
|
fixtureState.oop?.hostPID !== fixture.pid
|
|
) {
|
|
throw new Error('fixture identity changed during model execution');
|
|
}
|
|
if (terminalEvent?.type !== 'complete' || terminalEvent.stopReason !== 'end_turn') {
|
|
throw new Error(`model Runtime did not complete/end_turn: ${JSON.stringify(terminalEvent)}`);
|
|
}
|
|
if (scenario === 'ax-multi-step') {
|
|
const setIndex = actions.findIndex((record) => record.action.type === 'set_value');
|
|
const clickIndex = actions.findIndex((record) => record.action.type === 'click_element');
|
|
const setAction = actions[setIndex];
|
|
const clickAction = actions[clickIndex];
|
|
if (
|
|
setIndex < 0 ||
|
|
clickIndex <= setIndex ||
|
|
typeof setAction?.resultObservationId !== 'string' ||
|
|
clickAction?.sourceObservationId !== setAction.resultObservationId
|
|
) {
|
|
throw new Error(
|
|
`multi-step action order or observation lineage mismatch: ${JSON.stringify(
|
|
actions.map((record) => ({
|
|
type: record.action.type,
|
|
sourceObservationId: record.sourceObservationId,
|
|
resultObservationId: record.resultObservationId,
|
|
})),
|
|
)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const evidenceClass = scenario === 'intervention-recovery' ? 'fault-injection' : 'real-runtime';
|
|
const canonicalFixtureState = {
|
|
target: {
|
|
setValue: fixtureState.controls.setValue,
|
|
buttonClickCount: fixtureState.controls.buttonClickCount,
|
|
hierarchyMode: fixtureState.hierarchy.mode,
|
|
staleTargetClickCount: fixtureState.hierarchy.staleTargetClickCount,
|
|
wrongTargetClickCount: fixtureState.hierarchy.wrongTargetClickCount,
|
|
},
|
|
};
|
|
const fixtureIdentityInstances = [...fixtureInstances.entries()].map(([pid, windowIds]) => ({
|
|
pid,
|
|
windowIds: [...windowIds],
|
|
}));
|
|
const minimumActionsPassed =
|
|
scenario !== 'ambiguity' || actions.some((record) => record.action.type === 'click_element');
|
|
const actionsWithinBudget =
|
|
totalActionAttempts <= actionBudgetFor(scenario).total &&
|
|
[...actionAttemptCounts].every(
|
|
([action, count]) => count <= (actionBudgetFor(scenario).counts[action] ?? 0),
|
|
);
|
|
const actionResultsPassed = actions.every(
|
|
(record) => record.success === true || record.expectedFailure === true,
|
|
);
|
|
const dispatchPathPassed =
|
|
forbiddenDispatches.length === 0 &&
|
|
setValueDispatches.length === expectedSetValueDispatches &&
|
|
modelClickDispatches.length === expectedClickDispatches;
|
|
const qualified =
|
|
minimumActionsPassed && actionsWithinBudget && actionResultsPassed && dispatchPathPassed;
|
|
const sanitized = sanitizeCuDirectReport({
|
|
schemaVersion: 1,
|
|
...reportLineage,
|
|
evidenceClass,
|
|
scenarioId:
|
|
scenario === 'observe-only'
|
|
? 'appkit-ax-observe-only'
|
|
: scenario === 'intervention-recovery'
|
|
? 'appkit-ax-intervention-recovery'
|
|
: scenario === 'restart-recovery'
|
|
? 'appkit-ax-restart-recovery'
|
|
: scenario === 'ax-click'
|
|
? 'appkit-ax-click'
|
|
: scenario === 'ax-multi-step'
|
|
? 'appkit-ax-multi-step'
|
|
: scenario === 'ambiguity'
|
|
? 'appkit-ax-ambiguity'
|
|
: 'appkit-ax-set-value',
|
|
policyMode: 'enforced',
|
|
qualificationEligible: evidenceClass === 'real-runtime',
|
|
producer: 'computer-use/real-ax',
|
|
transportClass: 'live-network',
|
|
model: modelId,
|
|
provider,
|
|
baseUrl,
|
|
status: qualified ? 'pass' : 'fail',
|
|
terminal: terminalEvent,
|
|
fixtureIdentity: {
|
|
instances: fixtureIdentityInstances,
|
|
},
|
|
...(evidenceClass === 'fault-injection'
|
|
? {
|
|
faultInjection: {
|
|
layer: 'runtime',
|
|
kind: 'user_intervened',
|
|
},
|
|
}
|
|
: {}),
|
|
totalLatencyMs: Date.now() - startedAt,
|
|
loopStatus: 'completed',
|
|
state: {
|
|
fixtureIdentityValid: true,
|
|
freshAxValueMatched:
|
|
scenario === 'observe-only' || scenario === 'ax-click' || scenario === 'ambiguity'
|
|
? undefined
|
|
: freshAxValue === targetValue,
|
|
injectedIntervention,
|
|
staleRecoveryObserved,
|
|
ambiguousRecoveryObserved,
|
|
semanticStructureChangeRejected,
|
|
ambiguityFixtureMutated,
|
|
setValueDispatches: setValueDispatches.length,
|
|
clickDispatches: modelClickDispatches.length,
|
|
setupClickDispatches: setupClickDispatches.length,
|
|
pixelDispatches: forbiddenDispatches.length,
|
|
primaryButtonClicks: fixtureState.controls.buttonClickCount,
|
|
ambiguousClicks: fixtureState.hierarchy.staleTargetClickCount,
|
|
terminalEvent,
|
|
toolCallsPersisted: messages.filter((message) => message.type === 'tool_call').length,
|
|
toolResultsPersisted: messages.filter((message) => message.type === 'tool_result').length,
|
|
telemetrySuccesses: telemetry.filter((record) => record.status === 'success').length,
|
|
telemetryErrors: telemetry.filter((record) => record.status === 'error').length,
|
|
telemetryErrorClasses: [
|
|
...new Set(
|
|
telemetry
|
|
.filter((record) => record.status === 'error')
|
|
.map((record) => record.errorClass)
|
|
.filter(Boolean),
|
|
),
|
|
],
|
|
telemetryErrorActions: telemetry
|
|
.filter((record) => record.status === 'error')
|
|
.flatMap((record) => {
|
|
try {
|
|
const summary = JSON.parse(record.argsSummary ?? '{}');
|
|
return typeof summary.action === 'string' ? [summary.action] : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}),
|
|
semanticFailures: semanticOutcomes.filter((outcome) => !outcome.ok).length,
|
|
semanticFailureCode: semanticOutcomes.find((outcome) => !outcome.ok)?.error,
|
|
ambiguousSemanticFailures: semanticOutcomes.filter(
|
|
(outcome) => !outcome.ok && outcome.ambiguous,
|
|
).length,
|
|
},
|
|
actionAttempts: totalActionAttempts,
|
|
actionCount: actions.length,
|
|
actionCounts: Object.fromEntries(
|
|
actions.reduce((counts, record) => {
|
|
const type = record.action.type;
|
|
counts.set(type, (counts.get(type) ?? 0) + 1);
|
|
return counts;
|
|
}, new Map()),
|
|
),
|
|
minimumActionsPassed,
|
|
actionsWithinBudget,
|
|
dispatchPathPassed,
|
|
actions,
|
|
fixtureState: canonicalFixtureState,
|
|
expectedState: [
|
|
{
|
|
windowId: 'target',
|
|
path: 'buttonClickCount',
|
|
pass:
|
|
!['ax-click', 'ax-multi-step'].includes(scenario) ||
|
|
fixtureState.controls.buttonClickCount === 1,
|
|
actual: fixtureState.controls.buttonClickCount,
|
|
},
|
|
],
|
|
forbiddenEffects: {
|
|
status:
|
|
forbiddenDispatches.length === 0 && fixtureState.hierarchy.wrongTargetClickCount === 0
|
|
? 'pass'
|
|
: 'fail',
|
|
violations: [],
|
|
},
|
|
driverTraces: traces.filter((trace) => trace.toolCallId !== 'fixture-mutate-ambiguity'),
|
|
});
|
|
await writeFile(reportPath, `${JSON.stringify(sanitized, null, 2)}\n`, {
|
|
flag: 'wx',
|
|
mode: 0o600,
|
|
});
|
|
process.stdout.write(`${JSON.stringify(sanitized, null, 2)}\n`);
|
|
if (!qualified) {
|
|
throw new Error('real AX model qualification rejected canonical action evidence');
|
|
}
|
|
} finally {
|
|
await runtime.dispose();
|
|
backend.dispose();
|
|
}
|