agent-radar
7be535059e
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s
789 lines
28 KiB
JavaScript
789 lines
28 KiB
JavaScript
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
|
|
const READINESS = new Set(['real', 'contract', 'unsupported']);
|
|
const EVIDENCE_CLASSES = new Set([
|
|
'real-runtime',
|
|
'fault-injection',
|
|
'hermetic-protocol',
|
|
'static-contract',
|
|
]);
|
|
const REAL_REPORT_PRODUCERS = new Set(['computer-use/real-model', 'computer-use/real-ax']);
|
|
const ACTIONS_WITHOUT_TARGET_OWNERSHIP = new Set(['list_apps', 'wait', 'cursor_position']);
|
|
const ACTIONS_WITHOUT_OBSERVATION_LINEAGE = new Set([
|
|
'list_apps',
|
|
'observe',
|
|
'screenshot',
|
|
'wait',
|
|
'cursor_position',
|
|
]);
|
|
|
|
function optionValue(argv, names) {
|
|
for (const name of names) {
|
|
const index = argv.indexOf(name);
|
|
if (index >= 0) return argv[index + 1];
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function requiredOption(argv, names) {
|
|
const value = optionValue(argv, names);
|
|
if (!value || value.startsWith('--')) {
|
|
throw new Error(`missing required option ${names.join('/')}`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function asEntries(value, key, fileName) {
|
|
const entries = Array.isArray(value) ? value : value?.[key];
|
|
if (!Array.isArray(entries)) {
|
|
throw new Error(`${fileName} must be an array or contain a ${key} array`);
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
function requireId(entry, kind) {
|
|
if (!entry || typeof entry !== 'object' || typeof entry.id !== 'string' || !entry.id.trim()) {
|
|
throw new Error(`${kind} entries require a non-empty id`);
|
|
}
|
|
return entry.id.trim();
|
|
}
|
|
|
|
function select(value, scenarioId) {
|
|
if (!value || Array.isArray(value) || typeof value !== 'object') return value;
|
|
return value[scenarioId] ?? value.default;
|
|
}
|
|
|
|
function renderTemplate(value, variables) {
|
|
if (typeof value === 'string') {
|
|
return value.replace(/\{([a-zA-Z][a-zA-Z0-9]*)\}/g, (match, name) =>
|
|
Object.hasOwn(variables, name) ? String(variables[name]) : match,
|
|
);
|
|
}
|
|
if (Array.isArray(value)) {
|
|
return value.map((part) => renderTemplate(part, variables));
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function displayCommand(command) {
|
|
if (Array.isArray(command)) {
|
|
return command
|
|
.map((part) => {
|
|
const text = String(part);
|
|
return /^[a-zA-Z0-9_./:=+-]+$/.test(text) ? text : JSON.stringify(text);
|
|
})
|
|
.join(' ');
|
|
}
|
|
return typeof command === 'string' ? command : null;
|
|
}
|
|
|
|
function finiteNumbers(values) {
|
|
return values.flat(Infinity).filter((value) => Number.isFinite(value));
|
|
}
|
|
|
|
function percentile(sorted, value) {
|
|
if (sorted.length === 0) return null;
|
|
const index = Math.ceil((value / 100) * sorted.length) - 1;
|
|
return sorted[Math.max(0, Math.min(sorted.length - 1, index))];
|
|
}
|
|
|
|
export function summarizeLatency(values) {
|
|
const numbers = finiteNumbers(Array.isArray(values) ? values : [values]);
|
|
if (numbers.length === 0) return null;
|
|
const sorted = [...numbers].sort((a, b) => a - b);
|
|
const total = numbers.reduce((sum, value) => sum + value, 0);
|
|
return {
|
|
samples: numbers.length,
|
|
averageMs: Math.round((total / numbers.length) * 100) / 100,
|
|
p50Ms: percentile(sorted, 50),
|
|
p95Ms: percentile(sorted, 95),
|
|
maxMs: sorted.at(-1),
|
|
};
|
|
}
|
|
|
|
function valuesAtPath(value, path) {
|
|
const segments = String(path).split('.').filter(Boolean);
|
|
let current = value;
|
|
for (const segment of segments) {
|
|
if (current == null || typeof current !== 'object') return undefined;
|
|
current = current[segment];
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function deepSubsetEqual(actual, expected) {
|
|
if (Array.isArray(expected)) {
|
|
return (
|
|
Array.isArray(actual) &&
|
|
expected.length === actual.length &&
|
|
expected.every((value, index) => deepSubsetEqual(actual[index], value))
|
|
);
|
|
}
|
|
if (expected && typeof expected === 'object') {
|
|
return (
|
|
actual != null &&
|
|
typeof actual === 'object' &&
|
|
Object.entries(expected).every(([key, value]) => deepSubsetEqual(actual[key], value))
|
|
);
|
|
}
|
|
return Object.is(actual, expected);
|
|
}
|
|
|
|
function normalizeFixture(report, scenario) {
|
|
const assertions = Array.isArray(scenario.expectedState) ? scenario.expectedState : [];
|
|
if (assertions.length === 0) {
|
|
return { status: 'not-defined', expected: [], actual: report.fixtureState ?? null };
|
|
}
|
|
if (!report.fixtureState || typeof report.fixtureState !== 'object') {
|
|
return { status: 'unknown', expected: assertions, actual: null };
|
|
}
|
|
const results = assertions.map((assertion) => {
|
|
const actual = valuesAtPath(report.fixtureState?.[assertion.windowId], assertion.path);
|
|
return {
|
|
...assertion,
|
|
actual,
|
|
pass: actual !== undefined && assertionMatches(assertion, actual),
|
|
};
|
|
});
|
|
return {
|
|
status: results.every((result) => result.pass) ? 'pass' : 'fail',
|
|
expected: assertions,
|
|
actual: report.fixtureState,
|
|
results,
|
|
};
|
|
}
|
|
|
|
function assertionMatches(assertion, actual) {
|
|
if (Object.hasOwn(assertion, 'equals')) return deepSubsetEqual(actual, assertion.equals);
|
|
if (Object.hasOwn(assertion, 'greaterThan')) return actual > assertion.greaterThan;
|
|
if (Object.hasOwn(assertion, 'greaterThanOrEqual')) return actual >= assertion.greaterThanOrEqual;
|
|
if (Object.hasOwn(assertion, 'lessThan')) return actual < assertion.lessThan;
|
|
if (Object.hasOwn(assertion, 'lessThanOrEqual')) return actual <= assertion.lessThanOrEqual;
|
|
return false;
|
|
}
|
|
|
|
function effectId(effect) {
|
|
if (typeof effect === 'string') return effect;
|
|
if (effect && typeof effect === 'object') return effect.id ?? effect.name ?? effect.path;
|
|
return undefined;
|
|
}
|
|
|
|
function normalizeForbiddenEffects(report, scenario) {
|
|
const definitions = Array.isArray(scenario.forbiddenEffects) ? scenario.forbiddenEffects : [];
|
|
const reported = report.forbiddenEffects;
|
|
const observed = Array.isArray(reported?.observed) ? reported.observed : [];
|
|
const observedIds = new Set(observed.map(effectId).filter(Boolean));
|
|
const violations = [];
|
|
let missingEvidence = false;
|
|
|
|
for (const definition of definitions) {
|
|
if (typeof definition === 'string') {
|
|
if (observedIds.has(definition)) violations.push({ id: definition, source: 'reported' });
|
|
continue;
|
|
}
|
|
if (!definition || typeof definition !== 'object') continue;
|
|
const id = effectId(definition) ?? 'unnamed';
|
|
if (observedIds.has(id)) {
|
|
violations.push({ id, source: 'reported' });
|
|
continue;
|
|
}
|
|
if (definition.path) {
|
|
const actual = valuesAtPath(report.fixtureState?.[definition.windowId], definition.path);
|
|
const safeValue = Object.hasOwn(definition, 'equals')
|
|
? definition.equals
|
|
: definition.allowed;
|
|
if (actual === undefined) {
|
|
missingEvidence = true;
|
|
} else if (safeValue !== undefined && !deepSubsetEqual(actual, safeValue)) {
|
|
violations.push({
|
|
id,
|
|
source: `${definition.windowId}.${definition.path}`,
|
|
expected: safeValue,
|
|
actual,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (reported?.status === 'fail' || reported?.pass === false) {
|
|
const reportedViolations = Array.isArray(reported.violations)
|
|
? reported.violations
|
|
: [{ id: 'reported-failure', source: 'report' }];
|
|
violations.push(...reportedViolations);
|
|
}
|
|
return {
|
|
status:
|
|
definitions.length === 0 && reported == null
|
|
? 'not-defined'
|
|
: missingEvidence
|
|
? 'unknown'
|
|
: violations.length > 0
|
|
? 'fail'
|
|
: 'pass',
|
|
forbidden: definitions,
|
|
observed,
|
|
violations,
|
|
};
|
|
}
|
|
|
|
function countRetries(report, actions) {
|
|
if (Number.isFinite(report.retries)) return report.retries;
|
|
return actions.reduce((total, action) => {
|
|
if (Number.isFinite(action?.retries)) return total + action.retries;
|
|
return total + (action?.retry === true ? 1 : 0);
|
|
}, 0);
|
|
}
|
|
|
|
export function normalizeReport(report, scenario) {
|
|
const actions = Array.isArray(report.actions) ? report.actions : [];
|
|
const actionModelValues = actions.map((action) => action?.modelLatencyMs ?? action?.modelLatency);
|
|
const actionToolValues = actions.map(
|
|
(action) => action?.toolLatencyMs ?? action?.toolLatency ?? action?.durationMs,
|
|
);
|
|
const actionDisplayValues = actions.map((action) => action?.displayLagMs ?? action?.displayLag);
|
|
const modelValues =
|
|
finiteNumbers(actionModelValues).length > 0
|
|
? actionModelValues
|
|
: [report.modelLatencyMs, report.modelLatency];
|
|
const toolValues =
|
|
finiteNumbers(actionToolValues).length > 0
|
|
? actionToolValues
|
|
: [report.toolLatencyMs, report.toolLatency];
|
|
const displayValues =
|
|
finiteNumbers(actionDisplayValues).length > 0
|
|
? actionDisplayValues
|
|
: [report.displayLagMs, report.displayLag];
|
|
return {
|
|
evidenceClass: EVIDENCE_CLASSES.has(report.evidenceClass) ? report.evidenceClass : null,
|
|
policyMode:
|
|
report.policyMode === 'enforced' || report.policyMode === 'bypassed'
|
|
? report.policyMode
|
|
: null,
|
|
modelLatency: summarizeLatency(modelValues),
|
|
toolLatency: summarizeLatency(toolValues),
|
|
displayLag: summarizeLatency(displayValues),
|
|
actionCount: Number.isFinite(report.actionCount) ? report.actionCount : actions.length,
|
|
retries: countRetries(report, actions),
|
|
fixture: normalizeFixture(report, scenario),
|
|
forbiddenEffects: normalizeForbiddenEffects(report, scenario),
|
|
};
|
|
}
|
|
|
|
export function validateRealReport(report, provider, scenario) {
|
|
const errors = [];
|
|
if (report.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
if (typeof report.runId !== 'string' || !report.runId) errors.push('runId missing');
|
|
if (typeof report.gitRevision !== 'string' || !/^[a-f0-9]{40}$/.test(report.gitRevision)) {
|
|
errors.push('gitRevision missing or invalid');
|
|
}
|
|
const generatedAt = Date.parse(report.generatedAt);
|
|
if (
|
|
typeof report.generatedAt !== 'string' ||
|
|
!Number.isFinite(generatedAt) ||
|
|
new Date(generatedAt).toISOString() !== report.generatedAt
|
|
)
|
|
errors.push('generatedAt missing or invalid');
|
|
if (
|
|
report.contentLineage?.generator !==
|
|
(report.producer === 'computer-use/real-model'
|
|
? 'scripts/computer-use/real-model.mjs'
|
|
: 'scripts/computer-use/real-ax-harness.mjs') ||
|
|
report.contentLineage?.gitRevision !== report.gitRevision ||
|
|
report.contentLineage?.generatedAt !== report.generatedAt
|
|
)
|
|
errors.push('contentLineage mismatch');
|
|
if (report.evidenceClass !== 'real-runtime') errors.push('evidenceClass must be real-runtime');
|
|
if (report.scenarioId !== scenario.id) errors.push('scenarioId mismatch');
|
|
if (report.status !== 'pass') errors.push('report status must be pass');
|
|
if (report.transportClass !== 'live-network') errors.push('transportClass must be live-network');
|
|
if (!REAL_REPORT_PRODUCERS.has(report.producer)) errors.push('producer missing or unknown');
|
|
const expectedProducer = select(provider.producer, scenario.id);
|
|
if (report.producer !== expectedProducer) errors.push('producer mismatch');
|
|
if (report.policyMode !== 'enforced' && report.policyMode !== 'bypassed') {
|
|
errors.push('policyMode missing or unknown');
|
|
}
|
|
if (report.qualificationEligible !== true) errors.push('qualificationEligible must be true');
|
|
if (report.deprecated === true) errors.push('deprecated reports cannot qualify');
|
|
if (report.provider !== provider.id) errors.push('provider mismatch');
|
|
const expectedModel = select(provider.model ?? provider.modelId, scenario.id);
|
|
if (typeof expectedModel !== 'string' || !expectedModel) errors.push('expected model missing');
|
|
else if (report.model !== expectedModel) errors.push('model mismatch');
|
|
if (report.terminal?.type !== 'complete' || report.terminal?.stopReason !== 'end_turn')
|
|
errors.push('terminal must be complete/end_turn');
|
|
const actions = Array.isArray(report.actions) ? report.actions : [];
|
|
const ledgerCounts = actionCounts(actions);
|
|
if (report.actionAttempts !== actions.length) errors.push('actionAttempts mismatch');
|
|
if (report.actionCount !== actions.length) errors.push('actionCount mismatch');
|
|
if (
|
|
!deepSubsetEqual(report.actionCounts, ledgerCounts) ||
|
|
Object.keys(report.actionCounts ?? {}).length !== Object.keys(ledgerCounts).length
|
|
) {
|
|
errors.push('actionCounts mismatch');
|
|
}
|
|
if (report.minimumActionsPassed !== true) errors.push('minimumActionsPassed must be true');
|
|
if (report.actionsWithinBudget !== true) errors.push('actionsWithinBudget must be true');
|
|
if (
|
|
actions.some(
|
|
(action) =>
|
|
!scenario.allowedActions.includes(action.type) || !actionResultAllowed(action, scenario),
|
|
)
|
|
) {
|
|
errors.push('actions must be successful or scenario-authorized expected failures, and allowed');
|
|
}
|
|
if (
|
|
actions.some(
|
|
(action) =>
|
|
!ACTIONS_WITHOUT_TARGET_OWNERSHIP.has(action.type) &&
|
|
!actionHasOwnedFixtureTrace(action, report.fixtureIdentity),
|
|
)
|
|
) {
|
|
errors.push('target ownership requires fixture PID/window trace evidence');
|
|
}
|
|
if (!actionLineageValid(actions)) {
|
|
errors.push('action observation lineage is incomplete or out of order');
|
|
}
|
|
if (
|
|
scenario.expectedFailures?.some(
|
|
(expected) => expected.action === 'set_value' && expected.error === 'target_missing',
|
|
) &&
|
|
!restartRecoveryValid(actions, report.driverTraces)
|
|
) {
|
|
errors.push(
|
|
'restart recovery requires target_missing, fresh observation, and successful AX retry',
|
|
);
|
|
}
|
|
if (
|
|
Array.isArray(scenario.expectedActionSequence) &&
|
|
!deepSubsetEqual(
|
|
actions.map((action) => action.type),
|
|
scenario.expectedActionSequence,
|
|
)
|
|
) {
|
|
errors.push('action sequence mismatch');
|
|
}
|
|
if (Number.isInteger(scenario.maxTotalActions) && actions.length > scenario.maxTotalActions)
|
|
errors.push('total action budget exceeded');
|
|
for (const [action, maximum] of Object.entries(scenario.maxActionCounts ?? {})) {
|
|
if (actions.filter((entry) => entry.type === action).length > maximum) {
|
|
errors.push(`${action} action budget exceeded`);
|
|
}
|
|
}
|
|
for (const [action, minimum] of Object.entries(scenario.minimumActionCounts ?? {})) {
|
|
if (actions.filter((entry) => entry.type === action).length < minimum) {
|
|
errors.push(`${action} minimum action count missing`);
|
|
}
|
|
}
|
|
const mutationActions = actions.filter(
|
|
(action) =>
|
|
action.success === true &&
|
|
!['list_apps', 'observe', 'screenshot', 'cursor_position', 'wait'].includes(action.type),
|
|
);
|
|
if (mutationActions.length > 0) {
|
|
const traces = Array.isArray(report.driverTraces) ? report.driverTraces : [];
|
|
const consumed = new Set();
|
|
const missingDispatch = mutationActions.find((action) => {
|
|
const index = traces.findIndex(
|
|
(trace, traceIndex) =>
|
|
!consumed.has(traceIndex) &&
|
|
trace.type === 'dispatch' &&
|
|
trace.toolCallId === action.toolCallId &&
|
|
trace.actionType === action.type &&
|
|
trace.pid === action.targetPid &&
|
|
trace.windowId === action.targetWindowId &&
|
|
(trace.address === 'ax' || trace.address === 'semantic'),
|
|
);
|
|
if (index < 0) return true;
|
|
consumed.add(index);
|
|
return false;
|
|
});
|
|
if (missingDispatch) {
|
|
errors.push(`safe dispatch evidence missing for ${missingDispatch.type}`);
|
|
}
|
|
if (report.dispatchPathPassed !== true) errors.push('dispatchPathPassed must be true');
|
|
}
|
|
return errors;
|
|
}
|
|
|
|
function actionCounts(actions) {
|
|
const counts = {};
|
|
for (const action of actions) {
|
|
const type = typeof action?.type === 'string' ? action.type : 'unknown';
|
|
counts[type] = (counts[type] ?? 0) + 1;
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
function restartRecoveryValid(actions, traces) {
|
|
const staleIndex = actions.findIndex(
|
|
(action) =>
|
|
action.type === 'set_value' &&
|
|
action.success === false &&
|
|
action.expectedFailure === true &&
|
|
action.resultCode === 'target_missing',
|
|
);
|
|
if (staleIndex < 0) return false;
|
|
const stale = actions[staleIndex];
|
|
const freshIndex = actions.findIndex(
|
|
(action, index) =>
|
|
index > staleIndex &&
|
|
action.type === 'observe' &&
|
|
action.success === true &&
|
|
typeof action.resultObservationId === 'string' &&
|
|
(action.targetPid !== stale.targetPid || action.targetWindowId !== stale.targetWindowId),
|
|
);
|
|
if (freshIndex < 0) return false;
|
|
const fresh = actions[freshIndex];
|
|
const retry = actions.find(
|
|
(action, index) =>
|
|
index > freshIndex &&
|
|
action.type === 'set_value' &&
|
|
action.success === true &&
|
|
action.sourceObservationId === fresh.resultObservationId &&
|
|
action.targetPid === fresh.targetPid &&
|
|
action.targetWindowId === fresh.targetWindowId,
|
|
);
|
|
if (!retry) return false;
|
|
return (
|
|
Array.isArray(traces) &&
|
|
traces.some(
|
|
(trace) =>
|
|
trace.type === 'dispatch' &&
|
|
trace.toolCallId === retry.toolCallId &&
|
|
trace.actionType === 'set_value' &&
|
|
trace.pid === retry.targetPid &&
|
|
trace.windowId === retry.targetWindowId &&
|
|
trace.address === 'ax',
|
|
)
|
|
);
|
|
}
|
|
|
|
function actionHasOwnedFixtureTrace(action, fixtureIdentity) {
|
|
return (
|
|
action.targetOwned === true &&
|
|
Array.isArray(fixtureIdentity?.instances) &&
|
|
fixtureIdentity.instances.some(
|
|
(instance) =>
|
|
action.targetPid === instance.pid &&
|
|
Array.isArray(instance.windowIds) &&
|
|
instance.windowIds.includes(action.targetWindowId),
|
|
)
|
|
);
|
|
}
|
|
|
|
function actionResultAllowed(action, scenario) {
|
|
if (action.success === true) return true;
|
|
if (action.expectedFailure !== true || typeof action.resultCode !== 'string') return false;
|
|
return (
|
|
Array.isArray(scenario.expectedFailures) &&
|
|
scenario.expectedFailures.some(
|
|
(expected) => expected.action === action.type && expected.error === action.resultCode,
|
|
)
|
|
);
|
|
}
|
|
|
|
function actionLineageValid(actions) {
|
|
let latestObservationId;
|
|
let latestTarget;
|
|
for (const action of actions) {
|
|
if (
|
|
!ACTIONS_WITHOUT_OBSERVATION_LINEAGE.has(action.type) &&
|
|
(typeof action.sourceObservationId !== 'string' ||
|
|
action.sourceObservationId !== latestObservationId ||
|
|
action.targetPid !== latestTarget?.pid ||
|
|
action.targetWindowId !== latestTarget?.windowId)
|
|
) {
|
|
return false;
|
|
}
|
|
if (typeof action.resultObservationId === 'string') {
|
|
latestObservationId = action.resultObservationId;
|
|
latestTarget = { pid: action.targetPid, windowId: action.targetWindowId };
|
|
} else if (action.type === 'observe') {
|
|
return false;
|
|
}
|
|
if (action.expectedFailure === true && action.resultCode === 'target_missing') {
|
|
latestObservationId = undefined;
|
|
latestTarget = undefined;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function rowStatus(readiness, report, metrics) {
|
|
if (readiness === 'unsupported') return 'unsupported';
|
|
if (readiness === 'contract') return 'contract-only';
|
|
if (!report) return 'missing-report';
|
|
if (metrics.fixture.status === 'fail' || metrics.forbiddenEffects.status === 'fail')
|
|
return 'fail';
|
|
if (metrics.fixture.status === 'unknown' || metrics.forbiddenEffects.status === 'unknown')
|
|
return 'inconclusive';
|
|
if (report.policyMode === 'bypassed') return 'pass-policy-bypassed';
|
|
return 'pass';
|
|
}
|
|
|
|
async function readJson(path) {
|
|
return JSON.parse(await readFile(path, 'utf8'));
|
|
}
|
|
|
|
function resolveReportPath(rawPath, baseDir) {
|
|
if (!rawPath) return null;
|
|
return isAbsolute(rawPath) ? rawPath : resolve(baseDir, rawPath);
|
|
}
|
|
|
|
function reportTemplateFor(provider, scenario) {
|
|
return (
|
|
scenario.reports?.[provider.id] ??
|
|
select(provider.reports, scenario.id) ??
|
|
select(provider.reportTemplate ?? provider.report, scenario.id) ??
|
|
select(scenario.reportTemplate ?? scenario.report, provider.id)
|
|
);
|
|
}
|
|
|
|
export async function buildProviderMatrix({
|
|
scenarios,
|
|
providers,
|
|
baseDir = process.cwd(),
|
|
generatedAt = new Date().toISOString(),
|
|
loadReport = readJson,
|
|
}) {
|
|
const scenarioIds = new Set();
|
|
for (const scenario of scenarios) {
|
|
const id = requireId(scenario, 'scenario');
|
|
if (scenarioIds.has(id)) throw new Error(`duplicate scenario id: ${id}`);
|
|
scenarioIds.add(id);
|
|
}
|
|
const providerIds = new Set();
|
|
for (const provider of providers) {
|
|
const id = requireId(provider, 'provider');
|
|
if (providerIds.has(id)) throw new Error(`duplicate provider id: ${id}`);
|
|
providerIds.add(id);
|
|
}
|
|
|
|
const rows = [];
|
|
for (const provider of providers) {
|
|
for (const scenario of scenarios) {
|
|
const readiness = select(provider.readiness, scenario.id);
|
|
if (!READINESS.has(readiness)) {
|
|
throw new Error(
|
|
`provider ${provider.id} scenario ${scenario.id} has invalid readiness ${JSON.stringify(readiness)}`,
|
|
);
|
|
}
|
|
const producer = select(provider.producer, scenario.id);
|
|
if (readiness === 'real' && !REAL_REPORT_PRODUCERS.has(producer)) {
|
|
throw new Error(
|
|
`provider ${provider.id} scenario ${scenario.id} real readiness requires an explicit known producer`,
|
|
);
|
|
}
|
|
const model = select(provider.model ?? provider.modelId, scenario.id);
|
|
if (readiness === 'real' && (typeof model !== 'string' || !model)) {
|
|
throw new Error(
|
|
`provider ${provider.id} scenario ${scenario.id} real readiness requires an explicit model`,
|
|
);
|
|
}
|
|
const reportTemplate = reportTemplateFor(provider, scenario);
|
|
const variables = {
|
|
provider: provider.id,
|
|
providerId: provider.id,
|
|
scenario: scenario.id,
|
|
scenarioId: scenario.id,
|
|
prompt: scenario.prompt ?? '',
|
|
report: reportTemplate ?? '',
|
|
};
|
|
const commandTemplate = select(provider.commandTemplate ?? provider.command, scenario.id);
|
|
const command = displayCommand(renderTemplate(commandTemplate, variables));
|
|
const renderedReport = renderTemplate(reportTemplate, variables);
|
|
const reportPath = resolveReportPath(renderedReport, baseDir);
|
|
let report = null;
|
|
let reportError = null;
|
|
if (readiness === 'real' && reportPath) {
|
|
try {
|
|
report = await loadReport(reportPath);
|
|
const validationErrors = validateRealReport(report, provider, scenario);
|
|
if (validationErrors.length > 0) {
|
|
reportError = validationErrors.join('; ');
|
|
report = null;
|
|
}
|
|
} catch (error) {
|
|
if (error?.code !== 'ENOENT')
|
|
reportError = error instanceof Error ? error.message : String(error);
|
|
}
|
|
}
|
|
const metrics = report
|
|
? normalizeReport(report, scenario)
|
|
: {
|
|
evidenceClass: null,
|
|
policyMode: null,
|
|
modelLatency: null,
|
|
toolLatency: null,
|
|
displayLag: null,
|
|
actionCount: null,
|
|
retries: null,
|
|
fixture: {
|
|
status: scenario.fixture == null ? 'not-defined' : 'unknown',
|
|
expected: scenario.fixture?.expected ?? scenario.fixture ?? null,
|
|
actual: null,
|
|
},
|
|
forbiddenEffects: {
|
|
status: scenario.forbiddenEffects == null ? 'not-defined' : 'unknown',
|
|
forbidden: scenario.forbiddenEffects ?? [],
|
|
observed: [],
|
|
violations: [],
|
|
},
|
|
};
|
|
rows.push({
|
|
providerId: provider.id,
|
|
provider: provider.label ?? provider.name ?? provider.id,
|
|
scenarioId: scenario.id,
|
|
scenario: scenario.label ?? scenario.name ?? scenario.id,
|
|
readiness,
|
|
status: reportError ? 'invalid-report' : rowStatus(readiness, report, metrics),
|
|
command,
|
|
reportPath,
|
|
reportError,
|
|
...metrics,
|
|
});
|
|
}
|
|
}
|
|
|
|
const readinessCounts = Object.fromEntries(
|
|
[...READINESS].map((readiness) => [
|
|
readiness,
|
|
rows.filter((row) => row.readiness === readiness).length,
|
|
]),
|
|
);
|
|
const statusCounts = {};
|
|
for (const row of rows) statusCounts[row.status] = (statusCounts[row.status] ?? 0) + 1;
|
|
return {
|
|
schemaVersion: 1,
|
|
generatedAt,
|
|
summary: {
|
|
providers: providers.length,
|
|
scenarios: scenarios.length,
|
|
cells: rows.length,
|
|
readiness: readinessCounts,
|
|
status: statusCounts,
|
|
},
|
|
rows,
|
|
};
|
|
}
|
|
|
|
function latencyCell(summary) {
|
|
return summary ? `${summary.p50Ms}/${summary.p95Ms}/${summary.averageMs} ms` : '-';
|
|
}
|
|
|
|
function escapeCell(value) {
|
|
return String(value ?? '-')
|
|
.replace(/\|/g, '\\|')
|
|
.replace(/\r?\n/g, '<br>');
|
|
}
|
|
|
|
export function renderMarkdown(matrix) {
|
|
const lines = [
|
|
'# Computer Use Provider E2E Matrix',
|
|
'',
|
|
`Generated: ${matrix.generatedAt}`,
|
|
'',
|
|
`Providers: ${matrix.summary.providers} | Scenarios: ${matrix.summary.scenarios} | Cells: ${matrix.summary.cells}`,
|
|
'',
|
|
'| Provider | Scenario | Readiness | Evidence | Policy | Status | Model p50/p95/avg | Tool p50/p95/avg | Display p50/p95/avg | Actions | Retries | Fixture | Forbidden effects |',
|
|
'| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |',
|
|
];
|
|
for (const row of matrix.rows) {
|
|
lines.push(
|
|
[
|
|
row.provider,
|
|
row.scenario,
|
|
row.readiness,
|
|
row.evidenceClass,
|
|
row.policyMode,
|
|
row.status,
|
|
latencyCell(row.modelLatency),
|
|
latencyCell(row.toolLatency),
|
|
latencyCell(row.displayLag),
|
|
row.actionCount,
|
|
row.retries,
|
|
row.fixture.status,
|
|
row.forbiddenEffects.status,
|
|
]
|
|
.map(escapeCell)
|
|
.join(' | ')
|
|
.replace(/^/, '| ')
|
|
.replace(/$/, ' |'),
|
|
);
|
|
}
|
|
lines.push('', '## Commands', '');
|
|
for (const row of matrix.rows) {
|
|
lines.push(
|
|
`- **${row.provider} / ${row.scenario}**: ${row.command ? `\`${row.command.replace(/`/g, '\\`')}\`` : '_none_'}`,
|
|
);
|
|
}
|
|
lines.push('');
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
export async function runCli(argv = process.argv.slice(2)) {
|
|
if (argv.includes('--help') || argv.includes('-h')) {
|
|
process.stdout.write(
|
|
'Usage: node scripts/computer-use.mjs provider-matrix --scenarios scenarios.json ' +
|
|
'--providers providers.json --json matrix.json --markdown matrix.md\n',
|
|
);
|
|
return;
|
|
}
|
|
const scenariosPath = resolve(requiredOption(argv, ['--scenarios']));
|
|
const providersPath = resolve(requiredOption(argv, ['--providers']));
|
|
const jsonPath = resolve(requiredOption(argv, ['--json', '--out-json']));
|
|
const markdownPath = resolve(requiredOption(argv, ['--markdown', '--out-markdown']));
|
|
const [scenarioInput, providerInput] = await Promise.all([
|
|
readJson(scenariosPath),
|
|
readJson(providersPath),
|
|
]);
|
|
const scenarios = asEntries(scenarioInput, 'scenarios', scenariosPath);
|
|
const providers = asEntries(providerInput, 'providers', providersPath);
|
|
const matrix = await buildProviderMatrix({
|
|
scenarios,
|
|
providers,
|
|
baseDir: dirname(scenariosPath),
|
|
});
|
|
await Promise.all([
|
|
mkdir(dirname(jsonPath), { recursive: true }),
|
|
mkdir(dirname(markdownPath), { recursive: true }),
|
|
]);
|
|
await Promise.all([
|
|
writeFile(jsonPath, `${JSON.stringify(matrix, null, 2)}\n`, 'utf8'),
|
|
writeFile(markdownPath, renderMarkdown(matrix), 'utf8'),
|
|
]);
|
|
process.stdout.write(
|
|
`Computer Use provider matrix: ${jsonPath}\nMarkdown report: ${markdownPath}\n`,
|
|
);
|
|
}
|
|
|
|
const isMain = process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href;
|
|
if (isMain) {
|
|
runCli().catch((error) => {
|
|
console.error(
|
|
`Computer Use provider matrix failed: ${error instanceof Error ? error.message : error}`,
|
|
);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|