Files
maka/packages/storage/src/__tests__/recovery-persistence-authority.test.ts
T
agent-radar 7be535059e
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s
源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
2026-09-01 19:05:14 +08:00

1212 lines
40 KiB
TypeScript

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import assert from 'node:assert/strict';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { describe, it } from 'node:test';
import { TOOL_RECOVERY_BUNDLE_CAPABILITY_V1 } from '@maka/core/runtime-event-store';
import { type RuntimeEvent } from '@maka/core/runtime-event';
import { canonicalToolArgsHash } from '@maka/core/tool-args-identity';
import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js';
import type { SqliteRuntimeStoreFailpoint } from '../sqlite-runtime-store.js';
const ARGS_HASH = canonicalToolArgsHash('Write', {
path: 'notes.txt',
content: 'after',
});
// Mainline schema 4 persisted stableHash({ toolName, args }) for this exact
// provider call. Keep the literal independent from the current hash helper so
// an accidental identity-epoch change remains observable during migration.
const MAINLINE_SCHEMA_4_SPECIAL_ARGS_HASH =
'sha256:0002fcd132216e3442d4ab7579d9659019019c6b7000456fbef198b7ae53ee43';
const OBSERVATION_DIGEST =
'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' as const;
describe('SQLite recovery persistence authority', () => {
it('rebuilds a real schema 4 T1 dispatch written with the mainline args identity', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-recovery-mainline-t1-upgrade-'));
const dbPath = join(root, 'runtime.sqlite');
const store = createSqliteRuntimeStore(dbPath);
store.close();
const call = baseEvent({
id: 'schema-4-call',
role: 'model',
author: 'agent',
content: {
kind: 'function_call',
id: 'schema-4-provider-call',
name: 'X',
args: { required: ['b', 'a'] },
},
});
const dispatch = baseEvent({
id: 'schema-4-dispatch',
ts: 2,
actions: {
toolDispatch: {
protocol: 't1_after_preflight_v1',
operationId: 'schema-4-operation',
providerToolCallId: 'schema-4-provider-call',
toolName: 'X',
canonicalArgsHash: MAINLINE_SCHEMA_4_SPECIAL_ARGS_HASH,
recoveryMode: 'replay_safe',
},
},
refs: {
operationId: 'schema-4-operation',
toolCallId: 'schema-4-provider-call',
},
});
const db = new DatabaseSync(dbPath);
const insertEvent = db.prepare(`
INSERT INTO runtime_events(
event_id, session_id, invocation_id, run_id, turn_id,
event_seq, event_kind, payload_json, committed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
insertEvent.run(
call.id,
call.sessionId,
call.invocationId,
call.runId,
call.turnId,
1,
'function_call',
JSON.stringify(call),
call.ts,
);
insertEvent.run(
dispatch.id,
dispatch.sessionId,
dispatch.invocationId,
dispatch.runId,
dispatch.turnId,
2,
'tool_dispatch',
JSON.stringify(dispatch),
dispatch.ts,
);
db.exec(
'DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;',
);
db.close();
try {
const upgraded = createSqliteRuntimeStore(dbPath);
try {
assert.deepEqual(await upgraded.rebuildToolProjectionsFromRuntimeEvents(), {
operations: 1,
journalEvents: 1,
});
assert.deepEqual(await upgraded.readToolOperation('schema-4-operation'), {
operationId: 'schema-4-operation',
invocationId: 'invocation-1',
runId: 'run-1',
turnId: 'turn-1',
providerToolCallId: 'schema-4-provider-call',
toolName: 'X',
canonicalArgsHash: MAINLINE_SCHEMA_4_SPECIAL_ARGS_HASH,
recoveryMode: 'replay_safe',
currentState: 'prepared',
callEventId: 'schema-4-call',
dispatchEventId: 'schema-4-dispatch',
version: 1,
});
} finally {
upgraded.close();
}
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('upgrades and quarantines populated mainline schema 4 tool projections', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-recovery-mainline-upgrade-'));
const dbPath = join(root, 'runtime.sqlite');
const store = createSqliteRuntimeStore(dbPath);
const preparedCall = callEvent();
preparedCall.id = 'legacy-prepared-call';
preparedCall.content = {
kind: 'function_call',
id: 'legacy-prepared-provider-call',
name: 'Read',
args: { path: 'prepared.txt' },
};
const completedCall = callEvent();
completedCall.id = 'legacy-completed-call';
completedCall.content = {
kind: 'function_call',
id: 'legacy-completed-provider-call',
name: 'Read',
args: { path: 'completed.txt' },
};
const completedResponse = outcomeEvent();
completedResponse.id = 'legacy-completed-response';
completedResponse.content = {
kind: 'function_response',
id: 'legacy-completed-provider-call',
name: 'Read',
result: 'contents',
};
completedResponse.refs = { toolCallId: 'legacy-completed-provider-call' };
await store.importRuntimeEventsBatch({
sessionId: 'session-1',
runId: 'run-1',
events: [preparedCall, completedCall, completedResponse],
});
store.close();
const db = new DatabaseSync(dbPath);
const insertOperation = db.prepare(`
INSERT INTO tool_operations(
operation_id, invocation_id, run_id, turn_id, provider_tool_call_id,
tool_name, canonical_args_hash, recovery_mode, current_state,
call_event_id, dispatch_event_id, result_event_id, version
) VALUES (?, 'invocation-1', 'run-1', 'turn-1', ?, 'Read', ?, 'replay_safe', ?, ?, NULL, ?, ?)
`);
insertOperation.run(
'legacy-prepared-operation',
'legacy-prepared-provider-call',
canonicalToolArgsHash('Read', { path: 'prepared.txt' }),
'prepared',
'legacy-prepared-call',
null,
1,
);
insertOperation.run(
'legacy-completed-operation',
'legacy-completed-provider-call',
canonicalToolArgsHash('Read', { path: 'completed.txt' }),
'outcome_committed',
'legacy-completed-call',
'legacy-completed-response',
2,
);
db.exec(
'DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;',
);
db.close();
try {
const upgraded = createSqliteRuntimeStore(dbPath);
try {
assert.equal(
(await upgraded.readToolOperation('legacy-prepared-operation'))?.currentState,
'prepared',
);
assert.equal(
(await upgraded.readToolOperation('legacy-completed-operation'))?.currentState,
'outcome_committed',
);
assert.deepEqual(await upgraded.listUnsettledToolOperations(), []);
assert.deepEqual(await upgraded.rebuildToolProjectionsFromRuntimeEvents(), {
operations: 0,
journalEvents: 0,
});
assert.equal(
(await upgraded.readToolOperation('legacy-prepared-operation'))?.currentState,
'prepared',
);
assert.equal(
(await upgraded.readToolOperation('legacy-completed-operation'))?.currentState,
'outcome_committed',
);
assert.equal(upgraded.recoveryBundleCapability, TOOL_RECOVERY_BUNDLE_CAPABILITY_V1);
} finally {
upgraded.close();
}
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects the old experimental capability marker instead of guessing compatibility', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-recovery-epoch-'));
const dbPath = join(root, 'runtime.sqlite');
const store = createSqliteRuntimeStore(dbPath);
store.close();
const db = new DatabaseSync(dbPath);
db.prepare('DELETE FROM runtime_capabilities').run();
db.prepare('INSERT INTO runtime_capabilities(capability, version) VALUES (?, ?)').run(
'tool_recovery_bundle',
1,
);
db.close();
try {
assert.throws(
() => createSqliteRuntimeStore(dbPath),
/runtime_recovery_authority@1 is unavailable/,
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('rejects reserved recovery facts through every generic append path', async () => {
await withStore(async (store) => {
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', reconcileEvent()),
/atomic recovery bundle writer/,
);
await assert.rejects(
store.importRuntimeEventsBatch({
sessionId: 'session-1',
runId: 'run-1',
events: [reconcileEvent()],
}),
/atomic recovery bundle writer/,
);
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', dispatchEvent()),
/atomic tool boundary writer/,
);
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', outcomeEvent()),
/atomic tool boundary writer/,
);
});
});
it('recomputes T1 identity from the persisted function-call arguments', async () => {
await withStore(async (store) => {
const dispatch = dispatchEvent();
dispatch.actions!.toolDispatch!.canonicalArgsHash = 'sha256:wrong';
await assert.rejects(
store.commitToolPrepared({
operationId: 'operation-1',
journalEventId: 'operation-1_prepared',
runtimeEvent: callEvent(),
dispatchRuntimeEvent: dispatch,
providerToolCallId: 'provider-call-1',
toolName: 'Write',
canonicalArgsHash: 'sha256:wrong',
recoveryMode: 'reconcile',
committedAt: 10,
}),
/canonical function call/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), []);
});
});
it('derives projection-local journal identities instead of trusting callers', async () => {
await withStore(async (store) => {
await assert.rejects(
store.commitToolPrepared({
operationId: 'operation-1',
journalEventId: 'caller-selected-id',
runtimeEvent: callEvent(),
dispatchRuntimeEvent: dispatchEvent(),
providerToolCallId: 'provider-call-1',
toolName: 'Write',
canonicalArgsHash: ARGS_HASH,
recoveryMode: 'reconcile',
committedAt: 10,
}),
/journal identity/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), []);
});
});
it('rejects duplicate call identities through the generic writer', async () => {
await withStore(async (store) => {
await store.appendRuntimeEvent('session-1', 'run-1', callEvent());
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', {
...callEvent(),
id: 'call-event-duplicate',
}),
/duplicate_call/,
);
assert.deepEqual(
(await store.readImmutableRuntimeEvents('session-1', 'run-1')).map(({ id }) => id),
['call-event-1'],
);
});
});
it('rejects one invocation identity crossing run, turn, or session boundaries', async () => {
await withStore(async (store) => {
await store.appendRuntimeEvent('session-1', 'run-1', userEvent());
for (const event of [
baseEvent({
id: 'other-run-event',
runId: 'run-2',
content: { kind: 'text', text: 'other run' },
}),
baseEvent({
id: 'other-turn-event',
turnId: 'turn-2',
content: { kind: 'text', text: 'other turn' },
}),
baseEvent({
id: 'other-session-event',
sessionId: 'session-2',
content: { kind: 'text', text: 'other session' },
}),
]) {
await assert.rejects(
store.appendRuntimeEvent(event.sessionId, event.runId, event),
/invocation identity conflict/,
);
}
assert.deepEqual(
(await store.readImmutableRuntimeEvents('session-1', 'run-1')).map((event) => event.id),
['user-event-1'],
);
});
});
it('rejects an unbound generic response after T1', async () => {
await withStore(async (store) => {
await prepare(store);
const unboundResponse = outcomeEvent();
unboundResponse.refs = { toolCallId: 'provider-call-1' };
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', unboundResponse),
/identity_conflict/,
);
assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared');
});
});
it('rejects a T1 dispatch after a pre-T1 synthetic response settled the call', async () => {
await withStore(async (store) => {
const unboundResponse = outcomeEvent();
unboundResponse.refs = { toolCallId: 'provider-call-1' };
await store.importRuntimeEventsBatch({
sessionId: 'session-1',
runId: 'run-1',
events: [callEvent(), unboundResponse],
});
await assert.rejects(
store.commitToolPrepared({
operationId: 'operation-1',
journalEventId: 'operation-1_prepared',
runtimeEvent: callEvent(),
dispatchRuntimeEvent: dispatchEvent(),
providerToolCallId: 'provider-call-1',
toolName: 'Write',
canonicalArgsHash: ARGS_HASH,
recoveryMode: 'reconcile',
committedAt: 10,
}),
/event_order_conflict/,
);
assert.equal(await store.readToolOperation('operation-1'), undefined);
});
});
it('rejects an out-of-order tool-bearing batch import atomically', async () => {
await withStore(async (store) => {
const unboundResponse = outcomeEvent();
unboundResponse.refs = { toolCallId: 'provider-call-1' };
await assert.rejects(
store.importRuntimeEventsBatch({
sessionId: 'session-1',
runId: 'run-1',
events: [unboundResponse, callEvent()],
}),
/orphan_response/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), []);
});
});
it('rejects a T2 row that also claims another authoritative semantic lane', async () => {
await withStore(async (store) => {
await prepare(store);
const outcome = outcomeEvent();
outcome.actions = { endInvocation: true };
await assert.rejects(
store.commitToolOutcome({
operationId: 'operation-1',
journalEventId: 'operation-1_outcome',
runtimeEvent: outcome,
committedAt: 20,
}),
/semantic lane/,
);
assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared');
assert.deepEqual(
(await store.readImmutableRuntimeEvents('session-1', 'run-1')).map(({ id }) => id),
['call-event-1', 'dispatch-event-1'],
);
});
});
it('atomically settles completed recovery and rebuilds the same projection after reopen', async () => {
await withStore(async (store, dbPath) => {
assert.equal(store.recoveryBundleCapability, TOOL_RECOVERY_BUNDLE_CAPABILITY_V1);
await prepare(store);
await store.commitToolRecoveryBundle({
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decisionEvent(),
});
const onlineOperation = await store.readToolOperation('operation-1');
const onlineJournal = await store.readToolJournal('operation-1');
assert.equal(onlineOperation?.currentState, 'recovery_completed');
assert.equal(onlineOperation?.resultEventId, 'outcome-event-1');
assert.deepEqual(
onlineJournal.map(({ state }) => state),
['prepared', 'reconcile_observed', 'outcome_committed', 'recovery_completed'],
);
store.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
assert.deepEqual(await reopened.readToolOperation('operation-1'), onlineOperation);
assert.deepEqual(await reopened.readToolJournal('operation-1'), onlineJournal);
assert.deepEqual(await reopened.rebuildToolProjectionsFromRuntimeEvents(), {
operations: 1,
journalEvents: 4,
});
assert.deepEqual(await reopened.readToolOperation('operation-1'), onlineOperation);
assert.deepEqual(await reopened.readToolJournal('operation-1'), onlineJournal);
} finally {
reopened.close();
}
});
});
it('rebuilds interleaved operations to the same projections and journal identities', async () => {
await withStore(async (store) => {
await prepare(store);
const secondHash = canonicalToolArgsHash('Write', {
path: 'other.txt',
content: 'after',
});
const secondCall = callEvent();
secondCall.id = 'call-event-2';
secondCall.content = {
kind: 'function_call',
id: 'provider-call-2',
name: 'Write',
args: { path: 'other.txt', content: 'after' },
};
const secondDispatch = dispatchEvent();
secondDispatch.id = 'dispatch-event-2';
secondDispatch.actions!.toolDispatch = {
protocol: 't1_after_preflight_v1',
operationId: 'operation-2',
providerToolCallId: 'provider-call-2',
toolName: 'Write',
canonicalArgsHash: secondHash,
recoveryMode: 'reconcile',
};
secondDispatch.refs = { operationId: 'operation-2', toolCallId: 'provider-call-2' };
await store.commitToolPrepared({
operationId: 'operation-2',
journalEventId: 'operation-2_prepared',
runtimeEvent: secondCall,
dispatchRuntimeEvent: secondDispatch,
providerToolCallId: 'provider-call-2',
toolName: 'Write',
canonicalArgsHash: secondHash,
recoveryMode: 'reconcile',
committedAt: 11,
});
const secondOutcome = outcomeEvent();
secondOutcome.id = 'outcome-event-2';
secondOutcome.content = {
kind: 'function_response',
id: 'provider-call-2',
name: 'Write',
result: 'ok',
};
secondOutcome.refs = { operationId: 'operation-2', toolCallId: 'provider-call-2' };
await store.commitToolOutcome({
operationId: 'operation-2',
journalEventId: 'operation-2_outcome',
runtimeEvent: secondOutcome,
committedAt: 20,
});
await store.commitToolOutcome({
operationId: 'operation-1',
journalEventId: 'operation-1_outcome',
runtimeEvent: outcomeEvent(),
committedAt: 21,
});
const onlineOperations = await Promise.all([
store.readToolOperation('operation-1'),
store.readToolOperation('operation-2'),
]);
const onlineJournals = await Promise.all([
store.readToolJournal('operation-1'),
store.readToolJournal('operation-2'),
]);
await store.rebuildToolProjectionsFromRuntimeEvents();
assert.deepEqual(
await Promise.all([
store.readToolOperation('operation-1'),
store.readToolOperation('operation-2'),
]),
onlineOperations,
);
assert.deepEqual(
await Promise.all([
store.readToolJournal('operation-1'),
store.readToolJournal('operation-2'),
]),
onlineJournals,
);
});
});
it('rolls the whole recovery bundle back at every internal crash boundary', async () => {
for (const failpoint of [
'after_recovery_reconcile',
'after_recovery_outcome',
'after_recovery_decision',
] as const satisfies readonly SqliteRuntimeStoreFailpoint[]) {
const root = await mkdtemp(join(tmpdir(), 'maka-recovery-failpoint-'));
const dbPath = join(root, 'runtime.sqlite');
let active: SqliteRuntimeStoreFailpoint | undefined;
const store = createSqliteRuntimeStore(dbPath, {
failpoint: (point) => {
if (point === active) throw new Error(`recovery failpoint: ${point}`);
},
});
try {
await prepare(store);
active = failpoint;
await assert.rejects(
store.commitToolRecoveryBundle({
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decisionEvent(),
}),
new RegExp(failpoint),
);
active = undefined;
assert.deepEqual(
(await store.readImmutableRuntimeEvents('session-1', 'run-1')).map(({ id }) => id),
['call-event-1', 'dispatch-event-1'],
);
assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared');
assert.deepEqual(
(await store.readToolJournal('operation-1')).map(({ state }) => state),
['prepared'],
);
} finally {
store.close();
await rm(root, { recursive: true, force: true });
}
}
});
it('makes one parked bundle terminal and only permits its exact retry', async () => {
await withStore(async (store) => {
await prepare(store);
const parked = {
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent('diverged'),
decisionRuntimeEvent: decisionEvent('parked'),
} as const;
await store.commitToolRecoveryBundle(parked);
await store.commitToolRecoveryBundle(parked);
assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'recovery_parked');
assert.deepEqual(
(await store.readToolJournal('operation-1')).map(({ state }) => state),
['prepared', 'reconcile_observed', 'recovery_parked'],
);
await assert.rejects(
store.commitToolRecoveryBundle({
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decisionEvent(),
}),
/duplicate_event_id/,
);
});
});
it('fails immutable row/payload identity mismatches closed', async () => {
await withStore(async (store, dbPath) => {
await store.appendRuntimeEvent('session-1', 'run-1', userEvent());
store.close();
const db = new DatabaseSync(dbPath);
const payload = { ...userEvent(), runId: 'run-other' };
db.prepare('UPDATE runtime_events SET payload_json = ? WHERE event_id = ?').run(
JSON.stringify(payload),
'user-event-1',
);
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(
reopened.readImmutableRuntimeEvents('session-1', 'run-1'),
/row\/payload identity mismatch/,
);
} finally {
reopened.close();
}
});
});
it('persists the decoder canonical RuntimeEvent and makes raw retries idempotent', async () => {
await withStore(async (store) => {
const raw = userEvent();
raw.content = {
kind: 'text',
text: 'write it',
displayText: 'write it',
attachments: [],
quotes: [],
};
await store.appendRuntimeEvent('session-1', 'run-1', raw);
await store.appendRuntimeEvent('session-1', 'run-1', raw);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), [userEvent()]);
});
});
it('rejects RuntimeEvents that lose required fields during JSON serialization', async () => {
await withStore(async (store) => {
const lossy = callEvent();
if (lossy.content?.kind !== 'function_call') throw new Error('invalid fixture');
lossy.content.args = undefined;
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', lossy),
/RuntimeEvent schema|losslessly serializable/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), []);
});
});
it('rejects nested JSON loss in a function response before persistence', async () => {
await withStore(async (store) => {
await store.appendRuntimeEvent('session-1', 'run-1', callEvent());
const lossy = outcomeEvent();
if (lossy.content?.kind !== 'function_response') throw new Error('invalid fixture');
lossy.content.result = { value: undefined };
lossy.refs = { toolCallId: 'provider-call-1' };
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', lossy),
/losslessly serializable/,
);
assert.deepEqual(
(await store.readImmutableRuntimeEvents('session-1', 'run-1')).map((event) => event.id),
['call-event-1'],
);
});
});
it('rejects provider metadata that rewrites itself through toJSON', async () => {
await withStore(async (store) => {
const lossy = callEvent();
if (lossy.content?.kind !== 'function_call') throw new Error('invalid fixture');
lossy.content.providerOptions = {
value: 1,
toJSON() {
return { value: 2 };
},
};
await assert.rejects(
store.appendRuntimeEvent('session-1', 'run-1', lossy),
/losslessly serializable/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), []);
});
});
it('rejects recovery evidence whose JSON bytes can rewrite validated event ids', async () => {
await withStore(async (store) => {
await prepare(store);
const decision = decisionEvent();
const fact = decision.actions?.toolRecovery;
if (fact?.kind !== 'maka.tool.recovery_decision') throw new Error('invalid fixture');
Object.defineProperty(fact.payload.evidenceEventIds, 'toJSON', {
value: () => ['call-event-1', 'dispatch-event-1', 'reconcile-event-1', 'forged-outcome'],
});
await assert.rejects(
store.commitToolRecoveryBundle({
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decision,
}),
/losslessly serializable/,
);
assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared');
});
});
it('rejects row/payload identity corruption through the online bundle writer', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
store.close();
const db = new DatabaseSync(dbPath);
db.prepare(`UPDATE runtime_events SET run_id = 'run-other' WHERE event_id = ?`).run(
'call-event-1',
);
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(
reopened.commitToolRecoveryBundle({
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decisionEvent(),
}),
/row\/payload identity mismatch/,
);
assert.equal((await reopened.readToolOperation('operation-1'))?.currentState, 'prepared');
} finally {
reopened.close();
}
});
});
it('fails an exact T1 retry closed when the immutable ledger is already corrupt', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
store.close();
injectDuplicateCall(dbPath, 3);
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(prepare(reopened), /duplicate_call/);
} finally {
reopened.close();
}
});
});
it('fails an exact T2 retry closed when the immutable ledger is already corrupt', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
await store.commitToolOutcome({
operationId: 'operation-1',
journalEventId: 'operation-1_outcome',
runtimeEvent: outcomeEvent(),
committedAt: 20,
});
store.close();
injectDuplicateCall(dbPath, 4);
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(
reopened.commitToolOutcome({
operationId: 'operation-1',
journalEventId: 'operation-1_outcome',
runtimeEvent: outcomeEvent(),
committedAt: 20,
}),
/duplicate_call/,
);
} finally {
reopened.close();
}
});
});
it('fails an exact recovery retry closed when the immutable ledger is already corrupt', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
const bundle = {
operationId: 'operation-1',
reconcileRuntimeEvent: reconcileEvent(),
outcomeRuntimeEvent: outcomeEvent(),
decisionRuntimeEvent: decisionEvent(),
} as const;
await store.commitToolRecoveryBundle(bundle);
store.close();
injectDuplicateCall(dbPath, 6);
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(reopened.commitToolRecoveryBundle(bundle), /duplicate_call/);
} finally {
reopened.close();
}
});
});
it('fail-stops a new session tool boundary when another session ledger is corrupt', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
injectDuplicateCall(dbPath, 3);
const unrelatedCall = callEvent();
unrelatedCall.id = 'unrelated-call-event';
unrelatedCall.sessionId = 'session-2';
unrelatedCall.invocationId = 'invocation-2';
unrelatedCall.runId = 'run-2';
unrelatedCall.turnId = 'turn-2';
if (unrelatedCall.content?.kind !== 'function_call') {
throw new Error('invalid unrelated call fixture');
}
unrelatedCall.content.id = 'provider-call-2';
unrelatedCall.refs = { toolCallId: 'provider-call-2' };
await assert.rejects(
store.appendRuntimeEvent('session-2', 'run-2', unrelatedCall),
/duplicate_call/,
);
assert.deepEqual(await store.readImmutableRuntimeEvents('session-2', 'run-2'), []);
});
});
it('rejects duplicate call identity while rebuilding canonical projections', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
store.close();
const db = new DatabaseSync(dbPath);
const row = db
.prepare(`
SELECT session_id, invocation_id, run_id, turn_id, event_kind,
payload_json, committed_at
FROM runtime_events
WHERE event_id = 'call-event-1'
`)
.get() as {
session_id: string;
invocation_id: string;
run_id: string;
turn_id: string;
event_kind: string;
payload_json: string;
committed_at: number;
};
const payload = JSON.parse(row.payload_json) as RuntimeEvent;
payload.id = 'call-event-duplicate';
db.prepare(`
INSERT INTO runtime_events(
event_id, session_id, invocation_id, run_id, turn_id,
event_seq, event_kind, payload_json, committed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
payload.id,
row.session_id,
row.invocation_id,
row.run_id,
row.turn_id,
3,
row.event_kind,
JSON.stringify(payload),
row.committed_at,
);
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(reopened.rebuildToolProjectionsFromRuntimeEvents(), /duplicate_call/);
} finally {
reopened.close();
}
});
});
it('rejects dispatch-before-call physical order while rebuilding projections', async () => {
await withStore(async (store, dbPath) => {
await prepare(store);
store.close();
const db = new DatabaseSync(dbPath);
db.prepare(`UPDATE runtime_events SET event_seq = 100 WHERE event_id = 'call-event-1'`).run();
db.prepare(
`UPDATE runtime_events SET event_seq = 1 WHERE event_id = 'dispatch-event-1'`,
).run();
db.prepare(`UPDATE runtime_events SET event_seq = 2 WHERE event_id = 'call-event-1'`).run();
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
await assert.rejects(
reopened.rebuildToolProjectionsFromRuntimeEvents(),
/event_order_conflict/,
);
} finally {
reopened.close();
}
});
});
it('skips a corrupt mutable partial snapshot without hiding immutable history', async () => {
await withStore(async (store, dbPath) => {
await store.appendRuntimeEvent('session-1', 'run-1', userEvent());
await store.appendRuntimeEvent('session-1', 'run-1', partialTextEvent());
store.close();
const db = new DatabaseSync(dbPath);
db.prepare('UPDATE runtime_partial_snapshots SET payload_json = ?').run('{broken json');
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
assert.deepEqual(await reopened.readRuntimeEvents('session-1', 'run-1'), [userEvent()]);
} finally {
reopened.close();
}
});
});
it('skips a mutable partial whose SQL identity disagrees with its payload', async () => {
await withStore(async (store, dbPath) => {
await store.appendRuntimeEvent('session-1', 'run-1', userEvent());
await store.appendRuntimeEvent('session-1', 'run-1', partialTextEvent());
store.close();
const db = new DatabaseSync(dbPath);
db.prepare(`UPDATE runtime_partial_snapshots SET invocation_id = 'invocation-other'`).run();
db.close();
const reopened = createSqliteRuntimeStore(dbPath);
try {
assert.deepEqual(await reopened.readRuntimeEvents('session-1', 'run-1'), [userEvent()]);
} finally {
reopened.close();
}
});
});
});
type Store = ReturnType<typeof createSqliteRuntimeStore>;
function injectDuplicateCall(dbPath: string, eventSeq: number): void {
const db = new DatabaseSync(dbPath);
try {
const duplicate = callEvent();
duplicate.id = `call-event-duplicate-${eventSeq}`;
db.prepare(`
INSERT INTO runtime_events (
event_id, session_id, invocation_id, run_id, turn_id,
event_seq, event_kind, payload_json, committed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
duplicate.id,
duplicate.sessionId,
duplicate.invocationId,
duplicate.runId,
duplicate.turnId,
eventSeq,
'function_call',
JSON.stringify(duplicate),
duplicate.ts,
);
} finally {
db.close();
}
}
async function withStore(run: (store: Store, dbPath: string) => Promise<void>): Promise<void> {
const root = await mkdtemp(join(tmpdir(), 'maka-recovery-authority-'));
const dbPath = join(root, 'runtime.sqlite');
const store = createSqliteRuntimeStore(dbPath);
try {
await run(store, dbPath);
} finally {
store.close();
await rm(root, { recursive: true, force: true });
}
}
async function prepare(store: Store): Promise<void> {
await store.commitToolPrepared({
operationId: 'operation-1',
journalEventId: 'operation-1_prepared',
runtimeEvent: callEvent(),
dispatchRuntimeEvent: dispatchEvent(),
providerToolCallId: 'provider-call-1',
toolName: 'Write',
canonicalArgsHash: ARGS_HASH,
recoveryMode: 'reconcile',
committedAt: 10,
});
}
function baseEvent(overrides: Partial<RuntimeEvent>): RuntimeEvent {
return {
id: 'event-1',
invocationId: 'invocation-1',
runId: 'run-1',
sessionId: 'session-1',
turnId: 'turn-1',
ts: 1,
partial: false,
role: 'system',
author: 'system',
...overrides,
};
}
function userEvent(): RuntimeEvent {
return baseEvent({
id: 'user-event-1',
role: 'user',
author: 'user',
content: { kind: 'text', text: 'write it' },
});
}
function partialTextEvent(): RuntimeEvent {
return baseEvent({
id: 'partial-event-1',
ts: 2,
partial: true,
role: 'model',
author: 'agent',
content: { kind: 'text', text: 'in progress' },
refs: { providerEventId: 'provider-text-1' },
});
}
function callEvent(): RuntimeEvent {
return baseEvent({
id: 'call-event-1',
role: 'model',
author: 'agent',
content: {
kind: 'function_call',
id: 'provider-call-1',
name: 'Write',
args: { path: 'notes.txt', content: 'after' },
},
});
}
function dispatchEvent(): RuntimeEvent {
return baseEvent({
id: 'dispatch-event-1',
actions: {
toolDispatch: {
protocol: 't1_after_preflight_v1',
operationId: 'operation-1',
providerToolCallId: 'provider-call-1',
toolName: 'Write',
canonicalArgsHash: ARGS_HASH,
recoveryMode: 'reconcile',
},
},
refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' },
});
}
function reconcileEvent(
observation:
| 'matches_expected_state'
| 'matches_prior_state'
| 'diverged'
| 'unreadable' = 'matches_expected_state',
): RuntimeEvent {
return baseEvent({
id: 'reconcile-event-1',
ts: 2,
actions: {
toolRecovery: {
kind: 'maka.tool.reconcile_result',
version: 1,
payload: {
protocol: 'tool_reconcile_v1',
operationId: 'operation-1',
observation,
observationSchema: 'state_identity_v1',
observationDigest: OBSERVATION_DIGEST,
},
},
},
refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' },
});
}
function outcomeEvent(): RuntimeEvent {
return baseEvent({
id: 'outcome-event-1',
ts: 3,
role: 'tool',
author: 'tool',
content: {
kind: 'function_response',
id: 'provider-call-1',
name: 'Write',
result: 'ok',
isError: false,
},
refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' },
});
}
function decisionEvent(disposition: 'completed' | 'parked' = 'completed'): RuntimeEvent {
return baseEvent({
id: 'decision-event-1',
ts: 4,
actions: {
toolRecovery:
disposition === 'completed'
? {
kind: 'maka.tool.recovery_decision',
version: 1,
payload: {
protocol: 'tool_recovery_v1',
operationId: 'operation-1',
disposition: 'completed',
reasonCode: 'reconcile_matches_expected_state',
outcomeEventId: 'outcome-event-1',
evidenceEventIds: [
'call-event-1',
'dispatch-event-1',
'reconcile-event-1',
'outcome-event-1',
],
},
}
: {
kind: 'maka.tool.recovery_decision',
version: 1,
payload: {
protocol: 'tool_recovery_v1',
operationId: 'operation-1',
disposition: 'parked',
reasonCode: 'reconcile_diverged',
evidenceEventIds: ['call-event-1', 'dispatch-event-1', 'reconcile-event-1'],
},
},
},
refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' },
});
}