/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import { isPartialRuntimeEvent, isTerminalRuntimeEvent } from '@maka/core/runtime-event'; import type { AgentRunEvent, AgentRunHeader, AgentRunEventType, AgentRunStore, } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import { classifyRuntimeEventTerminalFact, type RuntimeEventTerminalFact, } from './runtime-event-read-model.js'; export type TerminalAgentRunStatus = Extract< AgentRunHeader['status'], 'completed' | 'failed' | 'cancelled' >; export type TerminalRuntimeLedgerClassification = | { kind: 'fact'; fact: RuntimeEventTerminalFact; terminalEvents: readonly RuntimeEvent[]; } | { kind: 'none'; terminalEvents: readonly RuntimeEvent[]; } | { kind: 'incomplete_single_terminal'; terminalEvent: RuntimeEvent; terminalEvents: readonly RuntimeEvent[]; } | { kind: 'ambiguous'; terminalEvents: readonly RuntimeEvent[]; }; export function classifyTerminalRuntimeLedger( run: AgentRunHeader, events: readonly RuntimeEvent[], ): TerminalRuntimeLedgerClassification { const terminalEvents = matchingTerminalRuntimeEvents(run, events); if (terminalEvents.length === 0) { return { kind: 'none', terminalEvents }; } if (terminalEvents.length > 1) { return { kind: 'ambiguous', terminalEvents }; } const fact = classifyRuntimeEventTerminalFact(run, events).fact; if (fact) { return { kind: 'fact', fact, terminalEvents }; } return { kind: 'incomplete_single_terminal', terminalEvent: terminalEvents[0]!, terminalEvents, }; } export interface CommitTerminalRunWithRuntimeFactInput { runStore: AgentRunStore; runtimeEventStore: RuntimeEventStore; newId: () => string; sessionId: string; runId: string; turnId: string; status: TerminalAgentRunStatus; ts: number; terminalEvent: RuntimeEvent; failureClass?: string; failureMessage?: string; traceWriteError?: string; abortSource?: string; runEventData?: Record; runEventMessage?: string; existingEvents?: readonly Pick[]; } export async function commitTerminalRunWithRuntimeFact( input: CommitTerminalRunWithRuntimeFactInput, ): Promise { if (isPartialRuntimeEvent(input.terminalEvent)) { throw new Error('terminal RuntimeEvent must be final before terminal run header'); } const terminalStatus = terminalRunStatusFromRuntimeEvent(input.terminalEvent); if (!terminalStatus) { throw new Error('terminal RuntimeEvent must carry a terminal status'); } if (terminalStatus !== input.status) { throw new Error( `terminal RuntimeEvent status ${input.terminalEvent.status} cannot commit ${input.status} run header`, ); } if ( input.terminalEvent.sessionId !== input.sessionId || input.terminalEvent.runId !== input.runId || input.terminalEvent.turnId !== input.turnId ) { throw new Error('terminal RuntimeEvent identity does not match run header commit'); } await input.runtimeEventStore.ensureTerminalRuntimeEventDurable( input.sessionId, input.runId, input.terminalEvent, ); await commitTerminalRunProjection(input); } async function commitTerminalRunProjection( input: CommitTerminalRunWithRuntimeFactInput, ): Promise { const failureClass = input.status === 'failed' ? (input.failureClass ?? 'unknown') : undefined; const abortSource = input.status === 'cancelled' ? input.abortSource : undefined; await input.runStore.updateRun( input.sessionId, input.runId, { status: input.status, updatedAt: input.ts, completedAt: input.ts, ...(failureClass ? { failureClass } : {}), ...(input.failureMessage ? { failureMessage: input.failureMessage } : {}), ...(input.traceWriteError ? { traceWriteError: input.traceWriteError } : {}), ...(abortSource ? { abortSource } : {}), }, { durable: true }, ); if (hasTerminalAgentRunEvent(input.existingEvents ?? [])) return; const data = terminalRunEventData(input.status, failureClass, input.runEventData); await input.runStore.appendEvent( input.sessionId, input.runId, { type: terminalAgentRunEventType(input.status), id: input.newId(), runId: input.runId, sessionId: input.sessionId, turnId: input.turnId, ts: input.ts, ...(input.runEventMessage ? { message: input.runEventMessage } : {}), ...(Object.keys(data).length > 0 ? { data } : {}), }, { durable: true }, ); } export interface CommitOrCreateTerminalRunFactInput extends Omit { /** Runs after the terminal durability barrier, before the header commit. */ afterTerminalDurable?: () => Promise; terminalEvent?: RuntimeEvent; allowHeaderCommitFailure?: boolean; fallbackStatus: TerminalAgentRunStatus; fallbackInvocationId: string; fallbackFailureClass?: string; fallbackFailureMessage?: string; } export interface CommitOrCreateTerminalRunFactResult { terminalEvent: RuntimeEvent; status: TerminalAgentRunStatus; failureClass?: string; createdTerminalEvent: boolean; headerCommitted: boolean; headerCommitError?: unknown; } export async function commitOrCreateTerminalRunFact( input: CommitOrCreateTerminalRunFactInput, ): Promise { const createdTerminalEvent = !input.terminalEvent; const effectiveAbortSource = input.fallbackStatus === 'cancelled' ? (input.abortSource ?? 'user_stop') : input.abortSource; const terminalEvent = input.terminalEvent ?? buildSyntheticTerminalRuntimeEvent({ id: input.newId(), invocationId: input.fallbackInvocationId, run: { sessionId: input.sessionId, runId: input.runId, turnId: input.turnId, }, status: input.fallbackStatus, ts: input.ts, ...(input.fallbackFailureClass ? { failureClass: input.fallbackFailureClass } : {}), ...(effectiveAbortSource ? { abortSource: effectiveAbortSource } : {}), ...((input.fallbackFailureMessage ?? input.failureMessage) ? { message: input.fallbackFailureMessage ?? input.failureMessage } : {}), }); const status = terminalRunStatusFromRuntimeEvent(terminalEvent); if (!status) { throw new Error('terminal RuntimeEvent must carry a terminal status'); } if (isPartialRuntimeEvent(terminalEvent)) { throw new Error('terminal RuntimeEvent must be final before terminal run header'); } if ( terminalEvent.sessionId !== input.sessionId || terminalEvent.runId !== input.runId || terminalEvent.turnId !== input.turnId ) { throw new Error('terminal RuntimeEvent identity does not match run header commit'); } const failureClass = status === 'failed' ? (runtimeEventFailureClass(terminalEvent) ?? input.failureClass ?? 'unknown') : undefined; await input.runtimeEventStore.ensureTerminalRuntimeEventDurable( input.sessionId, input.runId, terminalEvent, ); // Between the terminal durability barrier and the header commit: the one // point where "the terminal fact is durable" is true and nothing else has // been projected yet. Callers that must order a crash boundary against // the barrier itself hang it here (#2313 corruption recovery, where the // claimed event's own write never ran). await input.afterTerminalDurable?.(); let headerCommitted = false; let headerCommitError: unknown; try { await commitTerminalRunProjection({ ...input, terminalEvent, status, ...(failureClass ? { failureClass } : {}), ...(effectiveAbortSource ? { abortSource: effectiveAbortSource } : {}), }); headerCommitted = true; } catch (error) { if (!input.allowHeaderCommitFailure) throw error; headerCommitError = error; } return { terminalEvent, status, ...(failureClass ? { failureClass } : {}), createdTerminalEvent, headerCommitted, ...(headerCommitError !== undefined ? { headerCommitError } : {}), }; } export interface BuildSyntheticTerminalRuntimeEventInput { id: string; invocationId: string; run: Pick; status: TerminalAgentRunStatus; ts: number; failureClass?: string; abortSource?: string; recoveryReason?: string; diagnostic?: Record; message?: string; } export function buildSyntheticTerminalRuntimeEvent( input: BuildSyntheticTerminalRuntimeEventInput, ): RuntimeEvent { const failureClass = input.status === 'failed' ? (input.failureClass ?? 'unknown') : undefined; const abortSource = input.status === 'cancelled' ? input.abortSource : undefined; return { id: input.id, invocationId: input.invocationId, runId: input.run.runId, sessionId: input.run.sessionId, turnId: input.run.turnId, ts: input.ts, partial: false, role: 'system', author: 'system', status: input.status === 'cancelled' ? 'aborted' : input.status, ...(failureClass ? { content: { kind: 'error', code: failureClass, reason: failureClass, message: input.message ?? failureClass, }, } : {}), actions: { endInvocation: true, stateDelta: { ...(input.recoveryReason ? { recovered: true, recoveryReason: input.recoveryReason } : {}), ...(input.diagnostic ?? {}), ...(failureClass ? { failureClass } : {}), ...(abortSource ? { abortSource } : {}), }, }, }; } export interface BuildRecoveredTerminalRuntimeEventInput { id: string; run: Pick; status: TerminalAgentRunStatus; ts: number; invocationId?: string; failureClass?: string; abortSource?: string; recoveryReason: string; diagnostic?: Record; message?: string; } export function buildRecoveredTerminalRuntimeEvent( input: BuildRecoveredTerminalRuntimeEventInput, ): RuntimeEvent { return buildSyntheticTerminalRuntimeEvent({ id: input.id, invocationId: input.run.invocationId ?? input.invocationId ?? `recovery-${input.run.runId}`, run: input.run, ts: input.ts, status: input.status, ...(input.failureClass ? { failureClass: input.failureClass } : {}), ...(input.status === 'cancelled' ? { abortSource: input.abortSource ?? 'unknown' } : {}), recoveryReason: input.recoveryReason, ...(input.diagnostic ? { diagnostic: input.diagnostic } : {}), ...(input.message ? { message: input.message } : {}), }); } export function hasTerminalAgentRunEvent(events: readonly Pick[]): boolean { return events.some( (event) => event.type === 'run_completed' || event.type === 'run_failed' || event.type === 'run_cancelled', ); } function terminalAgentRunEventType(status: TerminalAgentRunStatus): AgentRunEventType { if (status === 'cancelled') return 'run_cancelled'; if (status === 'failed') return 'run_failed'; return 'run_completed'; } function terminalRunEventData( status: TerminalAgentRunStatus, failureClass: string | undefined, runEventData: Record | undefined, ): Record { return { ...(status === 'failed' && failureClass ? { failureClass } : {}), ...(runEventData ?? {}), }; } function runtimeEventFailureClass(event: RuntimeEvent): string | undefined { const stateDelta = event.actions?.stateDelta; if (typeof stateDelta?.failureClass === 'string' && stateDelta.failureClass.length > 0) { return stateDelta.failureClass; } if (event.content?.kind === 'error') { return event.content.code ?? event.content.reason; } return undefined; } export function terminalRunStatusFromRuntimeEvent( event: RuntimeEvent, ): TerminalAgentRunStatus | undefined { if (event.status === 'completed') return 'completed'; if (event.status === 'failed') return 'failed'; if (event.status === 'aborted' || event.status === 'cancelled') return 'cancelled'; return undefined; } export function effectiveRunHeaderFromTerminalFact( run: AgentRunHeader, fact: RuntimeEventTerminalFact, ): AgentRunHeader { const completedAt = run.completedAt ?? fact.terminalEvent.ts; const base = { ...run }; delete base.failureClass; delete base.failureMessage; delete base.abortSource; return { ...base, status: fact.runStatus, updatedAt: Math.max(run.updatedAt, completedAt), completedAt, ...(fact.runStatus === 'failed' && fact.failureClass ? { failureClass: fact.failureClass } : {}), ...(fact.runStatus === 'failed' && run.failureMessage ? { failureMessage: run.failureMessage } : {}), ...(fact.runStatus === 'cancelled' && fact.abortSource ? { abortSource: fact.abortSource } : {}), }; } export function terminalRunHeaderMatchesFact( run: AgentRunHeader, fact: RuntimeEventTerminalFact, ): boolean { if (run.status !== fact.runStatus) return false; if (fact.runStatus === 'failed' && run.failureClass !== fact.failureClass) return false; if (fact.runStatus === 'cancelled' && run.abortSource !== fact.abortSource) return false; return true; } export function matchingTerminalRuntimeEvents( run: AgentRunHeader, events: readonly RuntimeEvent[], ): RuntimeEvent[] { return events.filter( (event) => !isPartialRuntimeEvent(event) && event.sessionId === run.sessionId && event.runId === run.runId && event.turnId === run.turnId && isTerminalRuntimeEvent(event), ); }