/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import type { AgentRunEvent, AgentRunHeader, AgentRunStore, EmittedAgentRunEvent, } from '@maka/core/agent-run'; import type { RuntimeEvent, ToolBoundaryProtocol } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import type { RunCompositionSnapshot } from '@maka/core/run-composition'; import { decodeRunCompositionSnapshot } from '@maka/core/run-composition'; import { DurableStoreWriteError, RunSealedError } from '@maka/core/runtime-event-store'; import { isSessionInlineRun } from '@maka/core/agent-run'; import { MODEL_PROJECTION_TRANSITION_EVENT_TYPE, type ModelProjectionTransition, } from '@maka/core/model-projection-transition'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; import { ToolLedgerCorruptionError, ToolLedgerRejectionError, } from '@maka/core/tool-ledger-scanner'; import type { ModelCallCommit } from '@maka/core/agent-run'; import { Buffer } from 'node:buffer'; import { isDeepStrictEqual } from 'node:util'; import { redactSecrets } from '@maka/core/redaction'; import { MODEL_CALL_ATTEMPT_EVENT_TYPE, type ModelCallAttempt, } from '@maka/core/model-call-attempt'; import type { SessionBlockedReason, SessionHeader, SessionHeaderPatch, SessionStatus, StoredMessage, SystemNoteMessage, TurnRecord, UserMessage, } from '@maka/core/session'; import type { UserMessageInput } from '@maka/core/runtime-inputs'; import { resolveEffectiveOrchestration, type EffectiveOrchestration, } from '@maka/core/orchestration'; import { messageContentsEqual, type SessionEvent } from '@maka/core/events'; import type { AgentBackend, BackendSendInput } from '@maka/core/backend-types'; import type { RunTraceEvent } from './run-trace.js'; import type { StopSessionInput } from './session-manager.js'; import type { HistoryCompactCheckpoint } from './history-compact-checkpoint.js'; import { projectRuntimeEventsToStoredMessages } from './runtime-event-read-model.js'; import { buildPriorRuntimeContext as buildPriorRuntimeContextProjection, type PriorRuntimeContext, } from './prior-run-context.js'; import { buildStatusPatch, isTerminalRunStatus, normalizeStopSessionSource, statusFromEvent, turnStatusFromEvent, } from './session-projection-helpers.js'; import { buildSyntheticTerminalRuntimeEvent, commitOrCreateTerminalRunFact, } from './terminal-run-commit.js'; import type { RuntimeContinuation } from './runtime-resume.js'; import { createRuntimeContinuationStartAdmissionProof, type RuntimeContinuationStartAdmissionProof, } from './runtime-continuation-admission.js'; import { DEFAULT_TOOL_MODE, isToolMode, type ToolMode } from '@maka/core/tool-mode'; import { materializeRuntimeEventTranscriptProjection } from './runtime-ledger-repair.js'; import { cloneAndFreezeRuntimeSnapshot } from './runtime-snapshot.js'; export interface AgentRunActiveSession { sessionId: string; backend: AgentBackend; cachedHeader: SessionHeader; activeRuns: Map; turnToRunId: Map; } export interface AgentRunHooks { reserveRun( sessionId: string, header: SessionHeader, run: AgentRun, ): Promise; unregisterRun(active: AgentRunActiveSession, run: AgentRun): void | Promise; updateHeader(sessionId: string, patch: SessionHeaderPatch): Promise; updateStatus( sessionId: string, status: SessionStatus, blockedReason?: SessionBlockedReason, ts?: number, ): Promise; appendTurnState( sessionId: string, turnId: string, status: TurnRecord['status'], lineage?: AgentRunLineage, options?: { ts?: number; errorClass?: string; abortSource?: string }, ): Promise; } export type AgentRunLineage = Partial< Pick & Pick< UserMessageInput, | 'parentTurnId' | 'retriedFromTurnId' | 'regeneratedFromTurnId' | 'branchOfTurnId' | 'parentSessionId' > >; export type AgentRunDurability = 'best_effort' | 'required'; export interface AgentRunInput { sessionId: string; header: SessionHeader; userInput: UserMessageInput; /** Internal lineage for runtime-owned continuations; never accepted by live turn input. */ runLineage?: Pick; rootExecutionKind?: AgentRunHeader['rootExecutionKind']; runId?: string; userMessageId?: string | null; durability?: AgentRunDurability; store: AgentRunSessionStore; runStore?: AgentRunStore; runtimeEventStore?: RuntimeEventStore; repairRunRuntimeLedger?: (sessionId: string, runId: string) => Promise; newId: () => string; now: () => number; workspaceIdentity?: string; continuationFailpoint?: (point: RuntimeContinuationFailpoint) => Promise; /** Exact target header already committed inside the durable continuation claim. */ claimedRunHeader?: AgentRunHeader; /** Commits the claimed continuation provider-call T1 after Run creation. */ commitContinuationStart?: (startedAt: number) => Promise<{ startEventId: string; created: true }>; hooks: AgentRunHooks; invocationId?: string; /** Pre-resolved snapshot used by continuations; normal turns derive it from header + input. */ effectiveOrchestration?: EffectiveOrchestration; /** Pre-resolved tool protocol used by continuations. */ effectiveToolMode?: ToolMode; /** Set only when this run's backend tool path is guarded by canonical T1. */ toolBoundaryProtocol?: ToolBoundaryProtocol; } export interface AgentRunSessionStore { appendMessage(sessionId: string, message: StoredMessage): Promise; readMessages(sessionId: string): Promise; } export type RuntimeContinuationFailpoint = | 'after_continuation_claim_committed' | 'after_run_created' | 'after_continuation_start_committed' | 'after_terminal_event_committed' | 'after_terminal_header_committed'; export class ContinuationStartCommitError extends Error { readonly name = 'ContinuationStartCommitError'; constructor(readonly storeCause: unknown) { super( `Continuation start was not durably committed: ${ storeCause instanceof Error ? storeCause.message : String(storeCause) }`, ); } } export interface AgentRunBeginResult { backend: AgentBackend; backendInput: BackendSendInput; initialRuntimeEvent: RuntimeEvent; } export interface AgentRunOperationBeginResult { backend: AgentBackend; runtimeContext: RuntimeEvent[]; runtimeContextRunHeaders: AgentRunHeader[]; startedAt: number; } export interface AgentRunContinuationBeginResult { backend: AgentBackend; startedAt: number; continuationStartAdmission: RuntimeContinuationStartAdmissionProof; } const RUNTIME_PARTIAL_FLUSH_INTERVAL_MS = 80; const RUNTIME_PARTIAL_BATCH_MAX_BYTES = 8 * 1024; export class AgentRun { readonly runId: string; readonly invocationId: string; readonly sessionId: string; readonly turnId: string; readonly toolBoundaryProtocol: ToolBoundaryProtocol | undefined; readonly lineage: AgentRunLineage; readonly effectiveOrchestration: EffectiveOrchestration; readonly toolMode: ToolMode; private readonly input: AgentRunInput; private readonly header: SessionHeader; private active: AgentRunActiveSession | undefined; private stopped = false; private abortSource: string | undefined; private traceQueue: Promise = Promise.resolve(); private runtimeEventQueue: Promise = Promise.resolve(); private runStoreAvailable = true; private runtimeEventStoreAvailable = true; private runtimeEventStoreFailure: unknown; private runtimePartialStreamKey: string | undefined; private runtimePartialBuffer: RuntimeEvent[] = []; private runtimePartialBufferBytes = 0; private runtimePartialFlushTimer: ReturnType | undefined; private traceWriteError: string | undefined; private runComposition: RunCompositionSnapshot | undefined; private runCompositionWrite: Promise | undefined; private failureClass: string | undefined; private failureMessage: string | undefined; private lastTs = 0; private sawCompletion = false; private finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined; private turnFailed = false; private finalized = false; private terminalRunHeaderCommitted = false; private continuationActive = false; private providerStateIdentity: `sha256:${string}` | undefined; private terminalClaim: | { owner: 'event' | 'stop'; event?: RuntimeEvent; write?: Promise; stopCompleted?: boolean; } | undefined; constructor(input: AgentRunInput) { const acceptedInput: AgentRunInput = { ...input, userInput: cloneAndFreezeRuntimeSnapshot(input.userInput), ...(input.effectiveOrchestration ? { effectiveOrchestration: cloneAndFreezeRuntimeSnapshot(input.effectiveOrchestration) } : {}), }; this.input = acceptedInput; if (acceptedInput.runStore && !acceptedInput.runtimeEventStore) { throw new Error('RuntimeEventStore is required when AgentRunStore is configured'); } if ( acceptedInput.durability === 'required' && (!acceptedInput.runStore || !acceptedInput.runtimeEventStore) ) { throw new Error('Required AgentRun durability needs AgentRunStore and RuntimeEventStore'); } this.runId = acceptedInput.runId ?? acceptedInput.newId(); this.invocationId = acceptedInput.invocationId ?? this.runId; this.sessionId = acceptedInput.sessionId; this.turnId = acceptedInput.userInput.turnId; this.toolBoundaryProtocol = acceptedInput.toolBoundaryProtocol; this.header = acceptedInput.header; this.effectiveOrchestration = acceptedInput.effectiveOrchestration ?? resolveEffectiveOrchestration( acceptedInput.header.orchestrationMode, acceptedInput.userInput.turnOrchestration, ); const requestedToolMode = acceptedInput.effectiveToolMode ?? acceptedInput.userInput.toolMode ?? DEFAULT_TOOL_MODE; if (!isToolMode(requestedToolMode)) { throw new Error(`Invalid tool mode: ${String(requestedToolMode)}`); } this.toolMode = requestedToolMode; this.lineage = { ...acceptedInput.runLineage, ...(acceptedInput.userInput.parentTurnId ? { parentTurnId: acceptedInput.userInput.parentTurnId } : {}), ...(acceptedInput.userInput.retriedFromTurnId ? { retriedFromTurnId: acceptedInput.userInput.retriedFromTurnId } : {}), ...(acceptedInput.userInput.regeneratedFromTurnId ? { regeneratedFromTurnId: acceptedInput.userInput.regeneratedFromTurnId } : {}), ...(acceptedInput.userInput.branchOfTurnId ? { branchOfTurnId: acceptedInput.userInput.branchOfTurnId } : {}), ...(acceptedInput.userInput.parentSessionId ? { parentSessionId: acceptedInput.userInput.parentSessionId } : {}), }; } stop(source: StopSessionInput['source'] | undefined): boolean { if (this.terminalClaim) return false; this.terminalClaim = { owner: 'stop' }; this.stopped = true; this.abortSource = normalizeStopSessionSource(source); return true; } isStopped(): boolean { return this.stopped; } headerSnapshot(): SessionHeader { return this.header; } bindProviderStateIdentity(identity: `sha256:${string}` | undefined): void { const claimed = this.input.claimedRunHeader?.providerStateIdentity; const expected = claimed ?? this.providerStateIdentity; if (expected !== undefined && expected !== identity) { throw new Error('Prepared backend provider state does not match the AgentRun admission'); } this.providerStateIdentity = identity; } isSessionInline(): boolean { return isSessionInlineRun({ ...(this.lineage.parentRunId ? { parentRunId: this.lineage.parentRunId } : {}), ...(this.continuationActive ? { continuationSource: true } : {}), }); } hasPendingStop(): boolean { return this.terminalClaim?.owner === 'stop' && this.terminalClaim.stopCompleted !== true; } completeStop(): void { if (this.terminalClaim?.owner === 'stop') this.terminalClaim.stopCompleted = true; } /** * Cash the terminal claim a stop already took. * * `stop()` claims the terminal outcome, but only `finalize()` — reached when * the backend's event stream ends — has ever cashed it. A turn parked on an * unanswered interaction never ends that stream, so the Session projection * read as aborted while the run stayed non-terminal in the ledger forever, * and every later turn dropped it from model context. Cash the claim at the * stop instead. The claim keeps this idempotent: a stream that later * produces its own terminal event finds the claim taken and writes nothing. */ async settleStopTerminal(): Promise { if (this.terminalClaim?.owner !== 'stop' || this.terminalRunHeaderCommitted) return; // Nothing durable is configured, so there is no fact to land. Every other // failure below is real and must reach the stop's caller: a stop that // reports success while the run stays non-terminal is the silent loss this // method exists to prevent. const runStore = this.input.runStore; if (!this.input.runtimeEventStore || !runStore) return; await this.flushRuntimePartialBuffer(true); // The claim only fences writers inside this Run. Another owner — a Host // recovery, a resumed continuation — may have sealed the ledger already, // and a sealed run rejects further appends. Nothing to land in that case: // the fact this method exists to guarantee is already there. This claim's // own reserved event is not foreign — that is a settlement being retried. const claimedEventId = this.terminalClaim.event?.id; const events = await this.loadTurnRuntimeEvents(); if (events.some((event) => isTerminalRuntimeEvent(event) && event.id !== claimedEventId)) return; if (!this.runStoreAvailable) { // The Run-store latch is best-effort history (one busy trace append // sets it) and commitTerminalRun silently skips under it, which here // would turn the stop into a reported success with no terminal fact: // the silent variant of the loss this method exists to prevent. // Probe like the RuntimeEvent read above; a store that answers lifts // the latch, one that cannot fails the settlement loudly so the stop // stays retryable. try { await runStore.readRun(this.sessionId, this.runId); this.runStoreAvailable = true; } catch (error) { throw new Error('AgentRun store is unavailable for stop settlement', { cause: error }); } } const ts = this.lastTs || this.input.now(); const finalStatus = { status: 'aborted' as const }; this.finalStatus ??= finalStatus; this.reserveFinalizationTerminal(finalStatus, ts); const runStoreAvailable = this.runStoreAvailable; try { await this.commitTerminalRun(finalStatus, ts); } catch (error) { // This commit runs ahead of the stream's own finalize, so one failure is // not evidence the store is gone for the rest of the run. Undo the marks // that would turn a single failed attempt into a permanently unwritable // run, and let the caller decide whether to retry. The reserved event // stays, so a retry lands the same terminal fact rather than a new one. this.runStoreAvailable = runStoreAvailable; if (this.terminalClaim) this.terminalClaim.write = undefined; throw error; } } recordRunTrace(event: RunTraceEvent): void { if (!this.input.runStore || !this.runStoreAvailable) return; this.enqueueRunStore('append trace event', async () => { await this.input.runStore?.appendEvent( this.sessionId, this.runId, traceToRunEvent(event, this.runId), ); }); } recordRunComposition(snapshot: RunCompositionSnapshot): Promise { if (!this.input.runStore) { return Promise.reject(new Error('AgentRun store is not configured')); } const normalized = decodeRunCompositionSnapshot(snapshot); if (this.runComposition && !isDeepStrictEqual(this.runComposition, normalized)) { return Promise.reject(new Error('AgentRun Run Composition changed after resolution')); } this.runComposition ??= normalized; if (this.runCompositionWrite) return this.runCompositionWrite; const write = this.enqueueRequiredRunStoreWrite('commit Run Composition', async () => { await this.input.runStore?.updateRun( this.sessionId, this.runId, { runComposition: normalized }, { durable: this.requiresDurablePersistence() }, ); }); this.runCompositionWrite = write; return write.catch((error: unknown) => { if (this.runCompositionWrite === write) this.runCompositionWrite = undefined; throw error; }); } /** * Canonical accounting record for one physical provider request (#1679). * * Durable, unlike the diagnostic attempt append above: this is the metering * source of truth, and a record lost to a crashed flush is spend nothing else * can reconstruct. * * It reports the failure as `trace_write_failed` and then rejects, so the * caller can tell whether the authority actually holds the record — the Usage * read model must not be written for a call the authority never committed. * Rejecting here is safe: settlement runs inside the model stream's `pull` * handler, and the seam swallows this so a billed, completed response is * never failed by its own bookkeeping. */ recordModelCallAttempt(commit: ModelCallCommit): Promise { const { attempt, latestContext } = commit; if (!this.input.runStore) return Promise.resolve(); return this.enqueueRequiredRunStoreWrite('append model call attempt', async () => { await this.input.runStore?.appendEvent( this.sessionId, this.runId, { type: MODEL_CALL_ATTEMPT_EVENT_TYPE, id: attempt.attemptId, runId: this.runId, sessionId: this.sessionId, turnId: attempt.turnId, ts: attempt.completedAt, data: { ...attempt }, }, // The latest-context projection rides this durable append rather than // racing it: one commit for the request, and derived state that cannot // survive a metering write that failed (#2323). { durable: true, ...(latestContext ? { latestContext } : {}) }, ); }); } /** * Durable append for one model-projection transition (#4283). * * Rethrows like the checkpoint recorder above: the caller may only show the * replacement once the ledger holds the record, so a failed append must be a * failed prune, not a silent one. */ recordModelProjectionTransition(transition: ModelProjectionTransition): Promise { if (!this.input.runStore) return Promise.reject(new Error('AgentRun store is not configured')); if (!this.runStoreAvailable) return Promise.reject(new Error('AgentRun store is unavailable')); return this.enqueueRunStore( 'append model projection transition', async () => { await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: MODEL_PROJECTION_TRANSITION_EVENT_TYPE, id: transition.transitionId, runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts: transition.createdAt, data: { runtimeEventId: transition.target.runtimeEventId, part: transition.target.part, transition, }, }); }, { rethrow: true }, ); } recordHistoryCompactCheckpoint(checkpoint: HistoryCompactCheckpoint): Promise { if (!this.input.runStore) return Promise.reject(new Error('AgentRun store is not configured')); if (!this.runStoreAvailable) return Promise.reject(new Error('AgentRun store is unavailable')); return this.enqueueRunStore( 'append history compact checkpoint', async () => { await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: 'history_compact_checkpoint_recorded', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts: this.input.now(), data: { checkpointId: checkpoint.checkpointId, highWaterName: checkpoint.highWaterName, highWaterSeq: checkpoint.highWaterSeq, boundaryKind: 'historyCompact', checkpoint, }, }); }, { rethrow: true }, ); } /** * Durable read of this run's RuntimeEvent ledger for the mid-turn capacity * invariant: waits for every write enqueued so far, then reads the store, so * a caller-derived coverage prefix can only ever span events that are * already persisted. Rejects when the store is unavailable — coverage must * never be computed over a projection the ledger cannot replay. */ async loadTurnRuntimeEvents(): Promise { const store = this.input.runtimeEventStore; if (!store) { throw new Error('RuntimeEvent store is unavailable for turn runtime events'); } await this.flushRuntimePartialBuffer(false); await this.runtimeEventQueue.catch(() => {}); if (this.runtimeEventStoreAvailable) { return await store.readRuntimeEvents(this.sessionId, this.runId); } // The unavailability latch records that a past write failed, not that // the store cannot answer now. This read is the probe that // disambiguates, the same way recordRuntimeEvents reads the ledger back // after an ambiguous append: a store that answers is available again // and the latch lifts, so a stop retried after one rejected write can // still settle its terminal fact (#2253) instead of failing on stale // history forever. A store that cannot answer keeps rejecting, and // coverage is never computed over a projection the ledger cannot // replay. try { const events = await store.readRuntimeEvents(this.sessionId, this.runId); this.runtimeEventStoreAvailable = true; return events; } catch (error) { throw new Error('RuntimeEvent store is unavailable for turn runtime events', { cause: error, }); } } async acceptMappedEvent( sessionEvent: SessionEvent, runtimeEvent: RuntimeEvent, options: { requireTerminalWrite?: boolean; allowInteractionResume?: boolean } = {}, ): Promise { const partialStreamKey = runtimePartialCoalescingKey(runtimeEvent); if (!partialStreamKey) await this.flushRuntimePartialBuffer(true); if (isTerminalRuntimeEvent(runtimeEvent)) { await this.recordRuntimeEvents([runtimeEvent], { requireTerminalWrite: options.requireTerminalWrite ?? Boolean(this.input.runtimeEventStore), }); await this.recordSessionEvent(sessionEvent, options); return; } if (this.requiresDurablePersistence() && isInteractionResumeAck(sessionEvent)) { // A hosted continuation may resume execution only after its identity-only // settlement fact is durable. Run status advances next, then Session // status; the queue consumer acknowledges the event only after all three. await this.recordRuntimeEvents([runtimeEvent], { requireDurableWrite: true }); await this.recordSessionEvent(sessionEvent, options); return; } await this.recordSessionEvent(sessionEvent, options); if (sessionEvent.type === 'provider_retry') return; if (partialStreamKey) { await this.recordRuntimePartial(runtimeEvent, partialStreamKey); return; } // ToolRuntime already persisted protocol-tagged tool calls/results through // the atomic RuntimeCommitSink. Re-appending the mapped UI event through // the generic lane would duplicate the fact and violate that boundary. if (isAtomicToolBoundaryProjection(runtimeEvent, this.toolBoundaryProtocol)) return; if (!isNonTerminalErrorRuntimeEvent(runtimeEvent)) { // A steered user message is fail-CLOSED: the backend's delivery ack // waits on this consume, and the provider must never execute a // directive the ledger does not carry. Every other non-terminal event // stays fail-open (a trace gap, not a correctness gap). const steering = runtimeEvent.content?.kind === 'text' && runtimeEvent.content.steering === true; await this.recordRuntimeEvents([runtimeEvent], steering ? { requireDurableWrite: true } : {}); await materializeRuntimeEventTranscriptProjection( this.input.store, this.sessionId, runtimeEvent, ); } } async begin(): Promise { await this.createRunRecord(); let initialRuntimeEventId: string; const userMessageTs = this.input.now(); if (this.input.userMessageId === null) { initialRuntimeEventId = this.input.newId(); } else { const userMessageId = this.input.userMessageId ?? this.input.newId(); initialRuntimeEventId = userMessageId; const userMsg = cloneAndFreezeRuntimeSnapshot({ type: 'user', id: userMessageId, turnId: this.turnId, ts: userMessageTs, text: this.input.userInput.text, ...(this.input.userInput.displayText !== undefined ? { displayText: this.input.userInput.displayText } : {}), ...(this.input.userInput.attachments ? { attachments: this.input.userInput.attachments } : {}), ...(this.input.userInput.directoryReferences ? { directoryReferences: this.input.userInput.directoryReferences } : {}), ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), ...(this.input.userInput.inlineReferences ? { inlineReferences: this.input.userInput.inlineReferences } : {}), ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), }); await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); } await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); this.lastTs = userMessageTs; const initialRuntimeEvent = cloneAndFreezeRuntimeSnapshot( this.buildInitialRuntimeEvent(initialRuntimeEventId, this.lastTs), ); await this.recordRuntimeEvents([initialRuntimeEvent], { requireDurableWrite: this.requiresDurablePersistence(), }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); await this.markRunStarted(this.lastTs); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, this.lastTs); const priorRuntimeContext = await this.buildPriorRuntimeContext(); const projectionContext = priorRuntimeContext ? projectRuntimeEventsToStoredMessages(priorRuntimeContext.events, { runHeaders: priorRuntimeContext.runs, }).messages : []; return { backend: this.active.backend, backendInput: cloneAndFreezeRuntimeSnapshot({ turnId: this.turnId, orchestration: this.effectiveOrchestration, toolMode: this.toolMode, ...(this.input.userInput.maxSteps !== undefined ? { maxSteps: this.input.userInput.maxSteps } : {}), text: this.input.userInput.text, ...(this.input.userInput.attachments ? { attachments: this.input.userInput.attachments } : {}), ...(this.input.userInput.directoryReferences ? { directoryReferences: this.input.userInput.directoryReferences } : {}), ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), context: projectionContext, ...(priorRuntimeContext ? { runtimeContext: priorRuntimeContext.events, runtimeContextRunHeaders: priorRuntimeContext.runs, } : {}), }), initialRuntimeEvent, }; } async beginOperation(): Promise { await this.createRunRecord(); const startedAt = this.input.now(); this.lastTs = startedAt; await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { ts: startedAt, }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); await this.markRunStarted(startedAt); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); const priorRuntimeContext = await this.buildPriorRuntimeContext(); return { backend: this.active.backend, runtimeContext: priorRuntimeContext?.events ?? [], runtimeContextRunHeaders: priorRuntimeContext?.runs ?? [], startedAt, }; } async beginContinuation( continuation: RuntimeContinuation, ): Promise { if ( continuation.sessionId !== this.sessionId || continuation.runId !== this.runId || continuation.turnId !== this.turnId ) { throw new Error('Runtime continuation identity does not match the target AgentRun'); } this.continuationActive = true; await this.createRunRecord(continuation); await this.input.continuationFailpoint?.('after_run_created'); const startedAt = this.input.now(); this.lastTs = startedAt; if (!this.input.commitContinuationStart) { throw new Error('Runtime continuation requires a durable continuation-start authority'); } let committedStart: { startEventId: string; created: true }; try { committedStart = await this.input.commitContinuationStart(startedAt); } catch (error) { throw new ContinuationStartCommitError(error); } await this.input.continuationFailpoint?.('after_continuation_start_committed'); await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { ts: startedAt, }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); await this.markRunStarted(startedAt); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); return { backend: this.active.backend, startedAt, continuationStartAdmission: createRuntimeContinuationStartAdmissionProof({ startEventId: committedStart.startEventId, claimId: continuation.claimId ?? '', boundaryDigest: continuation.boundary?.manifestDigest ?? 'sha256:', providerProjectionVersion: continuation.providerProjectionVersion ?? 1, providerReplayDigest: continuation.providerReplayDigest ?? 'sha256:', ...(this.toolBoundaryProtocol ? { toolBoundaryProtocol: this.toolBoundaryProtocol } : {}), target: { sessionId: continuation.sessionId, invocationId: continuation.invocationId, runId: continuation.runId, turnId: continuation.turnId, }, }), }; } private buildInitialRuntimeEvent(id: string, ts: number): RuntimeEvent { const input = this.input.userInput; return { id, invocationId: this.invocationId, runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, partial: false, role: 'user', author: input.origin ? 'host' : 'user', content: { kind: 'text', text: input.text, ...(input.displayText !== undefined ? { displayText: input.displayText } : {}), ...(input.origin !== undefined ? { origin: input.origin } : {}), ...(input.attachments !== undefined && input.attachments.length > 0 ? { attachments: input.attachments } : {}), ...(input.directoryReferences ? { directoryReferences: input.directoryReferences } : {}), ...(input.quotes !== undefined && input.quotes.length > 0 ? { quotes: input.quotes } : {}), ...(input.inlineReferences !== undefined ? { inlineReferences: input.inlineReferences } : {}), }, ...(this.toolBoundaryProtocol ? { actions: { runtimeProtocol: { toolBoundary: this.toolBoundaryProtocol } } } : {}), }; } async recordStoredSessionEvent(ev: SessionEvent): Promise { if (ev.type === 'token_usage') { await this.input.store.appendMessage(this.sessionId, { ...ev } satisfies StoredMessage); } } async recordSessionEvent( ev: SessionEvent, options: { allowInteractionResume?: boolean } = {}, ): Promise { this.lastTs = ev.ts; const transition = statusFromEvent(ev, options); const terminalSessionEvent = (ev.type === 'complete' || ev.type === 'abort') && !this.turnFailed; const turnStatus = terminalSessionEvent ? turnStatusFromEvent(ev) : undefined; if (terminalSessionEvent) { this.sawCompletion = true; if (ev.type === 'abort' && !this.abortSource) this.abortSource = ev.reason; if (ev.type === 'complete' && ev.stopReason === 'user_stop' && !this.abortSource) this.abortSource = 'user_stop'; this.finalStatus = this.stopped ? { status: 'aborted' } : (transition ?? { status: 'active' }); // A terminal complete event can carry a failure without a preceding // error event. Record it now so finalize preserves the precise class. if ( turnStatus?.status === 'failed' && turnStatus.errorClass && !this.failureClass && !this.stopped ) { this.markRunFailed( turnStatus.errorClass, `turn ended with stopReason=${ev.type === 'complete' ? ev.stopReason : 'unknown'}`, ev.ts, ); } } if (transition && !this.stopped) { const updateSessionStatus = async (): Promise => { if (terminalSessionEvent || ev.type === 'error') { await this.input.hooks .updateStatus(this.sessionId, transition.status, transition.blockedReason, ev.ts) .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); return; } await this.input.hooks.updateStatus( this.sessionId, transition.status, transition.blockedReason, ev.ts, ); }; // On resume, advance the Run before the Session so an interrupted pair // remains conservatively waiting rather than advertising false readiness. if (this.requiresDurablePersistence() && isInteractionResumeAck(ev)) { await this.recordStatusFromTransition(ev, transition, ev.ts); await updateSessionStatus(); } else { await updateSessionStatus(); await this.recordStatusFromTransition(ev, transition, ev.ts); } } if (turnStatus && !this.stopped) { const appendTurnState = this.input.hooks.appendTurnState( this.sessionId, this.turnId, turnStatus.status, this.lineage, { ts: ev.ts, errorClass: turnStatus.errorClass, ...(turnStatus.status === 'aborted' && this.abortSource ? { abortSource: this.abortSource } : {}), }, ); if (terminalSessionEvent || ev.type === 'error') { await appendTurnState.catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection'), ); } else { await appendTurnState; } } if (ev.type === 'error') { if (this.stopped) { this.finalStatus = { status: 'aborted' }; } else { this.turnFailed = true; this.finalStatus = transition ?? { status: 'blocked', blockedReason: 'unknown' }; await this.input.hooks .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { ts: ev.ts, errorClass: ev.reason ?? ev.code ?? 'unknown', }) .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message, ev.ts); } } } async recordRuntimeEvents( events: readonly RuntimeEvent[], options: { requireTerminalWrite?: boolean; requireDurableWrite?: boolean } = {}, ): Promise { if (events.length === 0) return; for (const event of events) { const terminal = isTerminalRuntimeEvent(event); const eventForStore = terminal ? this.reserveTerminalEvent(event) : event; if (!eventForStore) continue; if (!this.input.runtimeEventStore || !this.runtimeEventStoreAvailable) { if (this.input.runtimeEventStore?.durability === 'canonical') { throw ( this.runtimeEventStoreFailure ?? new Error('canonical RuntimeEvent store is unavailable') ); } if (terminal && options.requireTerminalWrite) { throw new Error('terminal RuntimeEvent store is unavailable'); } if (options.requireDurableWrite && this.input.runtimeEventStore) { // The store exists but earlier writes failed: a durability-required // event (steering) must not silently skip the ledger. throw new Error('RuntimeEvent store is unavailable for a durability-required event'); } continue; } const write = this.enqueueRuntimeEventStore( 'append runtime event', async () => { await this.input.runtimeEventStore?.appendRuntimeEvent( this.sessionId, this.runId, eventForStore, { durable: terminal || options.requireDurableWrite === true }, ); }, { rethrow: terminal || options.requireTerminalWrite || options.requireDurableWrite || this.input.runtimeEventStore.durability === 'canonical', }, ); if (terminal && this.terminalClaim) this.terminalClaim.write = write; if (options.requireDurableWrite && !terminal) { // An append error is AMBIGUOUS: the bytes may have landed before the // failure (e.g. a close error after the write). For a // durability-required event the caller settles a delivery lease on // this outcome, so a false "not durable" would redeliver a message // the ledger already owns. Read the ledger back to disambiguate: // present ⇒ durable (continue on the ack path); absent or read-back // also failing ⇒ fail closed (rethrow ⇒ nack). try { await write; } catch (error) { if (error instanceof DurableStoreWriteError) throw error; if (!(await this.eventLandedInLedger(eventForStore.id))) throw error; // The write landed and the ledger answered a fresh read — the // failure was in the reporting, not the store. Lift the // unavailability latch so the rest of the turn (including its // required terminal write) keeps persisting; a genuinely broken // store re-latches on its next write. this.runtimeEventStoreAvailable = true; } continue; } await write; } } private reserveTerminalEvent(event: RuntimeEvent): RuntimeEvent | undefined { if (this.terminalClaim?.event) return undefined; this.terminalClaim ??= { owner: 'event' }; const eventForStore = this.terminalClaim.owner === 'stop' ? this.abortedRuntimeEvent(event) : event; this.terminalClaim.event = eventForStore; return eventForStore; } private abortedRuntimeEvent(event: RuntimeEvent): RuntimeEvent { const { content: _content, ...rest } = event; void _content; return { ...rest, status: 'aborted', actions: { ...event.actions, endInvocation: true, stateDelta: { ...event.actions?.stateDelta, abortSource: this.abortSource ?? 'user_stop', }, }, }; } async recordFailure(error: unknown): Promise { if (this.stopped) { this.finalStatus = { status: 'aborted' }; return; } this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; await this.input.hooks .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { errorClass: error instanceof Error ? error.name : 'unknown', }) .catch(() => {}); this.markRunFailed( error instanceof Error ? error.name : 'unknown', errorMessage(error), this.input.now(), ); } async finalize(): Promise { if (this.finalized) return; this.finalized = true; await this.flushRuntimePartialBuffer(true); const lastTs = this.lastTs || this.input.now(); if (this.stopped) this.finalStatus = { status: 'aborted' }; if (!this.finalStatus) { this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; this.markRunFailed( 'missing_terminal_event', 'run finalized without a terminal SessionEvent', lastTs, ); } this.reserveFinalizationTerminal(this.finalStatus, lastTs); if (this.active) { await this.input.hooks.unregisterRun(this.active, this); } const nextStatus = this.active && this.active.activeRuns.size > 0 ? { status: 'running' as const } : (this.finalStatus ?? { status: 'active' as const }); try { await this.input.hooks.updateHeader(this.sessionId, { lastMessageAt: lastTs, hasUnread: true, ...buildStatusPatch(nextStatus.status, lastTs, nextStatus.blockedReason), }); } catch { // The user-visible turn already completed; preserve existing behavior. } if (this.sawCompletion) { await this.input.store .appendMessage(this.sessionId, { type: 'system_note', id: this.input.newId(), turnId: this.turnId, ts: lastTs, kind: 'session_resume', } satisfies SystemNoteMessage) .catch(() => {}); } await this.finishRun(this.finalStatus, lastTs); } private async createRunRecord(continuation?: RuntimeContinuation): Promise { if (!this.input.runStore) { if (continuation) throw new Error('Runtime continuation requires a durable run store'); return; } const createdAt = continuation && this.input.claimedRunHeader ? this.input.claimedRunHeader.createdAt : this.input.now(); const providerStateIdentity = this.input.claimedRunHeader?.providerStateIdentity ?? this.providerStateIdentity; this.providerStateIdentity = providerStateIdentity; const computedHeader: AgentRunHeader = { runId: this.runId, invocationId: this.invocationId, sessionId: this.sessionId, turnId: this.turnId, status: 'created', backendKind: this.header.backend, ...(this.header.llmConnectionId === undefined ? {} : { llmConnectionId: this.header.llmConnectionId }), ...(providerStateIdentity ? { providerStateIdentity } : {}), llmConnectionSlug: this.header.llmConnectionSlug, modelId: this.header.model, cwd: this.header.cwd, ...(this.input.workspaceIdentity ? { workspaceIdentity: this.input.workspaceIdentity } : {}), permissionMode: this.header.permissionMode, collaborationMode: this.header.collaborationMode ?? 'agent', orchestrationMode: this.effectiveOrchestration.mode, orchestrationSource: this.effectiveOrchestration.source, agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, toolMode: this.toolMode, createdAt, updatedAt: createdAt, ...this.lineage, ...(continuation ? { continuationSource: continuation.claimId && continuation.boundary ? { protocol: 'continuation_source_v2' as const, claimId: continuation.claimId, boundaryDigest: continuation.boundary.manifestDigest, sourceInvocationId: continuation.sourceInvocationId, sourceRunId: continuation.sourceRunId, sourceTurnId: continuation.sourceTurnId, sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, sourcePrefixDigest: continuation.boundary.segments.at(-1)!.prefixDigest, replayManifestDigest: continuation.boundary.manifestDigest, } : { sourceInvocationId: continuation.sourceInvocationId, sourceRunId: continuation.sourceRunId, sourceTurnId: continuation.sourceTurnId, sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, }, } : {}), ...(this.input.userInput.agentId ? { agentId: this.input.userInput.agentId } : {}), ...(this.input.userInput.agentName ? { agentName: this.input.userInput.agentName } : {}), ...(this.input.userInput.origin?.kind === 'scheduled_task' ? { scheduledTaskId: this.input.userInput.origin.scheduledTaskId } : {}), ...(this.input.userInput.origin?.kind === 'goal' ? { goalId: this.input.userInput.origin.goalId } : {}), ...(this.input.userInput.origin?.kind === 'agent_graph' ? { agentGraphWakeId: this.input.userInput.origin.wakeId, agentGraphWakeAttemptId: this.input.userInput.origin.attemptId, } : {}), ...(this.input.rootExecutionKind ? { rootExecutionKind: this.input.rootExecutionKind } : {}), }; const header = continuation && this.input.claimedRunHeader ? this.input.claimedRunHeader : computedHeader; if ( continuation && this.input.claimedRunHeader && !isDeepStrictEqual(this.input.claimedRunHeader, computedHeader) ) { throw new Error('Claimed continuation target Run header no longer matches execution'); } try { const durable = this.requiresDurablePersistence(); await this.input.runStore.createRun(header, { durable }); await this.input.runStore.appendEvent( this.sessionId, this.runId, { type: 'run_created', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts: createdAt, data: { textLength: this.input.userInput.text.length, attachmentCount: this.input.userInput.attachments?.length ?? 0, orchestrationMode: this.effectiveOrchestration.mode, orchestrationSource: this.effectiveOrchestration.source, agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, toolMode: this.toolMode, }, }, { durable }, ); } catch (error) { this.runStoreAvailable = false; if (this.requiresDurablePersistence()) throw error; this.enqueueTraceWriteFailure(error); if (continuation) throw error; } } private requiresDurablePersistence(): boolean { return this.input.durability === 'required'; } private async buildPriorRuntimeContext(): Promise { return await buildPriorRuntimeContextProjection({ sessionId: this.sessionId, currentRunId: this.runId, currentTurnId: this.turnId, runStore: this.input.runStore, runtimeEventStore: this.input.runtimeEventStore, runStoreAvailable: this.runStoreAvailable, runtimeEventStoreAvailable: this.runtimeEventStoreAvailable, repairRunRuntimeLedger: this.input.repairRunRuntimeLedger, readMessages: () => this.input.store.readMessages(this.sessionId), }); } private async markRunStarted(ts: number): Promise { if (!this.input.runStore || !this.runStoreAvailable) return; const durable = this.requiresDurablePersistence(); const write = this.enqueueRunStore( 'mark run started', async () => { await this.input.runStore?.appendEvent( this.sessionId, this.runId, { type: 'run_started', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, }, { durable }, ); await this.input.runStore?.updateRun( this.sessionId, this.runId, { status: 'running', updatedAt: ts }, { durable }, ); }, { rethrow: durable }, ); if (durable) await write; } private async recordStatusFromTransition( ev: SessionEvent, transition: { status: SessionStatus; blockedReason?: SessionBlockedReason }, ts: number, ): Promise { const durable = this.requiresDurablePersistence(); const runStore = this.input.runStore; if (!runStore) { if (durable) { throw new Error('AgentRun store is unavailable for a required status transition'); } return; } const status = transition.status === 'waiting_for_user' ? 'waiting_for_user' : transition.status === 'aborted' ? 'cancelled' : transition.status === 'blocked' ? 'failed' : transition.status === 'active' ? 'completed' : 'running'; if (isTerminalRunStatus(status)) return; const appendAudit = async (): Promise => { await runStore.appendEvent( this.sessionId, this.runId, { type: 'run_status_changed', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, data: { sessionStatus: transition.status, ...(transition.blockedReason ? { blockedReason: transition.blockedReason } : {}), }, }, { durable }, ); }; if (durable) { await this.enqueueRequiredRunStoreWrite('record required run status', async () => { await runStore.updateRun( this.sessionId, this.runId, { status, updatedAt: ts }, { durable: true }, ); }); // The audit remains best-effort, but its physical write belongs to this // required transition and must settle before the resume acknowledgement. await this.enqueueRunStore('append run status audit', appendAudit); } else { this.enqueueRunStore('record run status', async () => { await runStore.updateRun(this.sessionId, this.runId, { status, updatedAt: ts }); await appendAudit(); }); } if (ev.type === 'abort') { this.markRunCancelled(ev.reason, ts); } } private markRunFailed(failureClass: string, message: string, ts: number): void { if (!this.input.runStore || !this.runStoreAvailable) return; this.failureClass = failureClass; this.failureMessage = redactTraceString(message); if (this.input.runtimeEventStore) return; this.enqueueRunStore('mark run failed', async () => { await this.input.runStore?.updateRun(this.sessionId, this.runId, { status: 'failed', updatedAt: ts, completedAt: ts, failureClass, failureMessage: this.failureMessage, }); await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: 'run_failed', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, message: redactTraceString(message), data: { failureClass }, }); }); } private markRunCancelled(reason: string | undefined, ts: number): void { if (!this.input.runStore || !this.runStoreAvailable) return; if (this.input.runtimeEventStore) return; this.enqueueRunStore('mark run cancelled', async () => { await this.input.runStore?.updateRun(this.sessionId, this.runId, { status: 'cancelled', updatedAt: ts, completedAt: ts, }); await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: 'run_cancelled', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, ...(reason ? { message: redactTraceString(reason) } : {}), }); }); } private async finishRun( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ts: number, ): Promise { await this.traceQueue.catch(() => {}); if (!this.input.runStore || !this.runStoreAvailable) return; const status = this.runStatusForFinalStatus(finalStatus); const isTerminal = status === 'completed' || status === 'failed' || status === 'cancelled'; if (isTerminal && this.input.runtimeEventStore) { await this.commitTerminalRun(finalStatus, ts); return; } await this.enqueueRunStore('finish run', async () => { await this.input.runStore?.updateRun(this.sessionId, this.runId, { status, updatedAt: ts, ...(isTerminal ? { completedAt: ts } : {}), ...(status === 'failed' ? { failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown', ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), } : {}), }); await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: status === 'cancelled' ? 'run_cancelled' : status === 'failed' ? 'run_failed' : status === 'completed' ? 'run_completed' : 'run_status_changed', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts, ...(status === 'failed' ? { data: { failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown' } } : status === 'waiting_for_user' ? { data: { sessionStatus: 'waiting_for_user', blockedReason: finalStatus?.blockedReason ?? 'permission_required', }, } : {}), }); }); await this.traceQueue.catch(() => {}); } private runStatusForFinalStatus( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ): AgentRunHeader['status'] { if (this.stopped || finalStatus?.status === 'aborted') return 'cancelled'; if (this.failureClass || finalStatus?.status === 'blocked') return 'failed'; if (finalStatus?.status === 'waiting_for_user') return 'waiting_for_user'; return 'completed'; } private async commitTerminalRun( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ts: number, ): Promise { if (this.terminalRunHeaderCommitted) return; const runStore = this.input.runStore; const runtimeEventStore = this.input.runtimeEventStore; if (!runStore || !runtimeEventStore) return; // A latched RuntimeEvent store normally keeps the skip below: the latch // marks a write failure, and a transient one leaves the run non-terminal // on purpose so startup recovery repairs it with its own bookkeeping. // A corruption latch is the exception (#2313). The health scan refuses // tool-bearing appends only, so the terminal event is a write the // damaged ledger would have taken, and no recovery pass will ever be // safer than landing it now: the run must be able to say it ended. The // latch itself stays closed, nothing else may write; the terminal // durability barrier below doubles as the scoped probe, and if even // that write is refused the silent skip stands. let corruptionRecovery = false; if (!this.runtimeEventStoreAvailable) { if (!(this.runtimeEventStoreFailure instanceof ToolLedgerCorruptionError)) return; corruptionRecovery = true; } if (!this.runStoreAvailable) return; const fallbackStatus = this.stopped || finalStatus?.status === 'aborted' ? 'cancelled' : 'failed'; const fallbackFailureClass = 'missing_terminal_event'; const fallbackFailureMessage = this.failureMessage ?? 'run finalized without a terminal RuntimeEvent'; try { const terminalClaim = this.terminalClaim; const terminalEvent = terminalClaim?.event; if (!terminalEvent) throw new Error('terminal RuntimeEvent claim is missing'); try { await terminalClaim.write; } catch (error) { // Under a corruption latch the claimed event's write is the stale // latched failure, not the barrier's own verdict: clear it so // commitOrCreateTerminalRunFact lands the same claimed fact fresh // (#2313). On a store that was never latched the failure is live // and keeps propagating exactly as before. if (!corruptionRecovery) throw error; terminalClaim.write = undefined; } // Re-check after the await, not only at entry. Two callers — a stop // settling the claim and the stream's own finalize — can both pass the // entry guard and then queue behind the same write. The claim slot // dedupes the RuntimeEvent, but the run-store projection would append a // second terminal AgentRunEvent for the one run. if (this.terminalRunHeaderCommitted) return; // On the recovery path the claimed event's write never committed, so // the boundary named after that commit must wait for the durability // barrier inside commitOrCreateTerminalRunFact; firing it here would // let a crash leave a durable continuation start without the terminal // fact it is contracted to follow. const deferContinuationBoundary = corruptionRecovery && !terminalClaim.write; if (this.continuationActive && !deferContinuationBoundary) { await this.input.continuationFailpoint?.('after_terminal_event_committed'); } const commit = commitOrCreateTerminalRunFact({ runStore, runtimeEventStore, ...(this.continuationActive && deferContinuationBoundary ? { afterTerminalDurable: async () => { await this.input.continuationFailpoint?.('after_terminal_event_committed'); }, } : {}), newId: this.input.newId, sessionId: this.sessionId, runId: this.runId, turnId: this.turnId, ts, terminalEvent, ...((this.failureClass ?? finalStatus?.blockedReason) ? { failureClass: this.failureClass ?? finalStatus?.blockedReason } : {}), ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), ...(this.traceWriteError ? { traceWriteError: this.traceWriteError } : {}), ...(this.abortSource || fallbackStatus === 'cancelled' ? { abortSource: this.abortSource ?? 'user_stop' } : {}), fallbackStatus, fallbackInvocationId: this.runId, ...(fallbackStatus === 'failed' ? { fallbackFailureClass, fallbackFailureMessage } : {}), allowHeaderCommitFailure: true, }); if (!terminalClaim.write) { terminalClaim.write = commit.then(() => undefined); void terminalClaim.write.catch(() => {}); } const result = await commit; this.terminalRunHeaderCommitted = result.headerCommitted; if (result.headerCommitted && this.continuationActive) { await this.input.continuationFailpoint?.('after_terminal_header_committed'); } if (result.headerCommitError !== undefined) { await this.enqueueTraceWriteFailure(result.headerCommitError, 'commit terminal run header'); } } catch (error) { if (corruptionRecovery) { // The scoped barrier lost its bet: the ledger refused even the // terminal fact. The latch never lifted, so there is nothing to // restore; record the failure and keep the finalize path's // historical silence for a store that stays broken. await this.enqueueTraceWriteFailure(error, 'commit terminal run header'); return; } this.runStoreAvailable = false; await this.enqueueTraceWriteFailure(error, 'commit terminal run header'); throw error; } await this.traceQueue.catch(() => {}); } private reserveFinalizationTerminal( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ts: number, ): void { if (this.terminalClaim?.event) return; const runStatus = this.runStatusForFinalStatus(finalStatus); if (runStatus !== 'completed' && runStatus !== 'failed' && runStatus !== 'cancelled') return; const status = this.terminalClaim?.owner === 'stop' || this.stopped || finalStatus?.status === 'aborted' ? 'cancelled' : 'failed'; const failureClass = 'missing_terminal_event'; const failureMessage = this.failureMessage ?? 'run finalized without a terminal RuntimeEvent'; if (status === 'failed') { this.failureClass = failureClass; this.failureMessage = failureMessage; } this.reserveTerminalEvent( buildSyntheticTerminalRuntimeEvent({ id: this.input.newId(), invocationId: this.invocationId, run: { sessionId: this.sessionId, runId: this.runId, turnId: this.turnId }, status, ts, ...(status === 'failed' ? { failureClass, message: failureMessage } : {}), ...(status === 'cancelled' ? { abortSource: this.abortSource ?? 'user_stop' } : {}), }), ); } private enqueueRunStore( label: string, operation: () => Promise, options: { rethrow?: boolean } = {}, ): Promise { if (!this.input.runStore || !this.runStoreAvailable) return Promise.resolve(); const next = this.traceQueue.then(operation, operation).catch(async (error) => { this.runStoreAvailable = false; await this.enqueueTraceWriteFailure(error, label); if (options.rethrow) throw error; }); this.traceQueue = next.catch(() => {}); return next; } /** * Serialize a required Run-store write without consulting the best-effort * latch. A successful required write proves the store is available again; * a failed operation rejects its caller without changing the general latch. */ private enqueueRequiredRunStoreWrite( label: string, operation: () => Promise, ): Promise { const probe = async (): Promise => { await operation(); this.runStoreAvailable = true; }; const next = this.traceQueue.then(probe, probe).catch(async (error) => { await this.enqueueTraceWriteFailure(error, label); throw error; }); this.traceQueue = next.catch(() => {}); return next; } /** * Read-back disambiguation for a failed durability-required append: true * only when the ledger demonstrably contains the event. Any doubt (no * read-back capability, read failure, event absent) reports false so the * caller stays fail-closed. */ private async eventLandedInLedger(eventId: string): Promise { const store = this.input.runtimeEventStore; if (!store?.readImmutableRuntimeEvents) return false; try { const events = await store.readImmutableRuntimeEvents(this.sessionId, this.runId); return events.some((event) => event.id === eventId); } catch { return false; } } private enqueueRuntimeEventStore( label: string, operation: () => Promise, options: { rethrow?: boolean } = {}, ): Promise { if (!this.input.runtimeEventStore || !this.runtimeEventStoreAvailable) return Promise.resolve(); const next = this.runtimeEventQueue.then(operation, operation).catch(async (error) => { // A rejection is the ledger refusing one malformed candidate, not the // store going away: it stays healthy and readable, so the latch would // only cost this run the writes it still owes — above all its own // terminal event, which `recordRuntimeEvents` refuses once the store // reads unavailable. That is how a single refused append left a run at // `running` with no terminal event and no visible failure (#2234). The // append still fails the caller (a producer bug must not pass quietly), // but the ledger stays open so the turn can end the way every other // failure ends. // // Only that one class is exempt. A store that went away keeps latching: // nothing this run emits next can land. // // `ToolLedgerCorruptionError` also keeps latching, and that is the // right economy for THIS path: stream writes stay fail-closed against // a damaged ledger. What the latch must not cost is the terminal fact // (#2313): the health scan gates only tool-bearing appends, so the // terminal event is a write the damaged ledger would have taken. // `commitTerminalRun` therefore carries the one exception: under a // corruption latch it still attempts the terminal durability barrier // (the barrier is its own scoped probe), so the run says it ended // while everything routed through here keeps failing closed. if (error instanceof RunSealedError) { // A refusal that is correct in itself (#2311): the run already owns // its terminal fact, and a straggler from the still-draining stream // is by definition not part of it. Neither the store nor this run's // durable history is at fault, so no latch and no trace-write // failure; a caller that asked for the rejection still receives it. if (options.rethrow) throw error; return; } if (!(error instanceof ToolLedgerRejectionError)) { this.runtimeEventStoreAvailable = false; this.runtimeEventStoreFailure = error; } await this.enqueueTraceWriteFailure(error, label); if (options.rethrow) throw error; }); this.runtimeEventQueue = next.catch(() => {}); return next; } private async recordRuntimePartial(event: RuntimeEvent, streamKey: string): Promise { const store = this.input.runtimeEventStore; if (!store?.appendRuntimePartialBatch) { await this.recordRuntimeEvents([event]); return; } if (!this.runtimeEventStoreAvailable) { await this.recordRuntimeEvents([event]); return; } if (this.runtimePartialStreamKey !== streamKey) { await this.flushRuntimePartialBuffer(true); // Persist the first chunk synchronously. Besides bounding crash loss, this // captures the immutable anchor before an upstream tool boundary can // commit while later chunks are waiting in the coalescer. await this.recordRuntimeEvents([event]); this.runtimePartialStreamKey = streamKey; return; } this.runtimePartialBuffer.push(event); this.runtimePartialBufferBytes += runtimePartialTextBytes(event); if (this.runtimePartialBufferBytes >= RUNTIME_PARTIAL_BATCH_MAX_BYTES) { await this.flushRuntimePartialBuffer(false); return; } this.scheduleRuntimePartialFlush(); } private scheduleRuntimePartialFlush(): void { if (this.runtimePartialFlushTimer) return; this.runtimePartialFlushTimer = setTimeout(() => { this.runtimePartialFlushTimer = undefined; void this.flushRuntimePartialBuffer(false).catch(() => { // enqueueRuntimeEventStore latches and reports the failure. The next // event or execution boundary observes that latch and fails closed. }); }, RUNTIME_PARTIAL_FLUSH_INTERVAL_MS); } private async flushRuntimePartialBuffer(closeStream: boolean): Promise { const ownedPartialWork = this.runtimePartialStreamKey !== undefined || this.runtimePartialBuffer.length > 0 || this.runtimePartialFlushTimer !== undefined; if (this.runtimePartialFlushTimer) { clearTimeout(this.runtimePartialFlushTimer); this.runtimePartialFlushTimer = undefined; } const events = this.runtimePartialBuffer; this.runtimePartialBuffer = []; this.runtimePartialBufferBytes = 0; if (closeStream) this.runtimePartialStreamKey = undefined; if (events.length === 0) { // A timer flush may already be queued. Waiting here preserves the rule // that an immutable boundary never overtakes prior presentation text. if (closeStream) { await this.runtimeEventQueue; if ( ownedPartialWork && !this.runtimeEventStoreAvailable && this.input.runtimeEventStore?.durability === 'canonical' ) { throw ( this.runtimeEventStoreFailure ?? new Error('canonical RuntimeEvent store is unavailable') ); } } return; } const store = this.input.runtimeEventStore; if (!store?.appendRuntimePartialBatch) { await this.recordRuntimeEvents(events); return; } await this.enqueueRuntimeEventStore( 'append runtime partial batch', async () => { await store.appendRuntimePartialBatch?.(this.sessionId, this.runId, events); }, { rethrow: store.durability === 'canonical' }, ); } private async enqueueTraceWriteFailure( error: unknown, label = 'agent run store write', ): Promise { const message = errorMessage(error); this.traceWriteError ??= `${label}: ${message}`; try { await this.input.runStore?.updateRun(this.sessionId, this.runId, { traceWriteError: this.traceWriteError, updatedAt: this.input.now(), }); } catch { // The terminal header commit retries the in-memory latch. } try { await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: 'trace_write_failed', id: this.input.newId(), runId: this.runId, sessionId: this.sessionId, turnId: this.turnId, ts: this.input.now(), message, }); } catch { // Diagnostic persistence is best effort; never perturb model/tool execution. } } } function runtimePartialCoalescingKey(event: RuntimeEvent): string | undefined { if (!event.partial || event.status !== undefined || event.actions) return undefined; const content = event.content; if (content?.kind !== 'text' && content?.kind !== 'thinking') return undefined; if (content.kind === 'text' && content.attachments !== undefined) return undefined; if (content.kind === 'thinking' && content.signature !== undefined) return undefined; const providerEventId = event.refs?.providerEventId; if (!providerEventId || Object.keys(event.refs ?? {}).some((key) => key !== 'providerEventId')) { return undefined; } return JSON.stringify([ content.kind, providerEventId, event.sessionId, event.invocationId, event.runId, event.turnId, event.branch ?? null, event.role, event.author, ]); } function runtimePartialTextBytes(event: RuntimeEvent): number { const content = event.content; return content?.kind === 'text' || content?.kind === 'thinking' ? Buffer.byteLength(content.text, 'utf8') : 0; } function traceToRunEvent(event: RunTraceEvent, runId: string): EmittedAgentRunEvent { return { type: event.type, id: event.id, runId, sessionId: event.sessionId, turnId: event.turnId, ts: event.ts, message: redactTraceString(event.message), data: sanitizeTraceData(event.data), }; } function sanitizeTraceData( data: Record | undefined, ): Record | undefined { if (!data) return undefined; return Object.fromEntries( Object.entries(data) .filter(([, value]) => value !== undefined) .map(([key, value]) => [key, sanitizeTraceValue(value)]), ); } function sanitizeTraceValue(value: unknown): unknown { if (typeof value === 'string') return redactTraceString(value); if (Array.isArray(value)) return value.slice(0, 50).map(sanitizeTraceValue); if (value && typeof value === 'object') { return Object.fromEntries( Object.entries(value as Record) .slice(0, 50) .map(([key, nested]) => [key, sanitizeTraceValue(nested)]), ); } return value; } function redactTraceString(value: string): string { const redacted = redactSecrets(value); return redacted.length > 2_000 ? `${redacted.slice(0, 2_000)}...[truncated]` : redacted; } function errorMessage(error: unknown): string { return redactTraceString(error instanceof Error ? error.message : String(error)); } async function appendUserMessageOnce( store: AgentRunSessionStore, sessionId: string, message: UserMessage, ): Promise { const existing = (await store.readMessages(sessionId)).find( (candidate) => candidate.id === message.id, ); if (!existing) { await store.appendMessage(sessionId, message); return; } if ( existing.type !== 'user' || existing.turnId !== message.turnId || !messageContentsEqual(existing, message) ) { throw new Error(`Durable UserMessage identity ${message.id} has conflicting content`); } } function isInteractionResumeAck(event: SessionEvent): boolean { return ( event.type === 'sandbox_boundary_decision_ack' || event.type === 'user_question_answer_ack' ); } /** * Non-terminal error content never reaches the ledger: the trailing terminal * event carries the failure. Exported so readers can reason about which mapped * RuntimeEvents a projection will ever be asked to read. */ export function isNonTerminalErrorRuntimeEvent(event: RuntimeEvent): boolean { return event.content?.kind === 'error' && !isTerminalRuntimeEvent(event); } function isAtomicToolBoundaryProjection( event: RuntimeEvent, protocol: ToolBoundaryProtocol | undefined, ): boolean { if (!protocol || event.refs?.operationId === undefined) return false; return event.content?.kind === 'function_call' || event.content?.kind === 'function_response'; }