Files
tutti/packages/agent/host/session_fork_conformance_test.go
agent-radar 2ed3415fb7
Publish Claude Code Binaries / Stage and publish binaries (push) Canceled after 0s
Publish Tutti App Runtime / Build darwin-arm64 (push) Canceled after 0s
Publish Tutti App Runtime / Build darwin-amd64 (push) Canceled after 0s
Publish Tutti App Runtime / Build linux-amd64 (push) Canceled after 0s
Publish Tutti App Runtime / Build windows-amd64 (push) Canceled after 0s
Sync Tutti Agent Skills / sync (push) Canceled after 0s
Windows Agent Adapters / Test Windows agent adapters (push) Canceled after 0s
Windows Daemon Adapters / Test Windows daemon adapters (push) Canceled after 0s
Publish Tutti App Runtime / Publish runtime catalog (push) Canceled after 0s
源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
2026-09-01 20:17:27 +08:00

563 lines
20 KiB
Go

package agenthost_test
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
agenthost "github.com/tutti-os/tutti/packages/agent/host"
hostconformance "github.com/tutti-os/tutti/packages/agent/host/conformance"
storesqlite "github.com/tutti-os/tutti/packages/agent/store-sqlite"
_ "modernc.org/sqlite"
)
func TestSessionForkConformance(t *testing.T) {
for _, scenario := range hostconformance.SessionForkScenarios() {
scenario := scenario
t.Run(scenario.Name, func(t *testing.T) {
driver := &sqliteSessionForkConformanceDriver{t: t}
if err := hostconformance.RunSessionFork(t.Context(), driver, scenario); err != nil {
t.Fatal(err)
}
})
}
}
func TestPreparedSessionForkRecoveryKeepsSnapshotAndDoesNotFenceSourceWrites(t *testing.T) {
driver := &sqliteSessionForkConformanceDriver{t: t}
if err := driver.ResetSessionFork(t.Context(), hostconformance.SessionForkFixture{}); err != nil {
t.Fatal(err)
}
if _, _, err := driver.store.PrepareSessionFork(
t.Context(),
storesqlite.SessionForkPrepare{
OperationID: "operation-abandoned",
WorkspaceID: "workspace-fork",
RequestID: "request-abandoned",
RequestHash: "hash-abandoned",
SourceAgentSessionID: "session-source",
TargetAgentSessionID: "session-abandoned-target",
SourceTurnID: "turn-boundary",
PointKind: storesqlite.SessionForkPointThroughTurn,
DriverKind: "codex-app-server",
DriverVersion: "1",
OccurredAtUnixMS: 40,
},
); err != nil {
t.Fatal(err)
}
if err := driver.host.RecoverSessionForks(t.Context()); err != nil {
t.Fatal(err)
}
operation, found, err := driver.store.GetSessionForkOperation(
t.Context(), "workspace-fork", "operation-abandoned",
)
if err != nil || !found ||
operation.Status != storesqlite.SessionForkStatusCommitted {
t.Fatalf("recovered operation=%#v found=%v error=%v", operation, found, err)
}
if _, err := driver.store.ReportSessionState(
t.Context(),
storesqlite.SessionStateReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Kind: storesqlite.SessionKindRoot,
Provider: "codex",
ProviderSessionID: "provider-source",
OccurredAtUnixMS: 50,
},
); err != nil {
t.Fatalf("source write was fenced by prepared fork: %v", err)
}
}
func TestHostColdRecoveryQuarantinesPermanentlyInconsistentAcceptedFork(t *testing.T) {
driver := &sqliteSessionForkConformanceDriver{t: t}
if err := driver.ResetSessionFork(t.Context(), hostconformance.SessionForkFixture{
RecoverProviderAccepted: true,
RecoverPermanentlyInconsistent: true,
}); err != nil {
t.Fatal(err)
}
before, err := driver.store.ListSessionTurns(
t.Context(), "workspace-fork", "session-source",
)
if err != nil || len(before) != 29 {
t.Fatalf("source turns before recovery=%d error=%v, want 29", len(before), err)
}
if err := driver.host.Recover(t.Context()); err != nil {
t.Fatalf("Host.Recover(): %v", err)
}
operation, found, err := driver.store.GetSessionForkOperation(
t.Context(), "workspace-fork", "operation-fork",
)
if err != nil || !found || operation.Status != storesqlite.SessionForkStatusFailed {
t.Fatalf("quarantined operation=%#v found=%v error=%v", operation, found, err)
}
if operation.TargetProviderSessionID != "provider-target" ||
len(operation.TargetProviderTurnBindings) != 3 ||
operation.StateBindingMode != string(agenthost.SessionForkStateBindingProviderOwned) ||
operation.StateBindingReceipt != "conformance-provider-owned-receipt" {
t.Fatalf("provider acceptance evidence was not retained: %#v", operation)
}
if !strings.Contains(operation.LastError, "returned 3 bindings for 29 provider-bound canonical turns") {
t.Fatalf("quarantine diagnostic=%q", operation.LastError)
}
after, err := driver.store.ListSessionTurns(
t.Context(), "workspace-fork", "session-source",
)
if err != nil || len(after) != 29 {
t.Fatalf("source turns after recovery=%d error=%v, want 29", len(after), err)
}
if _, found, err := driver.store.GetSession(
t.Context(), "workspace-fork", "session-target",
); err != nil || found {
t.Fatalf("incomplete target found=%v error=%v", found, err)
}
for _, table := range []string{
"workspace_agent_session_fork_target_reservations",
"workspace_agent_session_fork_boundary_barriers",
} {
var count int
if err := driver.db.QueryRowContext(
t.Context(),
"SELECT COUNT(*) FROM "+table+" WHERE operation_id = ?",
"operation-fork",
).Scan(&count); err != nil || count != 0 {
t.Fatalf("%s count=%d error=%v, want 0", table, count, err)
}
}
nextAt := operation.CompletedAtUnixMS + 1
next, created, err := driver.store.PrepareSessionFork(
t.Context(),
storesqlite.SessionForkPrepare{
OperationID: "operation-next",
WorkspaceID: "workspace-fork",
RequestID: "request-next",
RequestHash: "recovery-next",
SourceAgentSessionID: "session-source",
TargetAgentSessionID: "session-next-target",
SourceTurnID: "turn-boundary",
PointKind: storesqlite.SessionForkPointThroughTurn,
DriverKind: "codex-app-server",
DriverVersion: "1",
OccurredAtUnixMS: nextAt,
},
)
if err != nil || !created {
t.Fatalf("prepare following fork created=%v error=%v", created, err)
}
if _, _, err := driver.store.MarkSessionForkDispatching(
t.Context(), next.WorkspaceID, next.OperationID, nextAt+1,
); err != nil {
t.Fatal(err)
}
bindings := make([]storesqlite.SessionForkProviderTurnBinding, 29)
for index := range bindings {
bindings[index] = storesqlite.SessionForkProviderTurnBinding{
ProviderTurnID: fmt.Sprintf("next-provider-turn-%02d", index+1),
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
}
}
if _, _, err := driver.store.RecordSessionForkProviderResult(
t.Context(),
storesqlite.SessionForkProviderResult{
WorkspaceID: next.WorkspaceID,
OperationID: next.OperationID,
Status: storesqlite.SessionForkStatusProviderAccepted,
TargetProviderSessionID: "provider-next-target",
TargetProviderTurnBindings: bindings,
StateBindingMode: string(agenthost.SessionForkStateBindingProviderOwned),
StateBindingReceipt: "next-provider-owned-receipt",
OccurredAtUnixMS: nextAt + 2,
},
); err != nil {
t.Fatal(err)
}
if err := driver.host.Recover(t.Context()); err != nil {
t.Fatalf("Host.Recover() following operation: %v", err)
}
next, found, err = driver.store.GetSessionForkOperation(
t.Context(), "workspace-fork", "operation-next",
)
if err != nil || !found || next.Status != storesqlite.SessionForkStatusCommitted {
t.Fatalf("following operation=%#v found=%v error=%v", next, found, err)
}
if driver.runtime.forkCalls != 0 {
t.Fatalf("provider ForkSession calls=%d, want 0", driver.runtime.forkCalls)
}
}
func TestHostColdRecoveryFailsWhenAcceptedForkQuarantineCannotPersist(t *testing.T) {
driver := &sqliteSessionForkConformanceDriver{t: t}
if err := driver.ResetSessionFork(t.Context(), hostconformance.SessionForkFixture{
RecoverProviderAccepted: true,
RecoverPermanentlyInconsistent: true,
}); err != nil {
t.Fatal(err)
}
persistErr := errors.New("injected accepted fork quarantine persistence failure")
failingStore := &failAcceptedSessionForkStore{
Store: driver.store,
err: persistErr,
}
driver.host = agenthost.New(agenthost.Config{
SessionForks: failingStore,
SessionForkRecovery: failingStore,
SessionForkRuntime: driver.runtime,
})
if err := driver.host.Recover(t.Context()); !errors.Is(err, persistErr) {
t.Fatalf("Host.Recover() error=%v, want persistence failure", err)
}
operation, found, err := driver.store.GetSessionForkOperation(
t.Context(), "workspace-fork", "operation-fork",
)
if err != nil || !found || operation.Status != storesqlite.SessionForkStatusProviderAccepted {
t.Fatalf("operation after failed quarantine=%#v found=%v error=%v", operation, found, err)
}
}
type sqliteSessionForkConformanceDriver struct {
t *testing.T
db *sql.DB
host *agenthost.Host
store *storesqlite.Store
runtime *sessionForkConformanceRuntime
}
func (d *sqliteSessionForkConformanceDriver) ResetSessionFork(
ctx context.Context,
fixture hostconformance.SessionForkFixture,
) error {
db, err := sql.Open(
"sqlite",
filepath.Join(d.t.TempDir(), "session-fork-conformance.db"),
)
if err != nil {
return err
}
d.t.Cleanup(func() { _ = db.Close() })
db.SetMaxOpenConns(1)
d.db = db
d.store = storesqlite.New(db, storesqlite.Options{})
if err := d.store.Migrate(ctx); err != nil {
return err
}
if _, err := d.store.ReportSessionState(ctx, storesqlite.SessionStateReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Kind: storesqlite.SessionKindRoot,
Origin: "user",
Provider: "codex",
ProviderSessionID: "provider-source",
Cwd: "/workspace",
OccurredAtUnixMS: 10,
}); err != nil {
return err
}
turnCount := 1
if fixture.RecoverPermanentlyInconsistent {
turnCount = 29
}
lastSeededAt := int64(0)
for index := 0; index < turnCount; index++ {
turnID := fmt.Sprintf("turn-history-%02d", index+1)
providerTurnID := fmt.Sprintf("provider-turn-history-%02d", index+1)
messageID := fmt.Sprintf("message-history-%02d", index+1)
if index == turnCount-1 {
turnID = "turn-boundary"
providerTurnID = "provider-turn"
messageID = "message-boundary"
}
runningAt := int64(20 + index*3)
if result, err := d.store.ReportActivityState(ctx, storesqlite.ActivityStateReport{
Session: storesqlite.SessionStateReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Kind: storesqlite.SessionKindRoot,
Origin: "user",
Provider: "codex",
ProviderSessionID: "provider-source",
Cwd: "/workspace",
OccurredAtUnixMS: runningAt,
},
Turn: &storesqlite.TurnTransition{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
TurnID: turnID,
Phase: storesqlite.TurnPhaseRunning,
OccurredAtUnixMS: runningAt,
},
RootProviderTurn: &storesqlite.RootProviderTurnTransition{
WorkspaceID: "workspace-fork",
RootAgentSessionID: "session-source",
RootTurnID: turnID,
ProviderTurnID: providerTurnID,
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
Phase: storesqlite.RootProviderTurnPhaseRunning,
OccurredAtUnixMS: runningAt,
},
}); err != nil || !result.TurnAccepted || !result.RootTurnAccepted {
return errors.Join(err, fmt.Errorf("seed running fork turn %d was rejected", index+1))
}
if _, err := d.store.ReportSessionMessages(ctx, storesqlite.SessionMessageReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Origin: "runtime",
Messages: []storesqlite.MessageUpdate{{
MessageID: messageID,
TurnID: turnID,
Role: "assistant",
Kind: "text",
Status: "completed",
Payload: map[string]any{"text": "complete"},
OccurredAtUnixMS: runningAt + 1,
}},
}); err != nil {
return err
}
lastSeededAt = runningAt + 2
if result, err := d.store.ReportActivityState(ctx, storesqlite.ActivityStateReport{
Session: storesqlite.SessionStateReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Kind: storesqlite.SessionKindRoot,
Origin: "user",
Provider: "codex",
ProviderSessionID: "provider-source",
Cwd: "/workspace",
OccurredAtUnixMS: lastSeededAt,
},
RootProviderTurn: &storesqlite.RootProviderTurnTransition{
WorkspaceID: "workspace-fork",
RootAgentSessionID: "session-source",
RootTurnID: turnID,
ProviderTurnID: providerTurnID,
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
Phase: storesqlite.RootProviderTurnPhaseCompleted,
Outcome: storesqlite.TurnOutcomeCompleted,
OccurredAtUnixMS: lastSeededAt,
},
}); err != nil || !result.RootTurnAccepted {
return errors.Join(err, fmt.Errorf("seed settled fork turn %d was rejected", index+1))
}
}
if fixture.KeepSourceActive {
if result, err := d.store.ReportActivityState(ctx, storesqlite.ActivityStateReport{
Session: storesqlite.SessionStateReport{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
Kind: storesqlite.SessionKindRoot,
Origin: "user",
Provider: "codex",
ProviderSessionID: "provider-source",
Cwd: "/workspace",
OccurredAtUnixMS: lastSeededAt + 1,
},
Turn: &storesqlite.TurnTransition{
WorkspaceID: "workspace-fork",
AgentSessionID: "session-source",
TurnID: "turn-active",
Phase: storesqlite.TurnPhaseRunning,
OccurredAtUnixMS: lastSeededAt + 1,
},
RootProviderTurn: &storesqlite.RootProviderTurnTransition{
WorkspaceID: "workspace-fork",
RootAgentSessionID: "session-source",
RootTurnID: "turn-active",
ProviderTurnID: "provider-turn-active",
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
Phase: storesqlite.RootProviderTurnPhaseRunning,
OccurredAtUnixMS: lastSeededAt + 1,
},
}); err != nil || !result.TurnAccepted || !result.RootTurnAccepted {
return errors.Join(err, errors.New("seed active source turn was rejected"))
}
}
forkStore := &failOnceSessionForkStore{
Store: d.store,
failNextCommit: fixture.FailFirstLocalCommit,
}
d.runtime = &sessionForkConformanceRuntime{}
d.host = agenthost.New(agenthost.Config{
SessionForks: forkStore,
SessionForkRecovery: forkStore,
SessionForkRuntime: d.runtime,
})
if fixture.KeepSourceActive {
if _, supported, err := forkStore.CheckSessionForkThroughTurn(
ctx, "workspace-fork", "session-source", "turn-boundary",
); err != nil || !supported {
session, sessionFound, sessionErr := forkStore.GetSession(
ctx, "workspace-fork", "session-source",
)
turn, turnFound, turnErr := forkStore.GetTurn(
ctx, "workspace-fork", "session-source", "turn-boundary",
)
return errors.Join(err, fmt.Errorf(
"settled fork boundary became ineligible while source was active: sessionFound=%v session=%#v sessionErr=%v turnFound=%v turn=%#v turnErr=%v",
sessionFound, session, sessionErr, turnFound, turn, turnErr,
))
}
}
if !fixture.RecoverProviderAccepted {
return nil
}
operationAt := lastSeededAt + 10
operation, _, err := d.store.PrepareSessionFork(ctx, storesqlite.SessionForkPrepare{
OperationID: "operation-fork",
WorkspaceID: "workspace-fork",
RequestID: "request-fork",
RequestHash: "recovery-fixture",
SourceAgentSessionID: "session-source",
TargetAgentSessionID: "session-target",
SourceTurnID: "turn-boundary",
PointKind: storesqlite.SessionForkPointThroughTurn,
DriverKind: "codex-app-server",
DriverVersion: "1",
OccurredAtUnixMS: operationAt,
})
if err != nil {
return err
}
if _, _, err := d.store.MarkSessionForkDispatching(
ctx, operation.WorkspaceID, operation.OperationID, operationAt+1,
); err != nil {
return err
}
targetBindings := []storesqlite.SessionForkProviderTurnBinding{{
ProviderTurnID: "forked-provider-turn",
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
}}
if fixture.RecoverPermanentlyInconsistent {
targetBindings = append(targetBindings,
storesqlite.SessionForkProviderTurnBinding{
ProviderTurnID: "forked-provider-turn-2",
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
},
storesqlite.SessionForkProviderTurnBinding{
ProviderTurnID: "forked-provider-turn-3",
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
},
)
}
_, _, err = d.store.RecordSessionForkProviderResult(
ctx,
storesqlite.SessionForkProviderResult{
WorkspaceID: operation.WorkspaceID,
OperationID: operation.OperationID,
Status: storesqlite.SessionForkStatusProviderAccepted,
TargetProviderSessionID: "provider-target",
TargetProviderTurnBindings: targetBindings,
StateBindingMode: string(agenthost.SessionForkStateBindingProviderOwned),
StateBindingReceipt: "conformance-provider-owned-receipt",
OccurredAtUnixMS: operationAt + 2,
},
)
return err
}
func (d *sqliteSessionForkConformanceDriver) ForkSession(
ctx context.Context,
input agenthost.ForkSessionInput,
) (agenthost.ForkSessionResult, error) {
return d.host.ForkSession(ctx, input)
}
func (d *sqliteSessionForkConformanceDriver) GetSessionForkOperation(
ctx context.Context,
workspaceID, operationID string,
) (agenthost.ForkSessionResult, bool, error) {
return d.host.GetSessionForkOperation(ctx, workspaceID, operationID)
}
func (d *sqliteSessionForkConformanceDriver) RecoverSessionForks(
ctx context.Context,
) error {
return d.host.RecoverSessionForks(ctx)
}
func (d *sqliteSessionForkConformanceDriver) SessionForkMetrics() hostconformance.SessionForkMetrics {
return hostconformance.SessionForkMetrics{
ProviderForkCalls: d.runtime.forkCalls,
}
}
type failOnceSessionForkStore struct {
*storesqlite.Store
failNextCommit bool
}
type failAcceptedSessionForkStore struct {
*storesqlite.Store
err error
}
func (s *failAcceptedSessionForkStore) FailAcceptedSessionFork(
context.Context,
string, string, string,
int64,
) (storesqlite.SessionForkOperation, bool, error) {
return storesqlite.SessionForkOperation{}, false, s.err
}
func (s *failOnceSessionForkStore) CommitSessionFork(
ctx context.Context,
workspaceID, operationID string,
occurredAtUnixMS int64,
) (storesqlite.SessionForkCommitResult, error) {
if s.failNextCommit {
s.failNextCommit = false
return storesqlite.SessionForkCommitResult{}, errors.New("injected local commit failure")
}
return s.Store.CommitSessionFork(ctx, workspaceID, operationID, occurredAtUnixMS)
}
type sessionForkConformanceRuntime struct {
forkCalls int
}
func (*sessionForkConformanceRuntime) ResolveSessionFork(
context.Context,
agenthost.ProviderRuntimeSession,
) (agenthost.SessionForkDriverDescriptor, error) {
return agenthost.SessionForkDriverDescriptor{
Kind: "codex-app-server", Version: "1", ThroughTurn: true,
StateBindingMode: agenthost.SessionForkStateBindingProviderOwned,
}, nil
}
func (*sessionForkConformanceRuntime) CanForkProviderTurn(
_ context.Context,
input agenthost.RuntimeProviderTurnForkabilityInput,
) (bool, error) {
return input.ProviderTurnID != "" &&
len(input.ProviderTurnBindingJSON) > 0, nil
}
func (r *sessionForkConformanceRuntime) ForkSession(
_ context.Context,
input agenthost.RuntimeSessionForkInput,
) (agenthost.RuntimeSessionForkResult, error) {
r.forkCalls++
return agenthost.RuntimeSessionForkResult{
ProviderSessionID: "provider-target",
TargetProviderTurnBindings: []agenthost.SessionForkProviderTurnBinding{{
ProviderTurnID: "forked-" + input.SourceProviderTurnID,
ProviderTurnBindingJSON: json.RawMessage(`{"schemaVersion":1}`),
}},
StateBindingMode: agenthost.SessionForkStateBindingProviderOwned,
StateBindingReceipt: "conformance-provider-owned-receipt",
DeliveryDisposition: agenthost.SessionForkDeliveryAccepted,
}, nil
}