Publish Claude Code Binaries / Stage and publish binaries (push) Canceled after 0s
Publish Tutti App Runtime / Build darwin-arm64 (push) Canceled after 0s
Publish Tutti App Runtime / Build darwin-amd64 (push) Canceled after 0s
Publish Tutti App Runtime / Build linux-amd64 (push) Canceled after 0s
Publish Tutti App Runtime / Build windows-amd64 (push) Canceled after 0s
Sync Tutti Agent Skills / sync (push) Canceled after 0s
Windows Agent Adapters / Test Windows agent adapters (push) Canceled after 0s
Windows Daemon Adapters / Test Windows daemon adapters (push) Canceled after 0s
Publish Tutti App Runtime / Publish runtime catalog (push) Canceled after 0s
777 lines
27 KiB
Go
777 lines
27 KiB
Go
package workspace
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
executionbiz "github.com/tutti-os/tutti/services/tuttid/biz/tuttimodeexecution"
|
|
)
|
|
|
|
func prepareTuttiModeMainWakeTx(
|
|
ctx context.Context,
|
|
tx *sql.Tx,
|
|
workspaceID string,
|
|
executionID string,
|
|
checkpoint executionbiz.Checkpoint,
|
|
now time.Time,
|
|
) error {
|
|
return prepareTuttiModeMainWakeSequenceTx(
|
|
ctx, tx, workspaceID, executionID, checkpoint, 1, now, now,
|
|
)
|
|
}
|
|
|
|
func prepareTuttiModeMainWakeSequenceTx(
|
|
ctx context.Context,
|
|
tx *sql.Tx,
|
|
workspaceID string,
|
|
executionID string,
|
|
checkpoint executionbiz.Checkpoint,
|
|
sequence int64,
|
|
dueAt time.Time,
|
|
now time.Time,
|
|
) error {
|
|
if checkpoint.Status != executionbiz.CheckpointStatusActive {
|
|
return nil
|
|
}
|
|
wakeID, ok := executionbiz.MainWakeID(checkpoint.ID, sequence)
|
|
if !ok {
|
|
return executionbiz.ErrWakeIntegrity
|
|
}
|
|
clientSubmitID, ok := executionbiz.MainWakeClientSubmitID(wakeID)
|
|
if !ok {
|
|
return executionbiz.ErrWakeIntegrity
|
|
}
|
|
result, err := tx.ExecContext(ctx, `
|
|
INSERT OR IGNORE INTO workspace_tutti_execution_wakes (
|
|
workspace_id, execution_id, checkpoint_id, wake_id, target_kind,
|
|
wake_sequence, client_submit_id, target_session_id, status,
|
|
due_at_unix_ms, created_at_unix_ms, updated_at_unix_ms
|
|
)
|
|
SELECT e.workspace_id, e.execution_id, ?, ?, 'main', ?, ?,
|
|
e.source_session_id, 'prepared', ?, ?, ?
|
|
FROM workspace_tutti_executions e
|
|
JOIN workspace_tutti_execution_checkpoints c
|
|
ON c.workspace_id = e.workspace_id AND c.execution_id = e.execution_id
|
|
WHERE e.workspace_id = ? AND e.execution_id = ?
|
|
AND c.checkpoint_id = ? AND c.status = 'active'
|
|
`, checkpoint.ID, wakeID, sequence, clientSubmitID, unixMs(dueAt), unixMs(now), unixMs(now),
|
|
workspaceID, executionID, checkpoint.ID)
|
|
if err != nil {
|
|
return fmt.Errorf("prepare Tutti mode main wake: %w", err)
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil {
|
|
return fmt.Errorf("inspect prepared Tutti mode main wake: %w", err)
|
|
}
|
|
if affected == 0 {
|
|
var equivalent int
|
|
err := tx.QueryRowContext(ctx, `
|
|
SELECT COUNT(*)
|
|
FROM workspace_tutti_execution_wakes w
|
|
JOIN workspace_tutti_executions e
|
|
ON e.workspace_id = w.workspace_id AND e.execution_id = w.execution_id
|
|
JOIN workspace_tutti_execution_checkpoints c
|
|
ON c.workspace_id = w.workspace_id AND c.execution_id = w.execution_id
|
|
AND c.checkpoint_id = w.checkpoint_id
|
|
WHERE w.workspace_id = ? AND w.execution_id = ? AND w.wake_id = ?
|
|
AND w.checkpoint_id = ? AND w.target_kind = 'main' AND w.wake_sequence = ?
|
|
AND w.client_submit_id = ? AND w.target_session_id = e.source_session_id
|
|
AND w.status IN ('prepared', 'leased', 'dispatched', 'turn_settled')
|
|
AND c.status = 'active'
|
|
`, workspaceID, executionID, wakeID, checkpoint.ID, sequence, clientSubmitID).Scan(&equivalent)
|
|
if err != nil || equivalent != 1 {
|
|
if err != nil {
|
|
return fmt.Errorf("verify Tutti mode main wake: %w", err)
|
|
}
|
|
return executionbiz.ErrWakeIntegrity
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *SQLiteStore) PrepareDueTuttiModeExecutionWatchdogs(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) error {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return err
|
|
}
|
|
workspaceID = strings.TrimSpace(workspaceID)
|
|
tx, err := s.writeDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("begin Tutti mode watchdog preparation: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
if err := repairStaleActiveTuttiModeCheckpointRevisionsTx(
|
|
ctx, tx, workspaceID, now,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
rows, err := tx.QueryContext(ctx, `
|
|
SELECT execution_id, graph_revision, watchdog_due_at_unix_ms
|
|
FROM workspace_tutti_executions e
|
|
JOIN workspace_issues i
|
|
ON i.workspace_id = e.workspace_id AND i.issue_id = e.issue_id
|
|
WHERE e.workspace_id = ?
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND i.dispatch_paused = 0
|
|
AND e.watchdog_due_at_unix_ms <= ?
|
|
ORDER BY e.watchdog_due_at_unix_ms ASC, e.execution_id ASC
|
|
`, workspaceID, unixMs(now))
|
|
if err != nil {
|
|
return fmt.Errorf("list due Tutti mode watchdogs: %w", err)
|
|
}
|
|
type dueExecution struct {
|
|
id string
|
|
graphRevision int64
|
|
dueAtUnixMS int64
|
|
}
|
|
var due []dueExecution
|
|
for rows.Next() {
|
|
var item dueExecution
|
|
if err := rows.Scan(&item.id, &item.graphRevision, &item.dueAtUnixMS); err != nil {
|
|
_ = rows.Close()
|
|
return fmt.Errorf("scan due Tutti mode watchdog: %w", err)
|
|
}
|
|
due = append(due, item)
|
|
}
|
|
if err := rows.Close(); err != nil {
|
|
return fmt.Errorf("close due Tutti mode watchdogs: %w", err)
|
|
}
|
|
for _, item := range due {
|
|
if err := prepareDueTuttiModeWatchdogTx(
|
|
ctx, tx, workspaceID, item.id, item.graphRevision,
|
|
time.UnixMilli(item.dueAtUnixMS).UTC(), now,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("commit Tutti mode watchdog preparation: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func repairStaleActiveTuttiModeCheckpointRevisionsTx(
|
|
ctx context.Context,
|
|
tx *sql.Tx,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) error {
|
|
_, err := tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_executions
|
|
SET watchdog_due_at_unix_ms = MIN(watchdog_due_at_unix_ms, ?),
|
|
updated_at_unix_ms = ?
|
|
WHERE workspace_id = ?
|
|
AND status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM workspace_tutti_execution_checkpoints c
|
|
WHERE c.workspace_id = workspace_tutti_executions.workspace_id
|
|
AND c.execution_id = workspace_tutti_executions.execution_id
|
|
AND c.status = 'active'
|
|
AND c.kind IN ('task_settled', 'task_failed', 'task_canceled', 'watchdog')
|
|
AND c.graph_revision < workspace_tutti_executions.graph_revision
|
|
)
|
|
`, unixMs(now), unixMs(now), workspaceID)
|
|
if err != nil {
|
|
return fmt.Errorf("advance stale Tutti mode checkpoint recovery: %w", err)
|
|
}
|
|
_, err = tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_checkpoints
|
|
SET graph_revision = (
|
|
SELECT e.graph_revision
|
|
FROM workspace_tutti_executions e
|
|
WHERE e.workspace_id = workspace_tutti_execution_checkpoints.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_checkpoints.execution_id
|
|
),
|
|
updated_at_unix_ms = ?
|
|
WHERE workspace_id = ?
|
|
AND status = 'active'
|
|
AND kind IN ('task_settled', 'task_failed', 'task_canceled', 'watchdog')
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM workspace_tutti_executions e
|
|
WHERE e.workspace_id = workspace_tutti_execution_checkpoints.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_checkpoints.execution_id
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND workspace_tutti_execution_checkpoints.graph_revision < e.graph_revision
|
|
)
|
|
`, unixMs(now), workspaceID)
|
|
if err != nil {
|
|
return fmt.Errorf("rebind stale active Tutti mode checkpoint revision: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func prepareDueTuttiModeWatchdogTx(
|
|
ctx context.Context,
|
|
tx *sql.Tx,
|
|
workspaceID string,
|
|
executionID string,
|
|
graphRevision int64,
|
|
dueAt time.Time,
|
|
now time.Time,
|
|
) error {
|
|
row := tx.QueryRowContext(ctx, `
|
|
SELECT checkpoint_id, execution_id, kind, status, sequence, graph_revision,
|
|
subject_task_id, subject_run_id, creation_reason, requires_goal_review,
|
|
created_at_unix_ms, updated_at_unix_ms, resolved_at_unix_ms
|
|
FROM workspace_tutti_execution_checkpoints
|
|
WHERE workspace_id = ? AND execution_id = ? AND status = 'active'
|
|
`, workspaceID, executionID)
|
|
checkpoint, err := scanTuttiModeExecutionCheckpoint(row)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
var maxSequence int64
|
|
if err := tx.QueryRowContext(ctx, `
|
|
SELECT COALESCE(MAX(sequence), 0)
|
|
FROM workspace_tutti_execution_checkpoints
|
|
WHERE workspace_id = ? AND execution_id = ?
|
|
`, workspaceID, executionID).Scan(&maxSequence); err != nil {
|
|
return fmt.Errorf("get watchdog checkpoint sequence: %w", err)
|
|
}
|
|
checkpointID, ok := executionbiz.WatchdogCheckpointID(
|
|
executionID, maxSequence+1,
|
|
)
|
|
if !ok {
|
|
return executionbiz.ErrWakeIntegrity
|
|
}
|
|
checkpoint = executionbiz.Checkpoint{
|
|
ID: checkpointID, ExecutionID: executionID,
|
|
Kind: executionbiz.CheckpointKindWatchdog, Status: executionbiz.CheckpointStatusActive,
|
|
Sequence: maxSequence + 1, GraphRevision: graphRevision,
|
|
CreationReason: "fixed_inactivity_watchdog",
|
|
CreatedAt: now, UpdatedAt: now,
|
|
}
|
|
if err := insertTuttiModeExecutionCheckpoint(
|
|
ctx, tx, workspaceID, checkpoint,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_executions
|
|
SET status = 'awaiting_main', updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND execution_id = ?
|
|
AND status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
`, unixMs(now), workspaceID, executionID); err != nil {
|
|
return fmt.Errorf("activate Tutti mode watchdog checkpoint: %w", err)
|
|
}
|
|
return prepareTuttiModeMainWakeSequenceTx(
|
|
ctx, tx, workspaceID, executionID, checkpoint, 1, dueAt, now,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("get active Tutti mode watchdog checkpoint: %w", err)
|
|
}
|
|
var latestStatus string
|
|
var latestSequence int64
|
|
err = tx.QueryRowContext(ctx, `
|
|
SELECT status, wake_sequence
|
|
FROM workspace_tutti_execution_wakes
|
|
WHERE workspace_id = ? AND execution_id = ? AND checkpoint_id = ?
|
|
AND target_kind = 'main'
|
|
ORDER BY wake_sequence DESC LIMIT 1
|
|
`, workspaceID, executionID, checkpoint.ID).Scan(&latestStatus, &latestSequence)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return prepareTuttiModeMainWakeSequenceTx(
|
|
ctx, tx, workspaceID, executionID, checkpoint, 1, dueAt, now,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("get active checkpoint wake generation: %w", err)
|
|
}
|
|
if latestStatus != string(executionbiz.WakeStatusTurnSettled) {
|
|
return nil
|
|
}
|
|
return prepareTuttiModeMainWakeSequenceTx(
|
|
ctx, tx, workspaceID, executionID, checkpoint,
|
|
latestSequence+1, dueAt, now,
|
|
)
|
|
}
|
|
|
|
func (s *SQLiteStore) ObserveTuttiModeSourceSessionActivity(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
sessionID string,
|
|
now time.Time,
|
|
) error {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return err
|
|
}
|
|
workspaceID = strings.TrimSpace(workspaceID)
|
|
sessionID = strings.TrimSpace(sessionID)
|
|
tx, err := s.writeDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("begin Tutti mode source-session activity: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
if err := observeTuttiModeSourceSessionActivityTx(
|
|
ctx, tx, workspaceID, sessionID, now,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("commit Tutti mode source-session activity: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func acknowledgeTuttiModeCheckpointWakesTx(
|
|
ctx context.Context,
|
|
tx *sql.Tx,
|
|
workspaceID string,
|
|
executionID string,
|
|
checkpointID string,
|
|
now time.Time,
|
|
) error {
|
|
result, err := tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'acknowledged', lease_owner = '', lease_expires_at_unix_ms = 0,
|
|
acknowledged_at_unix_ms = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND execution_id = ? AND checkpoint_id = ?
|
|
AND status IN ('prepared', 'leased', 'dispatched', 'turn_settled')
|
|
`, unixMs(now), unixMs(now), workspaceID, executionID, checkpointID)
|
|
if err != nil {
|
|
return fmt.Errorf("acknowledge Tutti mode checkpoint wakes: %w", err)
|
|
}
|
|
return requireRowsAffected(
|
|
result,
|
|
executionbiz.ErrWakeRejected,
|
|
"acknowledge Tutti mode checkpoint wake",
|
|
)
|
|
}
|
|
|
|
func (s *SQLiteStore) ListTuttiModeExecutionWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
issueID string,
|
|
) ([]executionbiz.Wake, error) {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return nil, err
|
|
}
|
|
rows, err := s.readDB.QueryContext(ctx, wakeSelectQuery+`
|
|
WHERE e.workspace_id = ? AND e.issue_id = ?
|
|
ORDER BY c.sequence ASC, w.wake_sequence ASC
|
|
`, strings.TrimSpace(workspaceID), strings.TrimSpace(issueID))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list Tutti mode execution wakes: %w", err)
|
|
}
|
|
return scanTuttiModeWakes(rows)
|
|
}
|
|
|
|
func (s *SQLiteStore) ListDispatchableTuttiModeMainWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) ([]executionbiz.Wake, error) {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return nil, err
|
|
}
|
|
rows, err := s.readDB.QueryContext(ctx, wakeSelectQuery+`
|
|
WHERE e.workspace_id = ?
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND c.status = 'active'
|
|
AND w.target_kind = 'main' AND w.status = 'prepared'
|
|
AND w.due_at_unix_ms <= ?
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM workspace_issues i
|
|
WHERE i.workspace_id = e.workspace_id
|
|
AND i.issue_id = e.issue_id
|
|
AND i.dispatch_paused = 0
|
|
)
|
|
ORDER BY w.due_at_unix_ms ASC, e.execution_id ASC, c.sequence ASC, w.wake_sequence ASC
|
|
`, strings.TrimSpace(workspaceID), unixMs(now))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list dispatchable Tutti mode main wakes: %w", err)
|
|
}
|
|
return scanTuttiModeWakes(rows)
|
|
}
|
|
|
|
func (s *SQLiteStore) ListDispatchedTuttiModeMainWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
) ([]executionbiz.Wake, error) {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return nil, err
|
|
}
|
|
rows, err := s.readDB.QueryContext(ctx, wakeSelectQuery+`
|
|
WHERE e.workspace_id = ?
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND c.status = 'active'
|
|
AND w.status = 'dispatched'
|
|
AND w.target_kind = 'main'
|
|
ORDER BY w.dispatched_at_unix_ms ASC, e.execution_id ASC, c.sequence ASC
|
|
`, strings.TrimSpace(workspaceID))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list dispatched Tutti mode main wakes: %w", err)
|
|
}
|
|
return scanTuttiModeWakes(rows)
|
|
}
|
|
|
|
func (s *SQLiteStore) ListCorruptedTuttiModeMainWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) ([]executionbiz.Wake, error) {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return nil, err
|
|
}
|
|
rows, err := s.readDB.QueryContext(ctx, wakeSelectQuery+`
|
|
WHERE e.workspace_id = ?
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND c.status = 'active'
|
|
AND w.status = 'prepared' AND w.due_at_unix_ms <= ?
|
|
AND w.target_kind != 'main'
|
|
AND w.wake_sequence > 0
|
|
AND w.wake_id = c.checkpoint_id || ':wake:main:' || CAST(w.wake_sequence AS TEXT)
|
|
AND w.client_submit_id = 'tutti-execution-wake:' || w.wake_id
|
|
ORDER BY w.due_at_unix_ms ASC, e.execution_id ASC, c.sequence ASC
|
|
`, strings.TrimSpace(workspaceID), unixMs(now))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list corrupted Tutti mode main wakes: %w", err)
|
|
}
|
|
return scanTuttiModeWakes(rows)
|
|
}
|
|
|
|
func (s *SQLiteStore) GetTuttiModeExecutionWake(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
wakeID string,
|
|
) (executionbiz.Wake, bool, error) {
|
|
if err := s.ensureIssueDatabase(); err != nil {
|
|
return executionbiz.Wake{}, false, err
|
|
}
|
|
wake, err := scanTuttiModeWake(s.readDB.QueryRowContext(ctx, wakeSelectQuery+`
|
|
WHERE e.workspace_id = ? AND w.wake_id = ?
|
|
`, strings.TrimSpace(workspaceID), strings.TrimSpace(wakeID)))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return executionbiz.Wake{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return executionbiz.Wake{}, false, fmt.Errorf("get Tutti mode execution wake: %w", err)
|
|
}
|
|
return wake, true, nil
|
|
}
|
|
|
|
func (s *SQLiteStore) ClaimTuttiModeExecutionWake(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
wakeID string,
|
|
leaseOwner string,
|
|
now time.Time,
|
|
leaseExpiresAt time.Time,
|
|
) (bool, error) {
|
|
futureActivityCutoff := unixMs(now.Add(-executionbiz.WatchdogInterval))
|
|
result, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'leased', attempt_count = attempt_count + 1,
|
|
lease_owner = ?, lease_expires_at_unix_ms = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND wake_id = ? AND status = 'prepared'
|
|
AND due_at_unix_ms <= ?
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM (`+tuttiModeRelevantSourceActivitySelectSQL+`) relevant_activity
|
|
WHERE relevant_activity.workspace_id =
|
|
workspace_tutti_execution_wakes.workspace_id
|
|
AND relevant_activity.agent_session_id =
|
|
workspace_tutti_execution_wakes.target_session_id
|
|
AND relevant_activity.activity_at_unix_ms > (
|
|
SELECT e.last_orchestrator_activity_at_unix_ms
|
|
FROM workspace_tutti_executions e
|
|
WHERE e.workspace_id = workspace_tutti_execution_wakes.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_wakes.execution_id
|
|
)
|
|
AND relevant_activity.activity_at_unix_ms > ?
|
|
)
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM workspace_tutti_executions e
|
|
JOIN workspace_tutti_execution_checkpoints c
|
|
ON c.workspace_id = e.workspace_id AND c.execution_id = e.execution_id
|
|
JOIN workspace_issues i
|
|
ON i.workspace_id = e.workspace_id AND i.issue_id = e.issue_id
|
|
WHERE e.workspace_id = workspace_tutti_execution_wakes.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_wakes.execution_id
|
|
AND e.source_session_id = workspace_tutti_execution_wakes.target_session_id
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND i.dispatch_paused = 0
|
|
AND c.checkpoint_id = workspace_tutti_execution_wakes.checkpoint_id
|
|
AND c.status = 'active'
|
|
)
|
|
`, leaseOwner, unixMs(leaseExpiresAt), unixMs(now), workspaceID, wakeID,
|
|
unixMs(now), futureActivityCutoff)
|
|
if err != nil {
|
|
return false, fmt.Errorf("claim Tutti mode execution wake: %w", err)
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
return affected == 1, err
|
|
}
|
|
|
|
func (s *SQLiteStore) ReleaseTuttiModeExecutionWake(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
wakeID string,
|
|
leaseOwner string,
|
|
message string,
|
|
now time.Time,
|
|
) error {
|
|
result, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'prepared', lease_owner = '', lease_expires_at_unix_ms = 0,
|
|
last_error = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND wake_id = ? AND status = 'leased' AND lease_owner = ?
|
|
`, strings.TrimSpace(message), unixMs(now), workspaceID, wakeID, leaseOwner)
|
|
if err != nil {
|
|
return fmt.Errorf("release Tutti mode execution wake: %w", err)
|
|
}
|
|
return requireRowsAffected(result, executionbiz.ErrWakeRejected, "release owned Tutti mode execution wake")
|
|
}
|
|
|
|
func (s *SQLiteStore) MarkTuttiModeExecutionWakeDispatched(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
wakeID string,
|
|
leaseOwner string,
|
|
canonicalSessionID string,
|
|
canonicalTurnID string,
|
|
expectedDueAt time.Time,
|
|
now time.Time,
|
|
) error {
|
|
futureActivityCutoff := unixMs(now.Add(-executionbiz.WatchdogInterval))
|
|
result, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'dispatched', canonical_session_id = ?, canonical_turn_id = ?,
|
|
lease_owner = '', lease_expires_at_unix_ms = 0, last_error = '',
|
|
dispatched_at_unix_ms = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND wake_id = ? AND status = 'leased' AND lease_owner = ?
|
|
AND lease_expires_at_unix_ms > ?
|
|
AND due_at_unix_ms = ? AND due_at_unix_ms <= ?
|
|
AND target_session_id = ?
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM (`+tuttiModeRelevantSourceActivitySelectSQL+`) relevant_activity
|
|
WHERE relevant_activity.workspace_id =
|
|
workspace_tutti_execution_wakes.workspace_id
|
|
AND relevant_activity.agent_session_id =
|
|
workspace_tutti_execution_wakes.target_session_id
|
|
AND relevant_activity.activity_at_unix_ms > (
|
|
SELECT e.last_orchestrator_activity_at_unix_ms
|
|
FROM workspace_tutti_executions e
|
|
WHERE e.workspace_id = workspace_tutti_execution_wakes.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_wakes.execution_id
|
|
)
|
|
AND relevant_activity.activity_at_unix_ms > ?
|
|
)
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM workspace_tutti_executions e
|
|
JOIN workspace_tutti_execution_checkpoints c
|
|
ON c.workspace_id = e.workspace_id AND c.execution_id = e.execution_id
|
|
JOIN workspace_issues i
|
|
ON i.workspace_id = e.workspace_id AND i.issue_id = e.issue_id
|
|
WHERE e.workspace_id = workspace_tutti_execution_wakes.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_wakes.execution_id
|
|
AND e.source_session_id = workspace_tutti_execution_wakes.target_session_id
|
|
AND e.status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND i.dispatch_paused = 0
|
|
AND c.checkpoint_id = workspace_tutti_execution_wakes.checkpoint_id
|
|
AND c.status = 'active'
|
|
)
|
|
`, canonicalSessionID, canonicalTurnID, unixMs(now), unixMs(now),
|
|
workspaceID, wakeID, leaseOwner, unixMs(now),
|
|
unixMs(expectedDueAt), unixMs(now), canonicalSessionID,
|
|
futureActivityCutoff)
|
|
if err != nil {
|
|
return fmt.Errorf("mark Tutti mode execution wake dispatched: %w", err)
|
|
}
|
|
return requireRowsAffected(result, executionbiz.ErrWakeRejected, "dispatch owned Tutti mode execution wake")
|
|
}
|
|
|
|
func (s *SQLiteStore) MarkTuttiModeExecutionWakeTurnSettled(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
sessionID string,
|
|
turnID string,
|
|
now time.Time,
|
|
) (bool, error) {
|
|
tx, err := s.writeDB.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return false, fmt.Errorf("begin Tutti mode main wake Turn settlement: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
var executionID string
|
|
err = tx.QueryRowContext(ctx, `
|
|
SELECT execution_id
|
|
FROM workspace_tutti_execution_wakes
|
|
WHERE workspace_id = ? AND target_kind = 'main' AND status = 'dispatched'
|
|
AND canonical_session_id = ? AND canonical_turn_id = ?
|
|
`, workspaceID, sessionID, turnID).Scan(&executionID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("find Tutti mode main wake Turn: %w", err)
|
|
}
|
|
result, err := tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'turn_settled', turn_settled_at_unix_ms = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND target_kind = 'main' AND status = 'dispatched'
|
|
AND canonical_session_id = ? AND canonical_turn_id = ?
|
|
`, unixMs(now), unixMs(now), workspaceID, sessionID, turnID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("settle Tutti mode main wake Turn: %w", err)
|
|
}
|
|
affected, err := result.RowsAffected()
|
|
if err != nil || affected != 1 {
|
|
return false, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_executions
|
|
SET last_orchestrator_activity_at_unix_ms = ?,
|
|
watchdog_due_at_unix_ms = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND execution_id = ?
|
|
AND status IN ('awaiting_schedule', 'running', 'awaiting_main', 'pending_goal_review')
|
|
AND last_orchestrator_activity_at_unix_ms < ?
|
|
`, unixMs(now), unixMs(now.Add(executionbiz.WatchdogInterval)), unixMs(now),
|
|
workspaceID, executionID, unixMs(now)); err != nil {
|
|
return false, fmt.Errorf("reset watchdog after main wake Turn: %w", err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return false, fmt.Errorf("commit Tutti mode main wake Turn settlement: %w", err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (s *SQLiteStore) FailTuttiModeExecutionWakeIntegrity(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
wakeID string,
|
|
message string,
|
|
now time.Time,
|
|
) error {
|
|
result, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'failed', lease_owner = '', lease_expires_at_unix_ms = 0,
|
|
last_error = ?, updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND wake_id = ? AND status IN ('prepared', 'leased')
|
|
`, strings.TrimSpace(message), unixMs(now), workspaceID, wakeID)
|
|
if err != nil {
|
|
return fmt.Errorf("fail invalid Tutti mode execution wake: %w", err)
|
|
}
|
|
return requireRowsAffected(result, executionbiz.ErrWakeRejected, "fail invalid Tutti mode execution wake")
|
|
}
|
|
|
|
func (s *SQLiteStore) RequeueExpiredTuttiModeExecutionWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) error {
|
|
_, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'prepared', lease_owner = '', lease_expires_at_unix_ms = 0,
|
|
updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND status = 'leased' AND lease_expires_at_unix_ms <= ?
|
|
`, unixMs(now), strings.TrimSpace(workspaceID), unixMs(now))
|
|
if err != nil {
|
|
return fmt.Errorf("requeue expired Tutti mode execution wakes: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *SQLiteStore) CancelSuppressedTuttiModeExecutionWakes(
|
|
ctx context.Context,
|
|
workspaceID string,
|
|
now time.Time,
|
|
) error {
|
|
_, err := s.writeDB.ExecContext(ctx, `
|
|
UPDATE workspace_tutti_execution_wakes
|
|
SET status = 'canceled', lease_owner = '', lease_expires_at_unix_ms = 0,
|
|
updated_at_unix_ms = ?
|
|
WHERE workspace_id = ? AND status IN ('prepared', 'leased', 'dispatched', 'turn_settled')
|
|
AND EXISTS (
|
|
SELECT 1 FROM workspace_tutti_executions e
|
|
WHERE e.workspace_id = workspace_tutti_execution_wakes.workspace_id
|
|
AND e.execution_id = workspace_tutti_execution_wakes.execution_id
|
|
AND e.status IN ('orphaned_source', 'completed', 'archiving', 'archived')
|
|
)
|
|
`, unixMs(now), strings.TrimSpace(workspaceID))
|
|
if err != nil {
|
|
return fmt.Errorf("cancel suppressed Tutti mode execution wakes: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const wakeSelectQuery = `
|
|
SELECT w.wake_id, w.workspace_id, w.execution_id, e.issue_id,
|
|
w.checkpoint_id, c.kind, c.graph_revision, w.target_kind,
|
|
e.review_mode, COALESCE(r.review_id, ''),
|
|
COALESCE(r.status, ''), COALESCE(r.verdict, ''),
|
|
COALESCE(r.summary, ''), COALESCE(r.failure_reason, ''),
|
|
w.wake_sequence, w.client_submit_id, e.source_session_id, w.target_session_id,
|
|
w.canonical_session_id, w.canonical_turn_id, w.status,
|
|
w.due_at_unix_ms, w.attempt_count, w.lease_owner,
|
|
w.lease_expires_at_unix_ms, w.dispatched_at_unix_ms,
|
|
w.turn_settled_at_unix_ms, w.acknowledged_at_unix_ms,
|
|
w.last_error, w.created_at_unix_ms, w.updated_at_unix_ms
|
|
FROM workspace_tutti_execution_wakes w
|
|
JOIN workspace_tutti_executions e
|
|
ON e.workspace_id = w.workspace_id AND e.execution_id = w.execution_id
|
|
JOIN workspace_tutti_execution_checkpoints c
|
|
ON c.workspace_id = w.workspace_id AND c.execution_id = w.execution_id
|
|
AND c.checkpoint_id = w.checkpoint_id
|
|
LEFT JOIN workspace_tutti_goal_reviews r
|
|
ON r.workspace_id = w.workspace_id AND r.execution_id = w.execution_id
|
|
AND r.checkpoint_id = w.checkpoint_id
|
|
`
|
|
|
|
func scanTuttiModeWakes(rows *sql.Rows) ([]executionbiz.Wake, error) {
|
|
defer func() { _ = rows.Close() }()
|
|
wakes := make([]executionbiz.Wake, 0)
|
|
for rows.Next() {
|
|
wake, err := scanTuttiModeWake(rows)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scan Tutti mode execution wake: %w", err)
|
|
}
|
|
wakes = append(wakes, wake)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("iterate Tutti mode execution wakes: %w", err)
|
|
}
|
|
return wakes, nil
|
|
}
|
|
|
|
func scanTuttiModeWake(scanner rowScanner) (executionbiz.Wake, error) {
|
|
var wake executionbiz.Wake
|
|
var dueAt, leaseExpiresAt, dispatchedAt, turnSettledAt, acknowledgedAt int64
|
|
var createdAt, updatedAt int64
|
|
err := scanner.Scan(
|
|
&wake.ID, &wake.WorkspaceID, &wake.ExecutionID, &wake.IssueID,
|
|
&wake.CheckpointID, &wake.CheckpointKind, &wake.CheckpointRevision,
|
|
&wake.TargetKind, &wake.ReviewMode, &wake.ReviewID,
|
|
&wake.ReviewStatus, &wake.ReviewVerdict, &wake.ReviewSummary,
|
|
&wake.ReviewFailureReason, &wake.Sequence, &wake.ClientSubmitID,
|
|
&wake.SourceSessionID, &wake.TargetSessionID,
|
|
&wake.CanonicalSessionID, &wake.CanonicalTurnID,
|
|
&wake.Status, &dueAt, &wake.AttemptCount, &wake.LeaseOwner,
|
|
&leaseExpiresAt, &dispatchedAt, &turnSettledAt, &acknowledgedAt,
|
|
&wake.LastError, &createdAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return executionbiz.Wake{}, err
|
|
}
|
|
wake.DueAt = time.UnixMilli(dueAt).UTC()
|
|
wake.LeaseExpiresAt = optionalTuttiModeExecutionTime(leaseExpiresAt)
|
|
wake.DispatchedAt = optionalTuttiModeExecutionTime(dispatchedAt)
|
|
wake.TurnSettledAt = optionalTuttiModeExecutionTime(turnSettledAt)
|
|
wake.AcknowledgedAt = optionalTuttiModeExecutionTime(acknowledgedAt)
|
|
wake.CreatedAt = time.UnixMilli(createdAt).UTC()
|
|
wake.UpdatedAt = time.UnixMilli(updatedAt).UTC()
|
|
return wake, nil
|
|
}
|