tutti/services/tuttid/service/workspace/apps_runner_test.go

package workspace

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	workspacebiz "github.com/tutti-os/tutti/services/tuttid/biz/workspace"
)

func TestAppRunnerStartsHealthyAppWithWorkspaceScopedCwdAndInjectedDirs(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner happy path test")
	}

	root := t.TempDir()
	stateRoot := filepath.Join(root, "state")
	packageDir := filepath.Join(root, "package")
	runtimeDir := filepath.Join(root, "runtime")
	dataDir := filepath.Join(root, "data")
	databaseDir := filepath.Join(root, "database")
	logDir := filepath.Join(root, "logs")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
echo runner-started
exec "$TUTTI_APP_PYTHON" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(pythonAppReadyServerScript("/ready", true)), 0o644); err != nil {
		t.Fatalf("WriteFile(server.py) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	t.Setenv("TUTTI_ENV", "production")
	t.Setenv("TUTTI_STATE_DIR", stateRoot)
	t.Setenv("TUTTI_WORKSPACE_ROOT", "/inherited/workspace")
	t.Setenv(removedWorkspaceRootCompatibilityEnvKey, "/inherited/workspace")
	runner := &AppRunner{HealthcheckTimeout: 10 * time.Second}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-runner",
		WorkspaceName:   "Runner Workspace",
		AppID:           "hello",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      runtimeDir,
		DataDir:         dataDir,
		DatabaseDir:     databaseDir,
		LogDir:          logDir,
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "hello")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing, lastError=%v", state.Status, state.LastError)
	}
	state = waitForRunnerStatus(t, runner, "ws-runner", "hello", workspacebiz.AppRuntimeStatusRunning)
	if state.LaunchURL == nil || !strings.HasPrefix(*state.LaunchURL, "http://127.0.0.1:") {
		t.Fatalf("LaunchURL = %v", state.LaunchURL)
	}
	if state.Port == nil || *state.Port <= 0 {
		t.Fatalf("Port = %v", state.Port)
	}
	if info, err := os.Stat(databaseDir); err != nil || !info.IsDir() {
		t.Fatalf("database directory stat = (%v, %v), want directory", info, err)
	}

	probePath := filepath.Join(dataDir, "probe.json")
	probe, err := os.ReadFile(probePath)
	if err != nil {
		t.Fatalf("ReadFile(%s) error = %v", probePath, err)
	}
	var probeValues map[string]string
	if err := json.Unmarshal(probe, &probeValues); err != nil {
		t.Fatalf("Unmarshal(probe) error = %v", err)
	}
	if samePath(t, probeValues["cwd"], runtimeDir) == false {
		t.Fatalf("probe cwd = %q, want %q", probeValues["cwd"], runtimeDir)
	}
	for key, want := range map[string]string{
		"packageDir":    packageDir,
		"runtimeDir":    runtimeDir,
		"dataDir":       dataDir,
		"databaseDir":   databaseDir,
		"logDir":        logDir,
		"toolchainRoot": filepath.Join(stateRoot, "app-toolchains"),
	} {
		if probeValues[key] != want {
			t.Fatalf("probe[%s] = %q, want %q", key, probeValues[key], want)
		}
	}
	for _, key := range []string{"tuttiWorkspaceRoot", "legacyWorkspaceRoot"} {
		if probeValues[key] != "" {
			t.Fatalf("probe[%s] = %q, want absent root contract", key, probeValues[key])
		}
	}
	for key, want := range map[string]string{
		"appId":         "hello",
		"workspaceId":   "ws-runner",
		"workspaceName": "Runner Workspace",
		"appHost":       "127.0.0.1",
		"appBaseUrl":    *state.LaunchURL,
		"platform":      runtime.GOOS + "-" + runtime.GOARCH,
	} {
		if probeValues[key] != want {
			t.Fatalf("probe[%s] = %q, want %q", key, probeValues[key], want)
		}
	}
	wantCLIPath := filepath.Join(stateRoot, "bin", "tutti")
	if probeValues["tuttiCli"] != wantCLIPath {
		t.Fatalf("probe[tuttiCli] = %q, want %q", probeValues["tuttiCli"], wantCLIPath)
	}
	pathDirs := filepath.SplitList(probeValues["path"])
	if len(pathDirs) == 0 || pathDirs[0] != filepath.Dir(wantCLIPath) {
		t.Fatalf("probe[path] = %q, want tutti CLI shim dir first", probeValues["path"])
	}

	logData, err := os.ReadFile(filepath.Join(logDir, "runtime.log"))
	if err != nil {
		t.Fatalf("ReadFile(runtime.log) error = %v", err)
	}
	if !strings.Contains(string(logData), "runner-started") {
		t.Fatalf("runtime.log = %q, want runner output", string(logData))
	}
	if !strings.Contains(string(logData), "tutti workspace app startup") || strings.Contains(string(logData), "workspaceRoot=") {
		t.Fatalf("runtime.log = %q, want startup diagnostic", string(logData))
	}
	if !strings.Contains(string(logData), "python=") || !strings.Contains(string(logData), "node=") {
		t.Fatalf("runtime.log = %q, want managed runtime diagnostic", string(logData))
	}

	stopped, err := runner.Stop(context.Background(), "ws-runner", "hello")
	if err != nil {
		t.Fatalf("Stop() error = %v", err)
	}
	if stopped.Status != workspacebiz.AppRuntimeStatusIdle {
		t.Fatalf("Stop() status = %q, want idle", stopped.Status)
	}
}

func TestAppRunnerStartsStandaloneAppWithoutResolvingManagedRuntime(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner standalone test")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	runtimeDir := filepath.Join(root, "runtime")
	dataDir := filepath.Join(root, "data")
	logDir := filepath.Join(root, "logs")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
exec python3 "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(pythonAppReadyServerScript("/healthz", false)), 0o644); err != nil {
		t.Fatalf("WriteFile(server.py) error = %v", err)
	}

	resolver := &appRuntimeResolverStub{called: make(chan struct{})}
	runner := &AppRunner{
		HealthcheckTimeout: 10 * time.Second,
		RuntimeResolver:    resolver,
	}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-runner",
		WorkspaceName:   "Runner Workspace",
		AppID:           "standalone",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/healthz",
		RuntimeProfile:  workspaceAppStandaloneRuntimeProfile,
		RuntimeDir:      runtimeDir,
		DataDir:         dataDir,
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          logDir,
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "standalone")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing, lastError=%v", state.Status, state.LastError)
	}
	state = waitForRunnerStatus(t, runner, "ws-runner", "standalone", workspacebiz.AppRuntimeStatusRunning)
	if state.LaunchURL == nil || !strings.HasPrefix(*state.LaunchURL, "http://127.0.0.1:") {
		t.Fatalf("LaunchURL = %v", state.LaunchURL)
	}

	select {
	case <-resolver.called:
		t.Fatal("standalone app resolved managed runtime")
	default:
	}
}

func TestAppRunnerRestartStartsFreshProcessAndWritesStartupDiagnostic(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner restart test")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	runtimeDir := filepath.Join(root, "runtime")
	dataDir := filepath.Join(root, "data")
	logDir := filepath.Join(root, "logs")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
echo runner-started
exec "$TUTTI_APP_PYTHON" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(pythonAppReadyServerScript("/ready", false)), 0o644); err != nil {
		t.Fatalf("WriteFile(server.py) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	runner := &AppRunner{HealthcheckTimeout: 10 * time.Second}
	input := AppStartInput{
		WorkspaceID:     "ws-runner",
		WorkspaceName:   "Runner Workspace",
		AppID:           "hello",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      runtimeDir,
		DataDir:         dataDir,
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          logDir,
	}
	if _, err := runner.Start(context.Background(), input); err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "hello")
	})
	first := waitForRunnerStatus(t, runner, "ws-runner", "hello", workspacebiz.AppRuntimeStatusRunning)
	if first.Port == nil {
		t.Fatalf("first Port = nil")
	}

	state, err := runner.Start(context.Background(), input)
	if err != nil {
		t.Fatalf("Start(no restart) error = %v", err)
	}
	if state.Status != workspacebiz.AppRuntimeStatusRunning {
		t.Fatalf("Start(no restart) status = %q, want running", state.Status)
	}
	if state.Port == nil || *state.Port != *first.Port {
		t.Fatalf("Start(no restart) port = %v, want %d", state.Port, *first.Port)
	}

	input.Restart = true
	state, err = runner.Start(context.Background(), input)
	if err != nil {
		t.Fatalf("Start(Restart) error = %v", err)
	}
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start(Restart) status = %q, want preparing", state.Status)
	}
	second := waitForRunnerStatus(t, runner, "ws-runner", "hello", workspacebiz.AppRuntimeStatusRunning)
	if second.Port == nil {
		t.Fatalf("second Port = nil")
	}

	logData, err := os.ReadFile(filepath.Join(logDir, "runtime.log"))
	if err != nil {
		t.Fatalf("ReadFile(runtime.log) error = %v", err)
	}
	if got := strings.Count(string(logData), "tutti workspace app startup"); got != 2 {
		t.Fatalf("startup diagnostics = %d, want 2; runtime.log=%q", got, string(logData))
	}
}

func TestAppRunnerStopProcessDoesNotOverwriteReplacementRuntime(t *testing.T) {
	runner := &AppRunner{}
	runner.ensure()
	key := appRuntimeKey("ws-runner", "hello")
	oldURL := "http://127.0.0.1:41001"
	newURL := "http://127.0.0.1:41002"
	oldPort := 41001
	newPort := 41002
	oldProcess := &appProcess{done: make(chan error, 1)}
	newProcess := &appProcess{done: make(chan error, 1)}

	runner.mu.Lock()
	runner.processes[key] = oldProcess
	runner.states[key] = workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: &oldURL,
		Port:      &oldPort,
	}
	runner.mu.Unlock()

	type stopResult struct {
		state workspacebiz.AppRuntimeState
		err   error
	}
	stopped := make(chan stopResult, 1)
	go func() {
		state, err := runner.stopProcess(context.Background(), key, oldProcess)
		stopped <- stopResult{state: state, err: err}
	}()

	waitForRunnerStatus(t, runner, "ws-runner", "hello", workspacebiz.AppRuntimeStatusStopping)

	runner.mu.Lock()
	runner.processes[key] = newProcess
	runner.states[key] = workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: &newURL,
		Port:      &newPort,
	}
	runner.mu.Unlock()

	oldProcess.done <- nil

	select {
	case result := <-stopped:
		if result.err != nil {
			t.Fatalf("stopProcess() error = %v", result.err)
		}
		if result.state.Status != workspacebiz.AppRuntimeStatusRunning {
			t.Fatalf("stopProcess() status = %q, want running", result.state.Status)
		}
		if result.state.LaunchURL == nil || *result.state.LaunchURL != newURL {
			t.Fatalf("stopProcess() launchURL = %v, want %q", result.state.LaunchURL, newURL)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("timed out waiting for stopProcess")
	}

	state := runner.State("ws-runner", "hello")
	if state.Status != workspacebiz.AppRuntimeStatusRunning {
		t.Fatalf("runner state = %q, want running", state.Status)
	}
	if state.LaunchURL == nil || *state.LaunchURL != newURL {
		t.Fatalf("runner launchURL = %v, want %q", state.LaunchURL, newURL)
	}
}

func TestAppRunnerStopProcessWaitsForProcessDoneWhenContextIsCanceled(t *testing.T) {
	runner := &AppRunner{}
	runner.ensure()
	key := appRuntimeKey("ws-runner", "hello")
	process := &appProcess{done: make(chan error)}
	runner.mu.Lock()
	runner.processes[key] = process
	runner.states[key] = workspacebiz.AppRuntimeState{
		Status: workspacebiz.AppRuntimeStatusRunning,
	}
	runner.mu.Unlock()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	type stopResult struct {
		state workspacebiz.AppRuntimeState
		err   error
	}
	stopped := make(chan stopResult, 1)
	go func() {
		state, err := runner.stopProcess(ctx, key, process)
		stopped <- stopResult{state: state, err: err}
	}()

	select {
	case result := <-stopped:
		t.Fatalf("stopProcess() returned before process done: %#v", result)
	case <-time.After(50 * time.Millisecond):
	}
	process.done <- nil
	select {
	case result := <-stopped:
		if !errors.Is(result.err, context.Canceled) {
			t.Fatalf("stopProcess() error = %v, want context.Canceled", result.err)
		}
		if result.state.Status != workspacebiz.AppRuntimeStatusFailed {
			t.Fatalf("stopProcess() status = %q, want failed", result.state.Status)
		}
	case <-time.After(time.Second):
		t.Fatal("timed out waiting for stopProcess")
	}
}

func TestAppRunnerStopAllClearsStateOnlyRuntime(t *testing.T) {
	runner := &AppRunner{}
	runner.setState(appRuntimeKey("ws-runner", "orphaned"), workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: stringPtr("http://127.0.0.1:43210"),
		Port:      intPtr(43210),
	})

	runner.StopAll(context.Background())

	assertRunnerStatus(t, runner, "ws-runner", "orphaned", workspacebiz.AppRuntimeStatusIdle)
}

func TestAppRunnerStopWorkspaceClearsMatchingStateOnlyRuntime(t *testing.T) {
	runner := &AppRunner{}
	runner.setState(appRuntimeKey("ws-runner", "orphaned"), workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: stringPtr("http://127.0.0.1:43210"),
		Port:      intPtr(43210),
	})
	runner.setState(appRuntimeKey("ws-other", "orphaned"), workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: stringPtr("http://127.0.0.1:43211"),
		Port:      intPtr(43211),
	})

	runner.StopWorkspace(context.Background(), "ws-runner")

	assertRunnerStatus(t, runner, "ws-runner", "orphaned", workspacebiz.AppRuntimeStatusIdle)
	assertRunnerStatus(t, runner, "ws-other", "orphaned", workspacebiz.AppRuntimeStatusRunning)
}

func TestAppRunnerStopAppClearsMatchingStateOnlyRuntime(t *testing.T) {
	runner := &AppRunner{}
	runner.setState(appRuntimeKey("ws-runner", "orphaned"), workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: stringPtr("http://127.0.0.1:43210"),
		Port:      intPtr(43210),
	})
	runner.setState(appRuntimeKey("ws-runner", "other"), workspacebiz.AppRuntimeState{
		Status:    workspacebiz.AppRuntimeStatusRunning,
		LaunchURL: stringPtr("http://127.0.0.1:43211"),
		Port:      intPtr(43211),
	})

	runner.StopApp(context.Background(), "orphaned")

	assertRunnerStatus(t, runner, "ws-runner", "orphaned", workspacebiz.AppRuntimeStatusIdle)
	assertRunnerStatus(t, runner, "ws-runner", "other", workspacebiz.AppRuntimeStatusRunning)
}

func TestAppRunnerStartWithoutRestartReusesQueuedStart(t *testing.T) {
	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte("#!/bin/sh\nsleep 30\n"), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}

	var eventsMu sync.Mutex
	var events []workspacebiz.AppRuntimeState
	runner := &AppRunner{
		RuntimeResolver: &appRuntimeResolverStub{called: make(chan struct{}), err: errors.New("skip runtime")},
		OnStateChanged: func(_ string, _ string, state workspacebiz.AppRuntimeState) {
			eventsMu.Lock()
			events = append(events, state)
			eventsMu.Unlock()
		},
		queue: make(chan struct{}, 1),
	}
	runner.queue <- struct{}{}
	input := AppStartInput{
		WorkspaceID:     "ws-runner",
		AppID:           "queued",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      filepath.Join(root, "runtime"),
		DataDir:         filepath.Join(root, "data"),
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          filepath.Join(root, "logs"),
	}
	state, err := runner.Start(context.Background(), input)
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing", state.Status)
	}

	state, err = runner.Start(context.Background(), input)
	if err != nil {
		t.Fatalf("Start(no restart) error = %v", err)
	}
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start(no restart) status = %q, want preparing", state.Status)
	}
	eventsMu.Lock()
	eventCount := len(events)
	eventsMu.Unlock()
	if eventCount != 1 {
		t.Fatalf("state change events = %d, want 1", eventCount)
	}

	<-runner.queue
	waitForRunnerStatus(t, runner, "ws-runner", "queued", workspacebiz.AppRuntimeStatusFailed)
}

func TestAppRunnerFinishStartIgnoresReplacedStart(t *testing.T) {
	runner := &AppRunner{}
	runner.ensure()
	key := appRuntimeKey("ws-runner", "queued")
	oldStart := &appStart{cancel: func() {}}
	newStart := &appStart{cancel: func() {}}
	cancelledCtx, cancel := context.WithCancel(context.Background())
	cancel()

	runner.mu.Lock()
	runner.starts[key] = newStart
	runner.states[key] = workspacebiz.AppRuntimeState{Status: workspacebiz.AppRuntimeStatusPreparing}
	runner.mu.Unlock()

	runner.finishStart(key, cancelledCtx, oldStart)

	runner.mu.Lock()
	defer runner.mu.Unlock()
	if runner.starts[key] != newStart {
		t.Fatalf("finishStart() replaced start = %v, want still active", runner.starts[key])
	}
	if state := runner.states[key]; state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("finishStart() status = %q, want preparing", state.Status)
	}
}

type barrierAppRuntimeResolver struct {
	started  chan struct{}
	release  chan error
	finished chan struct{}
}

func (r *barrierAppRuntimeResolver) Resolve(context.Context) (ResolvedAppRuntime, error) {
	close(r.started)
	err := <-r.release
	close(r.finished)
	return ResolvedAppRuntime{}, err
}

func TestAppRunnerStoppedStartCannotPublishLateStartupFailure(t *testing.T) {
	root := t.TempDir()
	resolver := &barrierAppRuntimeResolver{
		started: make(chan struct{}), release: make(chan error, 1), finished: make(chan struct{}),
	}
	runner := &AppRunner{RuntimeResolver: resolver}
	input := AppStartInput{
		WorkspaceID: "ws-runner", AppID: "stale-start", PackageDir: filepath.Join(root, "package"),
		RuntimeDir: filepath.Join(root, "runtime"), DataDir: filepath.Join(root, "data"),
		DatabaseDir: filepath.Join(root, "database"), LogDir: filepath.Join(root, "logs"),
	}
	if _, err := runner.Start(context.Background(), input); err != nil {
		t.Fatal(err)
	}
	select {
	case <-resolver.started:
	case <-time.After(time.Second):
		t.Fatal("runtime resolver did not start")
	}
	stopped, err := runner.Stop(context.Background(), input.WorkspaceID, input.AppID)
	if err != nil {
		t.Fatal(err)
	}
	if stopped.Status != workspacebiz.AppRuntimeStatusIdle {
		t.Fatalf("Stop() status = %q, want idle", stopped.Status)
	}
	resolver.release <- errors.New("late runtime resolution failure")
	select {
	case <-resolver.finished:
	case <-time.After(time.Second):
		t.Fatal("runtime resolver did not finish")
	}
	time.Sleep(20 * time.Millisecond)
	state := runner.State(input.WorkspaceID, input.AppID)
	if state.Status != workspacebiz.AppRuntimeStatusIdle || state.FailureReason != nil || state.LastError != nil {
		t.Fatalf("late startup failure replaced stopped state: %#v", state)
	}
}

func TestAppRunnerReplacedStartCannotCommitState(t *testing.T) {
	runner := &AppRunner{}
	runner.ensure()
	key := appRuntimeKey("ws-runner", "restarted")
	oldStart := &appStart{cancel: func() {}}
	newStart := &appStart{cancel: func() {}}
	runner.mu.Lock()
	runner.starts[key] = newStart
	runner.states[key] = workspacebiz.AppRuntimeState{Status: workspacebiz.AppRuntimeStatusPreparing}
	runner.mu.Unlock()

	if _, committed := runner.setStateForStart(key, oldStart, workspacebiz.AppRuntimeState{Status: workspacebiz.AppRuntimeStatusStarting}); committed {
		t.Fatal("replaced start committed starting state")
	}
	if _, committed := runner.setFailedForStart(key, oldStart, "healthcheck", errors.New("late health failure")); committed {
		t.Fatal("replaced start committed failure state")
	}
	state := runner.State("ws-runner", "restarted")
	if state.Status != workspacebiz.AppRuntimeStatusPreparing || state.FailureReason != nil {
		t.Fatalf("replacement state was overwritten: %#v", state)
	}
}

func TestAppRunnerStartWithoutRestartReusesStartingProcess(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte("#!/bin/sh\nsleep 30\n"), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	runner := &AppRunner{HealthcheckTimeout: 3 * time.Second}
	input := AppStartInput{
		WorkspaceID:     "ws-runner",
		AppID:           "starting",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      filepath.Join(root, "runtime"),
		DataDir:         filepath.Join(root, "data"),
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          filepath.Join(root, "logs"),
	}
	if _, err := runner.Start(context.Background(), input); err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "starting")
	})
	starting := waitForRunnerStatus(t, runner, "ws-runner", "starting", workspacebiz.AppRuntimeStatusStarting)
	if starting.Port == nil {
		t.Fatalf("starting Port = nil")
	}

	state, err := runner.Start(context.Background(), input)
	if err != nil {
		t.Fatalf("Start(no restart) error = %v", err)
	}
	if state.Status != workspacebiz.AppRuntimeStatusStarting {
		t.Fatalf("Start(no restart) status = %q, want starting", state.Status)
	}
	if state.Port == nil || *state.Port != *starting.Port {
		t.Fatalf("Start(no restart) port = %v, want %d", state.Port, *starting.Port)
	}
}

func TestAppRunnerStartsAppWithManagedNodeRuntimeEnv(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	runtimeDir := filepath.Join(root, "runtime")
	dataDir := filepath.Join(root, "data")
	logDir := filepath.Join(root, "logs")
	for _, dir := range []string{packageDir} {
		if err := os.MkdirAll(dir, 0o755); err != nil {
			t.Fatalf("MkdirAll(%s) error = %v", dir, err)
		}
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
exec "$TUTTI_APP_NODE" "$TUTTI_APP_PACKAGE_DIR/server.js"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(pythonAppReadyServerScript("/healthz", false)), 0o644); err != nil {
		t.Fatalf("WriteFile(server.py) error = %v", err)
	}

	runtimeRoot := createManagedAppRuntimeFixture(t, root)
	t.Setenv(tuttiAppRuntimeRootEnv, runtimeRoot)
	runner := &AppRunner{HealthcheckTimeout: 10 * time.Second}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-fnm",
		WorkspaceName:   "Fnm Workspace",
		AppID:           "fnm-node",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/healthz",
		RuntimeDir:      runtimeDir,
		DataDir:         dataDir,
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          logDir,
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-fnm", "fnm-node")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing, lastError=%v", state.Status, state.LastError)
	}
	waitForRunnerStatus(t, runner, "ws-fnm", "fnm-node", workspacebiz.AppRuntimeStatusRunning)
	logData, err := os.ReadFile(filepath.Join(logDir, "runtime.log"))
	if err != nil {
		t.Fatalf("ReadFile(runtime.log) error = %v", err)
	}
	if !strings.Contains(string(logData), filepath.Join(runtimeRoot, "node", "bin")) {
		t.Fatalf("runtime log PATH does not include managed node bin: %s", string(logData))
	}
}

func TestAppRunnerHealthcheckFailureIsBackgroundState(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
sleep 30
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	runner := &AppRunner{HealthcheckTimeout: 100 * time.Millisecond}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-runner",
		AppID:           "slow",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      filepath.Join(root, "runtime"),
		DataDir:         filepath.Join(root, "data"),
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          filepath.Join(root, "logs"),
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "slow")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing", state.Status)
	}

	state = waitForRunnerStatus(t, runner, "ws-runner", "slow", workspacebiz.AppRuntimeStatusFailed)
	if state.FailureReason == nil || *state.FailureReason != "healthcheck" {
		t.Fatalf("FailureReason = %v, want healthcheck", state.FailureReason)
	}
	if state.FailurePhase == nil || *state.FailurePhase != workspacebiz.AppFailurePhaseStarting {
		t.Fatalf("FailurePhase = %v, want starting", state.FailurePhase)
	}
}

func TestAppRuntimeLogHasPortBindFailureOnlyReadsCurrentAttempt(t *testing.T) {
	logPath := filepath.Join(t.TempDir(), "runtime.log")
	initial := "listen EACCES: permission denied 127.0.0.1:58239\n"
	if err := os.WriteFile(logPath, []byte(initial), 0o644); err != nil {
		t.Fatal(err)
	}
	info, err := os.Stat(logPath)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(logPath, append([]byte(initial), []byte("application failed before listening\n")...), 0o644); err != nil {
		t.Fatal(err)
	}
	if appRuntimeLogHasPortBindFailure(logPath, info.Size()) {
		t.Fatal("stale bind error from a previous attempt was treated as current")
	}
	if err := os.WriteFile(logPath, append([]byte(initial), []byte("listen EADDRINUSE: address already in use\n")...), 0o644); err != nil {
		t.Fatal(err)
	}
	if !appRuntimeLogHasPortBindFailure(logPath, info.Size()) {
		t.Fatal("current bind error was not detected")
	}
	if appRuntimeLogHasPortBindFailure(logPath, info.Size()+100) {
		t.Fatal("missing current log range was treated as a bind failure")
	}
}

func TestAppRunnerRetriesFreshPortAfterBindFailure(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner retry test")
	}

	root := t.TempDir()
	stateRoot := filepath.Join(root, "state")
	packageDir := filepath.Join(root, "package")
	runtimeDir := filepath.Join(root, "runtime")
	dataDir := filepath.Join(root, "data")
	databaseDir := filepath.Join(root, "database")
	logDir := filepath.Join(root, "logs")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
exec "$TUTTI_APP_PYTHON" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(`import http.server
import os
from pathlib import Path

attempt_path = Path(os.environ["TUTTI_APP_DATA_DIR"]) / "attempts"
attempt = int(attempt_path.read_text()) if attempt_path.exists() else 0
attempt += 1
attempt_path.write_text(str(attempt))
if attempt == 1:
    print("listen EADDRINUSE: address already in use", flush=True)
    raise SystemExit(1)

class Handler(http.server.BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == "/ready":
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b"ok")
        else:
            self.send_response(404)
            self.end_headers()
    def log_message(self, format, *args):
        pass

server = http.server.ThreadingHTTPServer(("127.0.0.1", int(os.environ["TUTTI_APP_PORT"])), Handler)
server.serve_forever()
`), 0o644); err != nil {
		t.Fatalf("WriteFile(server.py) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	t.Setenv("TUTTI_ENV", "production")
	t.Setenv("TUTTI_STATE_DIR", stateRoot)
	runner := &AppRunner{HealthcheckTimeout: 500 * time.Millisecond}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-runner-retry",
		WorkspaceName:   "Runner Retry Workspace",
		AppID:           "bind-race",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      runtimeDir,
		DataDir:         dataDir,
		DatabaseDir:     databaseDir,
		LogDir:          logDir,
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner-retry", "bind-race")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing", state.Status)
	}
	state = waitForRunnerStatus(t, runner, "ws-runner-retry", "bind-race", workspacebiz.AppRuntimeStatusRunning)
	if state.Port == nil || *state.Port <= 0 {
		t.Fatalf("Port = %v, want a running retry process", state.Port)
	}
	attempts, err := os.ReadFile(filepath.Join(dataDir, "attempts"))
	if err != nil {
		t.Fatalf("ReadFile(attempts) error = %v", err)
	}
	if strings.TrimSpace(string(attempts)) != "2" {
		t.Fatalf("attempt count = %q, want first bind failure plus one fresh-port retry", attempts)
	}
	logBody, err := os.ReadFile(filepath.Join(logDir, "runtime.log"))
	if err != nil {
		t.Fatalf("ReadFile(runtime.log) error = %v", err)
	}
	if !strings.Contains(string(logBody), "listen EADDRINUSE") {
		t.Fatalf("runtime log = %q, want the bind failure evidence", logBody)
	}
}

func TestAppRunnerDoesNotReturnToRunningAfterStopStarts(t *testing.T) {
	key := appRuntimeKey("ws-runner", "stopping")
	start := &appStart{cancel: func() {}}
	process := &appProcess{stopRequested: true}
	runner := &AppRunner{
		processes: map[string]*appProcess{key: process},
		starts:    map[string]*appStart{key: start},
		states: map[string]workspacebiz.AppRuntimeState{
			key: {Status: workspacebiz.AppRuntimeStatusStopping},
		},
	}

	if runner.setRunningIfProcessCurrent(key, start, process, workspacebiz.AppRuntimeState{Status: workspacebiz.AppRuntimeStatusRunning}) {
		t.Fatal("stopping process was moved back to running")
	}
	if state := runner.State("ws-runner", "stopping"); state.Status != workspacebiz.AppRuntimeStatusStopping {
		t.Fatalf("runner status = %q, want stopping", state.Status)
	}
}

func TestAppRunnerMarksExitAfterRunningAsRuntimeFailure(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner process exit test")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
exec "$TUTTI_APP_PYTHON" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(`import os
import socket
import time

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", int(os.environ["TUTTI_APP_PORT"])))
server.listen(1)
connection, _ = server.accept()
with connection:
    connection.recv(4096)
    connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
time.sleep(0.2)
raise SystemExit(17)
`), 0o644); err != nil {
		t.Fatal(err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	runner := &AppRunner{HealthcheckTimeout: 5 * time.Second}
	_, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID: "ws-runner", AppID: "exits", PackageDir: packageDir,
		Bootstrap: "bootstrap.sh", HealthcheckPath: "/ready",
		RuntimeDir: filepath.Join(root, "runtime"), DataDir: filepath.Join(root, "data"),
		DatabaseDir: filepath.Join(root, "database"), LogDir: filepath.Join(root, "logs"),
	})
	if err != nil {
		t.Fatal(err)
	}
	waitForRunnerStatus(t, runner, "ws-runner", "exits", workspacebiz.AppRuntimeStatusRunning)
	state := waitForRunnerStatus(t, runner, "ws-runner", "exits", workspacebiz.AppRuntimeStatusFailed)
	if state.FailurePhase == nil || *state.FailurePhase != workspacebiz.AppFailurePhaseRuntime {
		t.Fatalf("FailurePhase = %v, want runtime", state.FailurePhase)
	}
	if state.FailureReason == nil || *state.FailureReason != "process_exit" {
		t.Fatalf("FailureReason = %v, want process_exit", state.FailureReason)
	}
}

func TestAppRunnerMarksCleanExitAfterRunningAsRuntimeFailure(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}
	if _, err := exec.LookPath("python3"); err != nil {
		t.Skip("python3 is required for runner process exit test")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
exec "$TUTTI_APP_PYTHON" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "server.py"), []byte(`import os
import socket
import time

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", int(os.environ["TUTTI_APP_PORT"])))
server.listen(1)
connection, _ = server.accept()
with connection:
    connection.recv(4096)
    connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
time.sleep(0.2)
raise SystemExit(0)
`), 0o644); err != nil {
		t.Fatal(err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	runner := &AppRunner{HealthcheckTimeout: 5 * time.Second}
	_, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID: "ws-runner", AppID: "clean-exit", PackageDir: packageDir,
		Bootstrap: "bootstrap.sh", HealthcheckPath: "/ready",
		RuntimeDir: filepath.Join(root, "runtime"), DataDir: filepath.Join(root, "data"),
		DatabaseDir: filepath.Join(root, "database"), LogDir: filepath.Join(root, "logs"),
	})
	if err != nil {
		t.Fatal(err)
	}
	waitForRunnerStatus(t, runner, "ws-runner", "clean-exit", workspacebiz.AppRuntimeStatusRunning)
	state := waitForRunnerStatus(t, runner, "ws-runner", "clean-exit", workspacebiz.AppRuntimeStatusFailed)
	if state.FailurePhase == nil || *state.FailurePhase != workspacebiz.AppFailurePhaseRuntime {
		t.Fatalf("FailurePhase = %v, want runtime", state.FailurePhase)
	}
	if state.LastError == nil || !strings.Contains(*state.LastError, "status 0") {
		t.Fatalf("LastError = %v, want unexpected clean exit", state.LastError)
	}
}

func TestAppRunnerStopDuringHealthcheckLeavesAppIdle(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("bootstrap.sh runner test is POSIX-only")
	}

	root := t.TempDir()
	packageDir := filepath.Join(root, "package")
	if err := os.MkdirAll(packageDir, 0o755); err != nil {
		t.Fatalf("MkdirAll(packageDir) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(packageDir, "bootstrap.sh"), []byte(`#!/bin/sh
set -eu
sleep 30
`), 0o755); err != nil {
		t.Fatalf("WriteFile(bootstrap.sh) error = %v", err)
	}

	t.Setenv(tuttiAppRuntimeRootEnv, createManagedAppRuntimeFixture(t, root))
	healthcheckStarted := make(chan struct{})
	var healthcheckStartedOnce sync.Once
	var eventsMu sync.Mutex
	var events []workspacebiz.AppRuntimeState
	runner := &AppRunner{
		HealthcheckTimeout: 3 * time.Second,
		HTTPClient: &http.Client{Transport: roundTripperFunc(func(request *http.Request) (*http.Response, error) {
			healthcheckStartedOnce.Do(func() {
				close(healthcheckStarted)
			})
			<-request.Context().Done()
			return nil, request.Context().Err()
		})},
		OnStateChanged: func(_ string, _ string, state workspacebiz.AppRuntimeState) {
			eventsMu.Lock()
			events = append(events, state)
			eventsMu.Unlock()
		},
	}
	state, err := runner.Start(context.Background(), AppStartInput{
		WorkspaceID:     "ws-runner",
		AppID:           "slow",
		PackageDir:      packageDir,
		Bootstrap:       "bootstrap.sh",
		HealthcheckPath: "/ready",
		RuntimeDir:      filepath.Join(root, "runtime"),
		DataDir:         filepath.Join(root, "data"),
		DatabaseDir:     filepath.Join(root, "database"),
		LogDir:          filepath.Join(root, "logs"),
	})
	if err != nil {
		t.Fatalf("Start() error = %v", err)
	}
	t.Cleanup(func() {
		_, _ = runner.Stop(context.Background(), "ws-runner", "slow")
	})
	if state.Status != workspacebiz.AppRuntimeStatusPreparing {
		t.Fatalf("Start() status = %q, want preparing", state.Status)
	}

	select {
	case <-healthcheckStarted:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for healthcheck request")
	}
	stopped, err := runner.Stop(context.Background(), "ws-runner", "slow")
	if err != nil {
		t.Fatalf("Stop() error = %v", err)
	}
	if stopped.Status != workspacebiz.AppRuntimeStatusIdle {
		t.Fatalf("Stop() status = %q, want idle", stopped.Status)
	}

	state = waitForRunnerStatus(t, runner, "ws-runner", "slow", workspacebiz.AppRuntimeStatusIdle)
	if state.FailureReason != nil || state.LastError != nil {
		t.Fatalf("runner state after stop = %#v, want idle without failure", state)
	}

	eventsMu.Lock()
	defer eventsMu.Unlock()
	for _, event := range events {
		if event.Status != workspacebiz.AppRuntimeStatusFailed {
			continue
		}
		reason := ""
		if event.FailureReason != nil {
			reason = *event.FailureReason
		}
		lastError := ""
		if event.LastError != nil {
			lastError = *event.LastError
		}
		if reason == "healthcheck" && strings.Contains(lastError, context.Canceled.Error()) {
			t.Fatalf("recorded canceled healthcheck failure: %#v", event)
		}
	}
}

func TestTuttiCLIShimPathUsesProductionCommand(t *testing.T) {
	stateDir := t.TempDir()
	t.Setenv("TUTTI_STATE_DIR", stateDir)
	t.Setenv("TUTTI_ENV", "production")

	want := filepath.Join(stateDir, "bin", "tutti")
	if got := tuttiCLIShimPathForPlatform("darwin"); got != want {
		t.Fatalf("tuttiCLIShimPathForPlatform() = %q, want %q", got, want)
	}
}

func TestTuttiCLIShimPathUsesDevelopmentCommand(t *testing.T) {
	stateDir := t.TempDir()
	t.Setenv("TUTTI_STATE_DIR", stateDir)
	t.Setenv("TUTTI_ENV", "development")

	want := filepath.Join(stateDir, "bin", "tutti-dev")
	if got := tuttiCLIShimPathForPlatform("darwin"); got != want {
		t.Fatalf("tuttiCLIShimPathForPlatform() = %q, want %q", got, want)
	}
}

func TestWorkspaceAppCLIPathUsesNativeWindowsExecutable(t *testing.T) {
	target := filepath.Join(t.TempDir(), "Tutti CLI.exe")
	if err := os.WriteFile(target, []byte("fixture"), 0o755); err != nil {
		t.Fatalf("WriteFile() error = %v", err)
	}
	t.Setenv("TUTTI_WORKSPACE_APP_CLI_PATH", target)

	got, err := workspaceAppCLIPathForPlatform("windows")
	if err != nil {
		t.Fatalf("workspaceAppCLIPathForPlatform() error = %v", err)
	}
	if got != target {
		t.Fatalf("workspaceAppCLIPathForPlatform() = %q, want %q", got, target)
	}
}

func TestWorkspaceAppCLIPathRejectsWindowsBatchShim(t *testing.T) {
	target := filepath.Join(t.TempDir(), "tutti.cmd")
	if err := os.WriteFile(target, []byte("@echo off"), 0o755); err != nil {
		t.Fatalf("WriteFile() error = %v", err)
	}
	t.Setenv("TUTTI_WORKSPACE_APP_CLI_PATH", target)

	if _, err := workspaceAppCLIPathForPlatform("windows"); err == nil || !strings.Contains(err.Error(), ".exe") {
		t.Fatalf("workspaceAppCLIPathForPlatform() error = %v, want .exe validation", err)
	}
}

func TestWorkspaceAppCLIEnvOverridesIncludeWindowsListenerPath(t *testing.T) {
	listenerPath := filepath.Join(t.TempDir(), "run", "tuttid.listener.json")
	t.Setenv("TUTTID_LISTENER_INFO_PATH", listenerPath)

	overrides := workspaceAppCLIEnvOverrides("windows", `C:\Program Files\Tutti\tutti.exe`)
	if got := envValue(overrides, "TUTTI_CLI"); got != `C:\Program Files\Tutti\tutti.exe` {
		t.Fatalf("TUTTI_CLI = %q", got)
	}
	if got := envValue(overrides, "TUTTID_LISTENER_INFO_PATH"); got != listenerPath {
		t.Fatalf("TUTTID_LISTENER_INFO_PATH = %q, want %q", got, listenerPath)
	}
	if got := envValue(overrides, "TUTTI_STATE_DIR"); got != "" {
		t.Fatalf("TUTTI_STATE_DIR = %q, want omitted", got)
	}
}

type roundTripperFunc func(*http.Request) (*http.Response, error)

func (f roundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
	return f(request)
}

func samePath(t *testing.T, actual string, expected string) bool {
	t.Helper()

	actualResolved, err := filepath.EvalSymlinks(actual)
	if err != nil {
		actualResolved = actual
	}
	expectedResolved, err := filepath.EvalSymlinks(expected)
	if err != nil {
		expectedResolved = expected
	}
	return actualResolved == expectedResolved
}

func createManagedAppRuntimeFixture(t *testing.T, root string) string {
	t.Helper()

	pythonPath, err := exec.LookPath("python3")
	if err != nil {
		t.Skip("python3 is required for managed app runtime fixture")
	}

	runtimeRoot := filepath.Join(root, "managed-runtime")
	pythonBinDir := filepath.Join(runtimeRoot, "python", "bin")
	nodeBinDir := filepath.Join(runtimeRoot, "node", "bin")
	for _, dir := range []string{pythonBinDir, nodeBinDir} {
		if err := os.MkdirAll(dir, 0o755); err != nil {
			t.Fatalf("MkdirAll(%s) error = %v", dir, err)
		}
	}
	if err := os.WriteFile(filepath.Join(pythonBinDir, "python3"), []byte(`#!/bin/sh
exec "`+pythonPath+`" "$@"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(managed python) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(nodeBinDir, "node"), []byte(`#!/bin/sh
exec "`+pythonPath+`" "$TUTTI_APP_PACKAGE_DIR/server.py"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(managed node) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(nodeBinDir, "npm"), []byte(`#!/bin/sh
exit 0
`), 0o755); err != nil {
		t.Fatalf("WriteFile(managed npm) error = %v", err)
	}
	if err := os.WriteFile(filepath.Join(nodeBinDir, "corepack"), []byte(`#!/bin/sh
exec "$(dirname "$0")/node" "$(dirname "$0")/../lib/node_modules/corepack/dist/corepack.js" "$@"
`), 0o755); err != nil {
		t.Fatalf("WriteFile(managed corepack) error = %v", err)
	}
	return runtimeRoot
}

func pythonAppReadyServerScript(healthcheckPath string, writeProbe bool) string {
	probeImport := ""
	probeWrite := ""
	if writeProbe {
		probeImport = "import json\n"
		probeWrite = `        with open(os.path.join(os.environ["TUTTI_APP_DATA_DIR"], "probe.json"), "w") as f:
            json.dump({
                "cwd": os.getcwd(),
                "appId": os.environ["TUTTI_APP_ID"],
                "workspaceId": os.environ["TUTTI_WORKSPACE_ID"],
                "workspaceName": os.environ["TUTTI_WORKSPACE_NAME"],
                "tuttiWorkspaceRoot": os.environ.get("TUTTI_WORKSPACE_ROOT", ""),
                "legacyWorkspaceRoot": os.environ.get("NEX" + "TOP_WORKSPACE_ROOT", ""),
                "appHost": os.environ["TUTTI_APP_HOST"],
                "appBaseUrl": os.environ["TUTTI_APP_BASE_URL"],
                "platform": os.environ["TUTTI_PLATFORM"],
                "packageDir": os.environ["TUTTI_APP_PACKAGE_DIR"],
                "runtimeDir": os.environ["TUTTI_APP_RUNTIME_DIR"],
                "dataDir": os.environ["TUTTI_APP_DATA_DIR"],
                "databaseDir": os.environ["TUTTI_APP_DATABASE_DIR"],
                "logDir": os.environ["TUTTI_APP_LOG_DIR"],
                "toolchainRoot": os.environ["TUTTI_APP_TOOLCHAIN_ROOT"],
                "tuttiCli": os.environ["TUTTI_CLI"],
                "path": os.environ["PATH"],
            }, f)
`
	}

	script := `import os
__PROBE_IMPORT__import socket

HEALTHCHECK_PATH = "__HEALTHCHECK_PATH__"

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", int(os.environ["TUTTI_APP_PORT"])))
server.listen(16)

while True:
    connection, _ = server.accept()
    with connection:
        request = b""
        while b"\r\n\r\n" not in request:
            chunk = connection.recv(4096)
            if not chunk:
                break
            request += chunk
        request_line = request.split(b"\r\n", 1)[0].decode("ascii", "ignore")
        parts = request_line.split(" ")
        path = parts[1] if len(parts) > 1 else "/"
        if path != HEALTHCHECK_PATH:
            connection.sendall(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
            continue
__PROBE_WRITE__        connection.sendall(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
`
	script = strings.ReplaceAll(script, "__PROBE_IMPORT__", probeImport)
	script = strings.ReplaceAll(script, "__PROBE_WRITE__", probeWrite)
	script = strings.ReplaceAll(script, "__HEALTHCHECK_PATH__", healthcheckPath)
	return script
}

func waitForRunnerStatus(t *testing.T, runner *AppRunner, workspaceID string, appID string, want workspacebiz.AppRuntimeStatus) workspacebiz.AppRuntimeState {
	t.Helper()

	return waitForRunnerState(t, runner, workspaceID, appID, func(state workspacebiz.AppRuntimeState) bool {
		return state.Status == want
	})
}

func assertRunnerStatus(t *testing.T, runner *AppRunner, workspaceID string, appID string, want workspacebiz.AppRuntimeStatus) {
	t.Helper()

	state := runner.State(workspaceID, appID)
	if state.Status != want {
		t.Fatalf("State(%q, %q) status = %q, want %q", workspaceID, appID, state.Status, want)
	}
}

func waitForRunnerState(t *testing.T, runner *AppRunner, workspaceID string, appID string, matches func(workspacebiz.AppRuntimeState) bool) workspacebiz.AppRuntimeState {
	t.Helper()

	deadline := time.Now().Add(runnerStatusWaitTimeout(runner))
	var state workspacebiz.AppRuntimeState
	for time.Now().Before(deadline) {
		state = runner.State(workspaceID, appID)
		if matches(state) {
			return state
		}
		time.Sleep(25 * time.Millisecond)
	}
	t.Fatalf("runner state did not match before timeout: status=%q failureReason=%q lastError=%q launchURL=%v port=%v", state.Status, stringValue(state.FailureReason), stringValue(state.LastError), state.LaunchURL, state.Port)
	return state
}

func runnerStatusWaitTimeout(runner *AppRunner) time.Duration {
	timeout := 5 * time.Second
	if runner != nil && runner.HealthcheckTimeout > timeout {
		timeout = runner.HealthcheckTimeout + 2*time.Second
	}
	return timeout
}

func stringValue(value *string) string {
	if value == nil {
		return ""
	}
	return *value
}