import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { chmodSync, linkSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { connect, createServer, type Server } from 'node:net' import { DaemonServer } from './daemon-server' import type { ConnectedDaemonClient } from './daemon-client-connections' import { DaemonClient } from './client' import { isDaemonGoneError } from './daemon-endpoint-errors' import { DAEMON_ENDPOINT_LOST_MESSAGE } from './daemon-endpoint-ownership' import { encodeNdjson } from './ndjson' import { getDaemonSocketPath } from './daemon-spawner' import type { SubprocessHandle } from './session-subprocess-handle' import { PROTOCOL_VERSION } from './types' import { waitForEndpointUnreachable } from './daemon-endpoint-reachability-test-harness' // A killed process must actually report its exit: teardown waits // IMMEDIATE_KILL_PHYSICAL_EXIT_TIMEOUT_MS for one that never does. function createMockSubprocess(): SubprocessHandle { let notifyExit: ((code: number) => void) | null = null const exit = (): void => notifyExit?.(0) return { pid: 44444, getForegroundProcess: () => null, write() {}, resize() {}, kill: exit, terminateOwnedTree: () => 'unavailable' as const, forceKill: exit, signal() {}, onData() {}, onExit(callback) { notifyExit = callback }, dispose() {} } } type DaemonServerInternals = { endpoint: { hasLostOwnership(): boolean requestRetirementForLoss(): void checkOwnership(): void ownershipLossStreak: number ownershipLost: boolean } lifecycle: { retirementRequested: boolean state: string server: Server | null } connections: { clients: Map } } describe('daemon server error handling', () => { let dir: string let socketPath: string let tokenPath: string let server: DaemonServer let client: DaemonClient | null = null beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'daemon-server-errors-')) socketPath = getDaemonSocketPath(dir) tokenPath = join(dir, 'test.token') }) afterEach(async () => { client?.disconnect() client = null await server?.shutdown() rmSync(dir, { recursive: true, force: true }) }) it.skipIf(process.platform === 'win32')( 'refuses to create a session on an endpoint it no longer owns', async () => { // Why this matters: publishing cannot be made atomic against a publisher preempted between // proving an entry dead and replacing it, so this daemon can lose the endpoint at any // moment. If it accepted a session in that window the session would be reachable by // nobody — a terminal that acknowledges input and never runs it, which is the original // bug. Refusing makes that outcome unreachable rather than merely short-lived. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() // Sanity: a session is creatable while ownership holds. await expect( client.request('createOrAttach', { sessionId: 'before', cols: 80, rows: 24 }) ).resolves.toMatchObject({ isNew: true }) // Another daemon takes the canonical name, exactly as a late publisher's rename would. const usurper = createServer() const usurperBind = join(dir, '.u') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) await expect( client.request('createOrAttach', { sessionId: 'after', cols: 80, rows: 24 }) ).rejects.toThrow(/no longer owns its endpoint/) // And it stands down rather than lingering as an unreachable host. const daemon = server as unknown as DaemonServerInternals expect(daemon.lifecycle.retirementRequested).toBe(true) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( 'keeps serving sessions when endpoint ownership cannot be read', async () => { // Why: an unreadable stat proves nothing. Refusing sessions on it would take a daemon that // is serving every terminal on the machine offline because of a transient EACCES or EIO. // Only positive evidence of loss may refuse. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() const daemon = server as unknown as DaemonServerInternals try { // Drop search permission on the directory so the ownership stat fails EACCES. chmodSync(dir, 0o600) expect(daemon.endpoint.hasLostOwnership()).toBe(false) expect(daemon.lifecycle.retirementRequested).toBe(false) } finally { chmodSync(dir, 0o700) } } ) it.skipIf(process.platform === 'win32')( 'still attaches to a session it already hosts after losing the endpoint', async () => { // Why attach is exempt: it reaches a terminal already on this daemon, over a connection // that already exists, so it strands nothing. Refusing it would break the drain a retiring // daemon depends on to let live sessions finish. Note this must run without a prior // refusal, which would null the owned identity and disarm the guard anyway. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() await client.request('createOrAttach', { sessionId: 'live', cols: 80, rows: 24 }) const usurper = createServer() const usurperBind = join(dir, '.u2') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) // The guard is still armed here — no creation has been refused yet. const daemon = server as unknown as DaemonServerInternals expect(daemon.endpoint.hasLostOwnership()).toBe(true) await expect( client.request('createOrAttach', { sessionId: 'live', cols: 80, rows: 24, attachOnly: true }) ).resolves.toMatchObject({ isNew: false }) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it.skipIf(process.platform === 'win32')( 'never reopens session creation after the endpoint was lost', async () => { // Why: retiring nulls the owned identity, so an ownership check inferred from it answers // "not lost" forever after. A stream socket accepted before the takeover can then finish // its hello, clear the pending retirement, and reopen creation on a daemon nothing can // reach — reinstating the exact outcome the guard exists to prevent. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() // Keep one session alive: with nothing to drain the daemon now stands down the moment it // loses the endpoint, which is correct but would end the test before it can assert. await client.request('createOrAttach', { sessionId: 'draining', cols: 80, rows: 24 }) const usurper = createServer() const usurperBind = join(dir, '.u3') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) await expect( client.request('createOrAttach', { sessionId: 'a', cols: 80, rows: 24 }) ).rejects.toThrow(/no longer owns its endpoint/) // Simulate a late client completing its handshake and clearing pending retirement. const daemon = server as unknown as DaemonServerInternals daemon.lifecycle.retirementRequested = false // The loss must still be remembered, so creation stays closed. expect(daemon.endpoint.hasLostOwnership()).toBe(true) await expect( client.request('createOrAttach', { sessionId: 'b', cols: 80, rows: 24 }) ).rejects.toThrow(/no longer owns its endpoint/) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it('treats the endpoint-lost refusal as a reconnectable error', () => { // Why pinned: the server refuses so the client can reconnect to whoever owns the endpoint. // If the client's retry predicate does not recognise the refusal, it surfaces to the user // and the request dead-ends — barely better than the strand the refusal exists to avoid. expect(isDaemonGoneError(new Error(DAEMON_ENDPOINT_LOST_MESSAGE))).toBe(true) expect(isDaemonGoneError(new Error('something else entirely'))).toBe(false) }) it.skipIf(process.platform === 'win32')( 'stands down once drained even while a pre-takeover client stays connected', async () => { // Why connections stop counting after loss: retirement drains and then exits, but idleness // normally waits for every client to disconnect. A client that connected before the // takeover can hold that open indefinitely, so the daemon would outlive its last session // as an orphan nothing can route to. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() const usurper = createServer() const usurperBind = join(dir, '.u4') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) // Losing the endpoint with nothing left to drain must stand the daemon down, even // though this client is still connected. const daemon = server as unknown as DaemonServerInternals expect(daemon.lifecycle.state).toBe('running') daemon.endpoint.requestRetirementForLoss() // Drained and unreachable: it must begin standing down rather than wait for this // client, which can never make it routable again. expect(daemon.lifecycle.state).not.toBe('running') } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it.skipIf(process.platform === 'win32')( 'still answers the refusal when losing the endpoint also begins shutdown', async () => { // Why: the guard retires synchronously, and with nothing to drain that begins shutdown // before the reply is written. The client must still receive the refusal it can retry on; // a dropped connection here would surface as an opaque failure instead of a reconnect. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() const usurper = createServer() const usurperBind = join(dir, '.u5') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) // No sessions, so retiring drains immediately and shutdown starts inside this call. await expect( client.request('createOrAttach', { sessionId: 'x', cols: 80, rows: 24 }) ).rejects.toThrow(/no longer owns its endpoint/) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it.skipIf(process.platform === 'win32')( 'records endpoint loss even when already retiring for another reason', async () => { // Why: a daemon can already be retiring because its last authenticated client dropped // while a session runs. Losing the endpoint is a different fact, and gating one on the // other meant the loss went unrecorded — after which a later hello cleared the retirement // and put a daemon that demonstrably could not be reached back into ordinary service. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() await client.request('createOrAttach', { sessionId: 'keepalive', cols: 80, rows: 24 }) const usurper = createServer() const usurperBind = join(dir, '.u6') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { unlinkSync(socketPath) linkSync(usurperBind, socketPath) const daemon = server as unknown as DaemonServerInternals // Retirement is already pending for an unrelated reason. daemon.lifecycle.retirementRequested = true daemon.endpoint.checkOwnership() daemon.endpoint.checkOwnership() // The loss must be recorded regardless, so a later hello cannot undo it. expect(daemon.endpoint.ownershipLost).toBe(true) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it.skipIf(process.platform === 'win32')( 'does not retire on losses separated by a demonstrably owned observation', async () => { // Why: the watchdog retires on CONSECUTIVE losses, but an admission-time ownership read is // just as authoritative. When it did not reset the streak, two losses with a positive // owned reading between them counted as consecutive — permanently poisoning a healthy, // reachable daemon into refusing every later session. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() const daemon = server as unknown as DaemonServerInternals // A second name for our own socket inode, so ownership can be handed back. const ourAlias = join(dir, '.ours') linkSync(socketPath, ourAlias) const usurper = createServer() const usurperBind = join(dir, '.u7') await new Promise((resolve) => usurper.listen(usurperBind, resolve)) try { // First loss. unlinkSync(socketPath) linkSync(usurperBind, socketPath) daemon.endpoint.checkOwnership() expect(daemon.endpoint.ownershipLossStreak).toBe(1) expect(daemon.endpoint.ownershipLost).toBe(false) // Ownership demonstrably returns, observed through the admission path. unlinkSync(socketPath) linkSync(ourAlias, socketPath) expect(daemon.endpoint.hasLostOwnership()).toBe(false) expect(daemon.endpoint.ownershipLossStreak).toBe(0) // A later isolated loss is confirmation #1, not #2, so it must not retire. unlinkSync(socketPath) linkSync(usurperBind, socketPath) daemon.endpoint.checkOwnership() expect(daemon.endpoint.ownershipLost).toBe(false) } finally { await new Promise((resolve) => usurper.close(() => resolve())) } } ) it('keeps serving after an operational server error instead of dying', async () => { // Why: an unhandled 'error' on a net.Server is an uncaught exception. Detaching the startup // listener once start() settled left a daemon hosting every terminal on the machine one // failed accept away from termination. server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) await server.start() const daemon = server as unknown as DaemonServerInternals expect(daemon.lifecycle.server?.listenerCount('error')).toBe(1) const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) try { // Twice on purpose: a one-shot listener survives the first error and dies on the second, // so a single emit cannot tell a permanent handler from `once`. expect(() => daemon.lifecycle.server?.emit('error', new Error('EMFILE: accept failed')) ).not.toThrow() expect(() => daemon.lifecycle.server?.emit('error', new Error('EMFILE: accept failed again')) ).not.toThrow() expect(warn).toHaveBeenCalledTimes(2) expect(daemon.lifecycle.server?.listenerCount('error')).toBe(1) } finally { warn.mockRestore() } client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() expect(client.isConnected()).toBe(true) }) it('rejects unsupported hello roles without replacing an authenticated stream', async () => { const onAuthenticatedClientPair = vi.fn() server = new DaemonServer({ socketPath, tokenPath, onAuthenticatedClientPair, spawnSubprocess: () => createMockSubprocess() }) await server.start() client = new DaemonClient({ socketPath, tokenPath }) await client.ensureConnected() const daemon = server as unknown as DaemonServerInternals const connectedClient = [...daemon.connections.clients.values()][0] const originalStream = connectedClient.streamSocket const unsupported = connect(socketPath) unsupported.on('error', () => {}) await new Promise((resolve) => unsupported.once('connect', resolve)) const response = new Promise<{ ok: boolean; error?: string }>((resolve) => { unsupported.once('data', (data) => resolve(JSON.parse(data.toString()))) }) unsupported.write( encodeNdjson({ type: 'hello', version: PROTOCOL_VERSION, token: readFileSync(tokenPath, 'utf8'), clientId: connectedClient.clientId, role: 'observer' }) ) await expect(response).resolves.toMatchObject({ ok: false, error: 'Invalid role' }) unsupported.destroy() expect(connectedClient.streamSocket).toBe(originalStream) expect(connectedClient.authenticatedPairEstablished).toBe(true) expect(onAuthenticatedClientPair).toHaveBeenCalledOnce() }) it.skipIf(process.platform === 'win32')( 'stands down instead of serving when the server errors while publication is in flight', async () => { // Why: publishing awaits a liveness probe, so a server error can land mid-flight and the // rejection alone cannot stop it. Going on to serve would leave a live published daemon // behind a caller told startup failed — and a caller that responds by launching a // replacement recreates the split brain. // A dead entry on the canonical name forces publish down the probe-then-rename path, so // the error below lands inside the async window rather than before it. const stale = createServer() const stalePath = join(dir, '.bstale00001') await new Promise((resolve) => stale.listen(stalePath, resolve)) linkSync(stalePath, socketPath) await new Promise((resolve) => stale.close(() => resolve())) server = new DaemonServer({ socketPath, tokenPath, spawnSubprocess: () => createMockSubprocess() }) const started = server.start() const daemon = server as unknown as DaemonServerInternals await vi.waitFor(() => expect(daemon.lifecycle.server?.listening).toBe(true)) daemon.lifecycle.server?.emit('error', new Error('injected accept failure')) await expect(started).rejects.toThrow('injected accept failure') // Why wait: start() rejects the moment the error lands, while publication is still in // flight. Reporting failure must end with the daemon actually not serving. await vi.waitFor(() => expect(daemon.lifecycle.server).toBeNull()) await expect(waitForEndpointUnreachable(socketPath)).resolves.toBe(true) } ) })