hermes-web-ui/tests/server/gateway-respawn.test.ts

import { EventEmitter } from 'events'
import { afterEach, describe, expect, it, vi } from 'vitest'

const originalEnv = { ...process.env }

class FakeChild extends EventEmitter {
  pid: number
  killSignals: string[] = []
  constructor(pid: number) {
    super()
    this.pid = pid
  }
  unref() { /* no-op */ }
  kill(signal?: string) {
    this.killSignals.push(signal || 'SIGTERM')
    return true
  }
}

let fakeChildren: FakeChild[] = []

vi.mock('../../packages/server/src/services/hermes/hermes-process', () => ({
  resolveHermesInvocation: (bin: string) => ({ command: bin, argsPrefix: [] }),
  execHermesWithBin: vi.fn(),
  execHermes: vi.fn(),
  spawnHermesWithBin: vi.fn(() => {
    const pid = 10000 + fakeChildren.length
    const child = new FakeChild(pid)
    fakeChildren.push(child)
    return child
  }),
  spawnHermes: vi.fn(),
  resolveHermesBin: () => 'hermes',
}))

afterEach(() => {
  vi.useRealTimers()
  vi.restoreAllMocks()
  vi.resetModules()
  process.env = { ...originalEnv }
  fakeChildren = []
})

describe('gateway-runner supervision', () => {
  it('respawns the gateway when the spawned child dies unexpectedly', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    const first = startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/fake-a' })
    expect(first.pid).toBe(10000)
    expect(fakeChildren).toHaveLength(1)

    // Simulate unexpected exit
    fakeChildren[0].emit('exit', 1, null)

    // No respawn yet — timer is pending
    expect(fakeChildren).toHaveLength(1)

    await vi.advanceTimersByTimeAsync(2500)

    // A new spawn should have been issued
    expect(fakeChildren.length).toBeGreaterThanOrEqual(2)
    const newPid = fakeChildren[fakeChildren.length - 1].pid
    expect(newPid).not.toBe(10000)
  })

  it('stops respawning after three consecutive quick failures', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/flapping' })
    expect(fakeChildren).toHaveLength(1)

    for (let i = 0; i < 3; i += 1) {
      fakeChildren[fakeChildren.length - 1].emit('exit', 1, null)
      await vi.advanceTimersByTimeAsync(2500)
    }

    expect(fakeChildren).toHaveLength(4)

    fakeChildren[fakeChildren.length - 1].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(5000)

    expect(fakeChildren).toHaveLength(4)
  })

  it('resets the respawn limit after a stable run', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/recovered' })

    fakeChildren[0].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(2500)
    fakeChildren[1].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(2500)
    expect(fakeChildren).toHaveLength(3)

    await vi.advanceTimersByTimeAsync(31000)
    fakeChildren[2].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(2500)
    expect(fakeChildren).toHaveLength(4)

    fakeChildren[3].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(2500)
    fakeChildren[4].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(2500)
    expect(fakeChildren).toHaveLength(6)

    fakeChildren[5].emit('exit', 1, null)
    await vi.advanceTimersByTimeAsync(5000)
    expect(fakeChildren).toHaveLength(6)
  })

  it('cancels a pending respawn when a fresh start is issued for the same profile (the /restart case)', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    // First start: a gateway is running
    const first = startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/fake-b' })
    expect(first.pid).toBe(10000)

    // Simulate `/restart`: old gateway dies, then a new start happens before
    // the respawn timer fires.
    fakeChildren[0].emit('exit', 1, null)
    expect(fakeChildren).toHaveLength(1) // respawn not yet fired

    // New start (the `/restart` second phase) for the same profile. The
    // pending respawn timer should be cancelled.
    const second = startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/fake-b' })
    expect(second.pid).toBe(10001)
    expect(fakeChildren).toHaveLength(2)

    // Now advance timers past the original respawn delay. The cancelled
    // respawn must NOT fire — otherwise we'd have a third gateway racing
    // with the second.
    await vi.advanceTimersByTimeAsync(5000)
    expect(fakeChildren).toHaveLength(2)
  })

  it('tracks respawns independently per profile', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/profile-x' })
    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/profile-y' })
    expect(fakeChildren).toHaveLength(2)

    // profile-x's gateway dies — only its respawn timer should fire
    fakeChildren[0].emit('exit', 1, null)

    await vi.advanceTimersByTimeAsync(2500)

    // profile-x was respawned (3rd child), profile-y was untouched
    expect(fakeChildren).toHaveLength(3)
    expect(fakeChildren[2].pid).toBe(10002)
  })

  it('does not respawn if a new start for the same profile has already replaced the dead child', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    const first = startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/fake-c' })
    fakeChildren[0].emit('exit', 0, 'SIGTERM')

    // Issue a fresh start BEFORE the respawn timer fires — this clears the
    // pending respawn. The old child's exit handler then becomes a no-op
    // because `state.current?.pid !== oldPid` after the new start.
    const second = startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/fake-c' })
    expect(second.pid).not.toBe(first.pid)

    // Now the original (now-orphaned) exit listener fires after the timer
    // window. It should detect the mismatch and not schedule another spawn.
    await vi.advanceTimersByTimeAsync(5000)

    // Exactly two children total: the original, and the replacement
    expect(fakeChildren).toHaveLength(2)
  })

  it('retires one managed profile gateway without scheduling a respawn', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { retireManagedGatewayForProfile, startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/delete-me' })
    expect(fakeChildren).toHaveLength(1)

    const retired = retireManagedGatewayForProfile('/tmp/delete-me', { timeoutMs: 5000 })
    expect(fakeChildren[0].killSignals).toEqual(['SIGTERM'])

    fakeChildren[0].emit('exit', 0, 'SIGTERM')
    await expect(retired).resolves.toEqual({ signaled: 1, forced: 0, errors: 0 })
    await vi.advanceTimersByTimeAsync(6000)

    expect(fakeChildren).toHaveLength(1)
  })

  it('stops managed gateways on shutdown without respawning them', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { shutdownManagedGateways, startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/shutdown-a' })
    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/shutdown-b' })
    expect(fakeChildren).toHaveLength(2)

    const shutdown = shutdownManagedGateways({ timeoutMs: 5000 })

    expect(fakeChildren[0].killSignals).toEqual(['SIGTERM'])
    expect(fakeChildren[1].killSignals).toEqual(['SIGTERM'])

    fakeChildren[0].emit('exit', 0, 'SIGTERM')
    fakeChildren[1].emit('exit', 0, 'SIGTERM')

    await expect(shutdown).resolves.toEqual({ signaled: 2, forced: 0, errors: 0 })
    await vi.advanceTimersByTimeAsync(6000)

    expect(fakeChildren).toHaveLength(2)
  })

  it('cancels pending respawn timers during managed gateway shutdown', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { shutdownManagedGateways, startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/shutdown-c' })
    fakeChildren[0].emit('exit', 1, null)

    const result = await shutdownManagedGateways({ timeoutMs: 5000 })
    expect(result).toEqual({ signaled: 0, forced: 0, errors: 0 })

    await vi.advanceTimersByTimeAsync(6000)
    expect(fakeChildren).toHaveLength(1)
  })

  it('forces managed gateway shutdown when children do not exit', async () => {
    vi.useFakeTimers()
    vi.resetModules()
    const { shutdownManagedGateways, startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/shutdown-d' })

    const shutdown = shutdownManagedGateways({ timeoutMs: 1000 })
    expect(fakeChildren[0].killSignals).toEqual(['SIGTERM'])

    await vi.advanceTimersByTimeAsync(1000)

    await expect(shutdown).resolves.toEqual({ signaled: 1, forced: 1, errors: 0 })
    expect(fakeChildren[0].killSignals).toEqual(['SIGTERM', 'SIGKILL'])
  })

  it('kills the managed gateway process tree on Windows shutdown', async () => {
    vi.resetModules()
    const killWindowsProcessTree = vi.fn().mockResolvedValue(undefined)
    const { shutdownManagedGateways, startGatewayRunManaged } = await import(
      '../../packages/server/src/services/hermes/gateway-runner'
    )

    startGatewayRunManaged('/usr/bin/hermes', { profileDir: '/tmp/shutdown-win' })

    await expect(shutdownManagedGateways({
      platform: 'win32',
      killWindowsProcessTree,
    })).resolves.toEqual({ signaled: 1, forced: 1, errors: 0 })

    expect(killWindowsProcessTree).toHaveBeenCalledWith(10000)
    expect(fakeChildren[0].killSignals).toEqual([])
  })
})