hermes-webui/tests/test_terminal_linux_lifecycle.py

import queue
import os
import sys
import threading
import time

import pytest


pytestmark = pytest.mark.skipif(
    not sys.platform.startswith("linux"),
    reason="Linux-only terminal process lifecycle regression",
)


@pytest.mark.skip(
    reason="Test is timing-flaky in non-tty CI: depends on bash prompt rendering "
    "+ printf execution echoing back through the PTY within a fixed deadline. "
    "The actual supervisor invariants (concurrent spawn, timeout-race reap, "
    "Popen-failure recovery, supervisor singleton) are covered by the other 7 "
    "tests in this file which all pass deterministically."
)
def test_terminal_survives_short_lived_request_thread(tmp_path):
    # Mirrors ThreadingHTTPServer: the request worker exits after spawning the
    # shell, so terminal lifetime must not be tied to that worker thread.
    from api.terminal import close_terminal, start_terminal, write_terminal

    sid = f"terminal-linux-lifecycle-{os.getpid()}-{id(tmp_path)}"
    result = queue.Queue()

    def request_thread():
        try:
            result.put(start_terminal(sid, tmp_path, rows=8, cols=40, restart=True))
        except Exception as exc:
            result.put(exc)

    thread = threading.Thread(target=request_thread)
    thread.start()
    thread.join(timeout=1.0)
    assert not thread.is_alive()

    term = result.get(timeout=1.0)
    if isinstance(term, Exception):
        raise AssertionError("terminal worker thread failed") from term
    try:
        deadline = time.monotonic() + 0.5
        while time.monotonic() < deadline:
            assert term.proc.poll() is None
            time.sleep(0.02)
        assert term.is_alive()

        marker = f"lifecycle-ok-{os.getpid()}"
        write_terminal(sid, f"printf '{marker}\\n'\n")
        deadline = time.monotonic() + 1.0
        seen = ""
        while time.monotonic() < deadline:
            try:
                event, payload = term.output.get(timeout=0.1)
            except queue.Empty:
                continue
            if event == "output":
                seen += payload.get("text", "")
                if f"{marker}\r\n" in seen or f"{marker}\n" in seen:
                    break
        assert f"{marker}\r\n" in seen or f"{marker}\n" in seen
    finally:
        close_terminal(sid)


def test_terminal_spawn_delegates_popen_to_supervisor_thread(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import close_terminal, start_terminal

    sid = f"terminal-supervisor-delegation-{os.getpid()}-{id(tmp_path)}"
    request_thread_id = None
    popen_thread_id = None
    real_popen = terminal.subprocess.Popen

    def tracking_popen(*args, **kwargs):
        nonlocal popen_thread_id
        popen_thread_id = threading.get_ident()
        return real_popen(*args, **kwargs)

    monkeypatch.setattr(terminal.subprocess, "Popen", tracking_popen)

    result = queue.Queue()

    def request_thread():
        nonlocal request_thread_id
        request_thread_id = threading.get_ident()
        try:
            result.put(start_terminal(sid, tmp_path, rows=8, cols=40, restart=True))
        except Exception as exc:
            result.put(exc)

    thread = threading.Thread(target=request_thread)
    thread.start()
    thread.join(timeout=1.0)
    assert not thread.is_alive()

    term = result.get(timeout=1.0)
    if isinstance(term, Exception):
        raise AssertionError("terminal worker thread failed") from term
    try:
        assert term.proc.poll() is None
        assert popen_thread_id is not None
        assert popen_thread_id != request_thread_id
        assert popen_thread_id != thread.ident
    finally:
        close_terminal(sid)


def test_terminal_supervisor_handles_concurrent_spawns(tmp_path):
    from api.terminal import close_terminal, start_terminal

    results = queue.Queue()
    sids = [
        f"terminal-concurrent-{os.getpid()}-{idx}-{id(tmp_path)}"
        for idx in range(3)
    ]
    barrier = threading.Barrier(len(sids))

    def request_thread(sid):
        try:
            barrier.wait(timeout=1.0)
            results.put((sid, start_terminal(sid, tmp_path, rows=8, cols=40, restart=True)))
        except Exception as exc:
            results.put((sid, exc))

    threads = [threading.Thread(target=request_thread, args=(sid,)) for sid in sids]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(timeout=2.0)
        assert not thread.is_alive()

    terms = {}
    for _ in sids:
        sid, value = results.get(timeout=1.0)
        if isinstance(value, Exception):
            raise AssertionError(f"terminal spawn failed for {sid}") from value
        terms[sid] = value

    try:
        assert set(terms) == set(sids)
        assert all(term.proc.poll() is None for term in terms.values())
        assert all(term.is_alive() for term in terms.values())
    finally:
        for sid in sids:
            close_terminal(sid)


def test_terminal_supervisor_propagates_popen_failure(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import start_terminal

    expected = RuntimeError("spawn failed")
    captured = {}
    real_put = terminal._spawn_queue.put

    def failing_popen(*args, **kwargs):
        raise expected

    def capture_request(request):
        captured["request"] = request
        return real_put(request)

    monkeypatch.setattr(terminal.subprocess, "Popen", failing_popen)
    monkeypatch.setattr(terminal._spawn_queue, "put", capture_request)

    with pytest.raises(RuntimeError, match="spawn failed") as excinfo:
        start_terminal(
            f"terminal-spawn-failure-{os.getpid()}-{id(tmp_path)}",
            tmp_path,
            restart=True,
        )

    assert excinfo.value is expected
    request = captured["request"]
    assert request.done.is_set()
    assert request.error is expected
    assert request.proc is None


def test_terminal_spawn_timeout_abandons_late_process(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import start_terminal

    class FakeProc:
        pid = 987654

        def __init__(self):
            self.wait_calls = []
            self.returncode = None

        def poll(self):
            return self.returncode

        def wait(self, timeout=None):
            self.wait_calls.append(timeout)
            self.returncode = -1
            return -1

        def terminate(self):
            self.returncode = -15

        def kill(self):
            self.returncode = -9

    entered = threading.Event()
    release = threading.Event()
    proc = FakeProc()
    kills = []
    captured = {}
    real_put = terminal._spawn_queue.put

    class TimedOutEvent:
        def __init__(self):
            self.set_calls = 0

        def wait(self, timeout=None):
            return False

        def set(self):
            self.set_calls += 1

        def is_set(self):
            return self.set_calls > 0

    def slow_popen(*args, **kwargs):
        entered.set()
        release.wait(timeout=1.0)
        return proc

    def force_timeout(request):
        request.done = TimedOutEvent()
        captured["request"] = request
        return real_put(request)

    monkeypatch.setattr(terminal.subprocess, "Popen", slow_popen)
    monkeypatch.setattr(terminal._spawn_queue, "put", force_timeout)
    monkeypatch.setattr(terminal.os, "killpg", lambda pid, sig: kills.append((pid, sig)))

    sid = f"terminal-spawn-timeout-{os.getpid()}-{id(tmp_path)}"
    try:
        with pytest.raises(TimeoutError, match="terminal spawn timeout"):
            start_terminal(
                sid,
                tmp_path,
                restart=True,
            )
        assert entered.wait(timeout=1.0)
    finally:
        release.set()

    deadline = time.monotonic() + 1.0
    while time.monotonic() < deadline and not kills:
        time.sleep(0.01)

    assert kills == [(proc.pid, terminal.signal.SIGHUP)]
    assert proc.wait_calls == [1.0]
    assert proc.poll() is not None
    assert sid not in terminal._TERMINALS
    assert captured["request"].done.set_calls == 1
    assert getattr(proc, "returncode") is not None
    assert terminal._spawn_supervisor_thread is not None
    assert terminal._spawn_supervisor_thread.is_alive()


def test_terminal_timeout_race_after_spawn_completion_does_not_orphan(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import close_terminal, start_terminal

    class FakeProc:
        pid = 987657

        def poll(self):
            return None

        def wait(self, timeout=None):
            return 0

    class FalseAfterSetEvent:
        def __init__(self):
            self.inner = threading.Event()
            self.set_calls = 0

        def wait(self, timeout=None):
            self.inner.wait(timeout=1.0)
            return False

        def set(self):
            self.set_calls += 1
            self.inner.set()

        def is_set(self):
            return self.inner.is_set()

    proc = FakeProc()
    captured = {}
    reaped = []
    real_put = terminal._spawn_queue.put

    def fake_popen(*args, **kwargs):
        return proc

    def capture_request(request):
        request.done = FalseAfterSetEvent()
        captured["request"] = request
        return real_put(request)

    monkeypatch.setattr(terminal.subprocess, "Popen", fake_popen)
    monkeypatch.setattr(terminal._spawn_queue, "put", capture_request)
    monkeypatch.setattr(terminal.os, "killpg", lambda *args: None)
    monkeypatch.setattr(terminal, "_reap_abandoned_spawn", lambda proc: reaped.append(proc) or True)

    sid = f"terminal-timeout-race-complete-{os.getpid()}-{id(tmp_path)}"
    term = start_terminal(sid, tmp_path, restart=True)

    try:
        request = captured["request"]
        assert term.proc is proc
        assert request.done.set_calls == 1
        assert request.done.is_set()
        assert not request.timed_out.is_set()
        assert reaped == []
        assert terminal._TERMINALS[sid].proc is proc
    finally:
        close_terminal(sid)


def test_terminal_supervisor_continues_after_mixed_popen_failures(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import close_terminal, start_terminal

    class FakeProc:
        pid = 987655

        def poll(self):
            return None

        def wait(self, timeout=None):
            return 0

    attempts = iter([RuntimeError("first failure"), FakeProc(), RuntimeError("second failure"), FakeProc()])

    def flaky_popen(*args, **kwargs):
        result = next(attempts)
        if isinstance(result, BaseException):
            raise result
        return result

    monkeypatch.setattr(terminal.subprocess, "Popen", flaky_popen)
    monkeypatch.setattr(terminal.os, "killpg", lambda *args: None)

    with pytest.raises(RuntimeError, match="first failure"):
        start_terminal(f"terminal-mixed-fail-1-{os.getpid()}", tmp_path, restart=True)

    sid_ok_1 = f"terminal-mixed-ok-1-{os.getpid()}"
    term_1 = start_terminal(sid_ok_1, tmp_path, restart=True)

    with pytest.raises(RuntimeError, match="second failure"):
        start_terminal(f"terminal-mixed-fail-2-{os.getpid()}", tmp_path, restart=True)

    sid_ok_2 = f"terminal-mixed-ok-2-{os.getpid()}"
    term_2 = start_terminal(sid_ok_2, tmp_path, restart=True)

    try:
        assert term_1 is not term_2
        assert terminal._TERMINALS[sid_ok_1].proc is term_1.proc
        assert terminal._TERMINALS[sid_ok_2].proc is term_2.proc
        assert terminal._spawn_supervisor_thread is not None
        assert terminal._spawn_supervisor_thread.is_alive()
    finally:
        close_terminal(sid_ok_1)
        close_terminal(sid_ok_2)


def test_terminal_supervisor_survives_repeated_popen_failures(monkeypatch, tmp_path):
    import api.terminal as terminal
    from api.terminal import close_terminal, start_terminal

    class FakeProc:
        pid = 987656

        def poll(self):
            return None

        def wait(self, timeout=None):
            return 0

    attempts = {"count": 0}

    def failing_then_success(*args, **kwargs):
        attempts["count"] += 1
        if attempts["count"] <= 5:
            raise RuntimeError(f"spawn failure {attempts['count']}")
        return FakeProc()

    monkeypatch.setattr(terminal.subprocess, "Popen", failing_then_success)
    monkeypatch.setattr(terminal.os, "killpg", lambda *args: None)

    for idx in range(5):
        with pytest.raises(RuntimeError, match=f"spawn failure {idx + 1}"):
            start_terminal(f"terminal-repeat-fail-{idx}-{os.getpid()}", tmp_path, restart=True)

    assert terminal._spawn_supervisor_thread is not None
    assert terminal._spawn_supervisor_thread.is_alive()

    sid = f"terminal-repeat-success-{os.getpid()}"
    term = start_terminal(sid, tmp_path, restart=True)
    try:
        assert term.proc is terminal._TERMINALS[sid].proc
        assert terminal._spawn_supervisor_thread is not None
        assert terminal._spawn_supervisor_thread.is_alive()
    finally:
        close_terminal(sid)