diff --git a/CHANGELOG.md b/CHANGELOG.md index c020392ca..2dfce27de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ ## [Unreleased] +## [v0.51.304] — 2026-06-06 — Release JT (stage-p2a — un-held terminal reaper fix + opt-in Docker GPU image) + +### Fixed +- **Embedded-terminal descendants are now reaped without clobbering unrelated subprocess exit codes.** Terminal descendants reparented to the WebUI process could linger as zombies. The reaper now runs `os.waitpid(-terminal_pgid, WNOHANG)` scoped to the terminal's own process group (terminals spawn with `start_new_session=True`, so the child's PID is its PGID) instead of a process-wide `waitpid(-1)` — the latter could reap an unrelated WebUI subprocess (update/git/provider) before its owner called `.wait()`, silently coercing that child's real exit code to `0`. Bounded by a 64-iteration limit, lock-guarded, and run on both reader cleanup and terminal close. (#3725 fixes #2577, @rodboev) + +### Added +- **Optional GPU runtime image path.** The default image stays CPU-only. Building with `--build-arg INSTALL_GPU_LIBS=1` installs VA-API user-space libraries for users passing through host GPU devices, and `docker_init.bash` now preserves Docker `--group-add` supplemental device groups (e.g. `render`/`video` for `/dev/dri` access) when dropping privileges to the runtime user. With the default `INSTALL_GPU_LIBS=0` this is a no-op. (#3721 addresses #3243, @rodboev) + ## [v0.51.303] — 2026-06-06 — Release JS (stage-p1a — low-risk fixes: cron toggle, config var expansion, git-discard hardening) ### Fixed diff --git a/Dockerfile b/Dockerfile index 241f0d1ed..8d60c0bdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,26 @@ RUN apt-get update -y --fix-missing --no-install-recommends \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Optional GPU user-space acceleration libraries for users who pass through +# host GPU devices. The default image remains CPU-only. +ARG INSTALL_GPU_LIBS=0 +RUN if [ "$INSTALL_GPU_LIBS" = "1" ]; then \ + apt-get update -y --fix-missing --no-install-recommends \ + && apt-get install -y --no-install-recommends \ + libva2 \ + vainfo \ + mesa-va-drivers \ + && if apt-cache show intel-media-va-driver-non-free >/dev/null 2>&1; then \ + apt-get install -y --no-install-recommends intel-media-va-driver-non-free; \ + else \ + echo "intel-media-va-driver-non-free is not available from the configured Debian repositories; skipping Intel non-free VA-API driver."; \ + fi \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*; \ + else \ + echo "Skipping optional GPU user-space acceleration libraries (INSTALL_GPU_LIBS=0)."; \ + fi + # UTF-8 RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 ENV LANG=en_US.utf8 diff --git a/api/terminal.py b/api/terminal.py index c85e1f3ec..18c1fedbe 100644 --- a/api/terminal.py +++ b/api/terminal.py @@ -89,6 +89,8 @@ _spawn_queue: queue.Queue = queue.Queue() _spawn_supervisor_started = False _spawn_supervisor_lock = threading.Lock() _spawn_supervisor_thread: threading.Thread | None = None +_terminal_descendant_reaper_lock = threading.Lock() +_TERMINAL_DESCENDANT_REAPER_LIMIT = 64 @dataclass @@ -131,6 +133,32 @@ def _reap_abandoned_spawn(proc: subprocess.Popen) -> bool: return True +def _reap_terminal_descendants( + terminal_pgid: int, + limit: int = _TERMINAL_DESCENDANT_REAPER_LIMIT, +) -> int: + """Reap exited descendants that still belong to a terminal-owned process group.""" + if not _TERMINAL_SUPPORTED: + return 0 + try: + terminal_pgid = abs(int(terminal_pgid)) + except (TypeError, ValueError): + return 0 + if terminal_pgid <= 0: + return 0 + reaped = 0 + with _terminal_descendant_reaper_lock: + for _ in range(max(0, int(limit))): + try: + pid, _status = os.waitpid(-terminal_pgid, os.WNOHANG) + except (ChildProcessError, OSError): + break + if pid == 0: + break + reaped += 1 + return reaped + + def _spawn_supervisor_loop() -> None: while True: request = None @@ -248,6 +276,7 @@ def _reader_loop(term: TerminalSession) -> None: finally: term.closed.set() code = term.proc.poll() + _reap_terminal_descendants(term.proc.pid) term.put_output("terminal_closed", {"exit_code": code}) @@ -416,6 +445,7 @@ def close_terminal(session_id: str) -> bool: os.close(term.master_fd) except OSError: pass + _reap_terminal_descendants(term.proc.pid) return True diff --git a/docker_init.bash b/docker_init.bash index 2002de159..dc8cd740c 100644 --- a/docker_init.bash +++ b/docker_init.bash @@ -264,6 +264,29 @@ if [ "A${whoami}" == "Aroot" ]; then chmod 600 "$ENV_FILE" || error_exit "Failed to secure $ENV_FILE" export _HW_ROOT_ENV_PATH="$ENV_FILE" + # Preserve Docker --group-add supplemental groups (for example render/video + # for /dev/dri GPU access) when dropping privileges. `su` rebuilds the target + # user's groups from /etc/group, so host-passed numeric groups must be made + # visible to hermeswebui before re-entering as the runtime user. + for gid in $(id -G); do + if [ "$gid" = "0" ] || [ "$gid" = "$WANTED_GID" ]; then + continue + fi + group_name="$(getent group "$gid" | cut -d: -f1 || true)" + if [ -z "$group_name" ]; then + group_name="hostgpu${gid}" + groupadd -g "$gid" "$group_name" 2>/dev/null || true + group_name="$(getent group "$gid" | cut -d: -f1 || true)" + fi + if [ -z "$group_name" ]; then + echo "!! WARNING: Could not create supplemental group for GID $gid; GPU device access may be unavailable" + continue + fi + if [ -n "$group_name" ]; then + usermod -a -G "$group_name" hermeswebui 2>/dev/null || echo "!! WARNING: Could not add hermeswebui to supplemental group $group_name ($gid)" + fi + done + # restart the script as hermeswebui set with the correct UID/GID this time echo "-- Restarting as hermeswebui user with UID ${WANTED_UID} GID ${WANTED_GID}" exec su -s /bin/bash -c "exec \"${script_fullname}\"" hermeswebui || error_exit "subscript failed" diff --git a/docs/docker.md b/docs/docker.md index b9d254e83..cc7d96a85 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -66,6 +66,80 @@ isolated Hermes home and follow > for a one-off root run, use `sudo -E docker compose up -d` and verify the > rendered mount with `docker compose config` first. +## Optional GPU runtime image + +The default Hermes WebUI Docker image stays CPU-only. GPU user-space packages +are installed only when you build a custom image with the opt-in build arg: + +```bash +docker build --build-arg INSTALL_GPU_LIBS=1 -t hermes-webui:gpu . +``` + +That build path installs VA-API basics (`libva2`, `vainfo`), AMD Mesa VA-API +drivers (`mesa-va-drivers`), and the Intel non-free media driver when that +package is available from the configured Debian repositories. NVIDIA host +runtime tooling is not installed into the app image; use the NVIDIA Container +Toolkit on the host and pass GPUs through at runtime. + +GPU passthrough still depends on host drivers, Docker runtime support, and +device mappings. The commands below are configuration guidance for a suitable +Linux Docker host; they are not a claim that native GPU passthrough was verified +in this workspace. + +### Intel and AMD VA-API + +Expose the host render devices and add the runtime user to the common video and +render groups: + +```bash +docker run --rm \ + --device /dev/dri:/dev/dri \ + --group-add video \ + --group-add render \ + hermes-webui:gpu vainfo +``` + +For Compose, add the same mapping to a custom service definition: + +```yaml +services: + hermes-webui: + image: hermes-webui:gpu + devices: + - /dev/dri:/dev/dri + group_add: + - video + - render +``` + +`vainfo` should list the VA-API driver and supported profiles when the host +driver stack and container permissions are correct. The container entrypoint +preserves Docker-provided supplemental groups before it drops privileges to the +`hermeswebui` runtime user, so the WebUI process keeps access to `/dev/dri`. + +### NVIDIA + +Install and configure the NVIDIA Container Toolkit on the host first, then use +Docker's GPU runtime flag: + +```bash +docker run --rm --gpus all hermes-webui:gpu nvidia-smi +``` + +For Compose, use a custom service with GPU access enabled: + +```yaml +services: + hermes-webui: + image: hermes-webui:gpu + gpus: all +``` + +If `nvidia-smi` is unavailable or reports no devices, fix the host NVIDIA driver +and container toolkit setup before debugging Hermes WebUI. The container image +only supplies the WebUI plus optional user-space media libraries; it cannot +provide host kernel drivers or the NVIDIA runtime. + ## Scheduled jobs and the gateway daemon **Symptom**: Cron jobs created in the Tasks panel never fire. System Settings or Tasks shows: @@ -347,6 +421,7 @@ volumes: - #1399 — UID alignment in compose files (fixed in v0.50.260 via PR #1428 + this guide) - #3012 — host `localhost` API URLs fail from Docker containers (use `host.docker.internal` / `host.containers.internal`) - #3006 — `sudo docker compose` can mount `/root/.hermes` instead of the user's Hermes home +- #3243 — optional GPU runtime image/docs for containerized acceleration workloads - #858 — two-container `/opt/hermes` path confusion - #681 — tools running in WebUI container, not agent container (architectural) - #668 — auto-detect UID/GID from mounted volume diff --git a/tests/test_docker_gpu_runtime_docs.py b/tests/test_docker_gpu_runtime_docs.py new file mode 100644 index 000000000..95c2766e1 --- /dev/null +++ b/tests/test_docker_gpu_runtime_docs.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + + +def _repo_text(path): + return (REPO / path).read_text(encoding="utf-8") + + +def test_dockerfile_gpu_libraries_are_opt_in(): + """The production image must stay CPU-only unless the GPU build arg is set.""" + dockerfile = _repo_text("Dockerfile") + + assert "ARG INSTALL_GPU_LIBS=0" in dockerfile + assert 'if [ "$INSTALL_GPU_LIBS" = "1" ]' in dockerfile + + opt_in_block = dockerfile[dockerfile.index("ARG INSTALL_GPU_LIBS=0"):] + for package in ( + "libva2", + "vainfo", + "mesa-va-drivers", + "intel-media-va-driver-non-free", + ): + assert package in opt_in_block, ( + f"{package} must only appear in the INSTALL_GPU_LIBS opt-in block." + ) + assert package not in dockerfile[:dockerfile.index("ARG INSTALL_GPU_LIBS=0")] + + +def test_dockerfile_handles_missing_intel_non_free_driver(): + """Debian slim repos may not expose the non-free Intel VA-API package.""" + dockerfile = _repo_text("Dockerfile") + + assert "apt-cache show intel-media-va-driver-non-free" in dockerfile + assert "skipping Intel non-free VA-API driver" in dockerfile + + +def test_docker_docs_show_gpu_build_command(): + docker_docs = _repo_text("docs/docker.md") + + assert "Optional GPU runtime image" in docker_docs + assert "--build-arg INSTALL_GPU_LIBS=1" in docker_docs + assert "default Hermes WebUI Docker image stays CPU-only" in docker_docs + + +def test_docker_docs_cover_intel_amd_dri_mapping(): + docker_docs = _repo_text("docs/docker.md") + + assert "Intel and AMD VA-API" in docker_docs + assert "--device /dev/dri:/dev/dri" in docker_docs + assert "/dev/dri:/dev/dri" in docker_docs + assert "group_add:" in docker_docs + assert "video" in docker_docs + assert "render" in docker_docs + assert "vainfo" in docker_docs + assert "preserves Docker-provided supplemental groups" in docker_docs + + +def test_docker_docs_cover_nvidia_host_runtime_guidance(): + docker_docs = _repo_text("docs/docker.md") + + assert "NVIDIA Container Toolkit" in docker_docs + assert "--gpus all" in docker_docs + assert "gpus: all" in docker_docs + assert "host NVIDIA driver" in docker_docs + assert "host kernel drivers" in docker_docs + assert "NVIDIA runtime" in docker_docs + + +def test_docker_docs_do_not_claim_native_gpu_passthrough_verification(): + docker_docs = _repo_text("docs/docker.md") + + assert "not a claim that native GPU passthrough was verified" in docker_docs + assert "depends on host drivers" in docker_docs + + +def test_docker_init_preserves_supplemental_device_groups_for_runtime_user(): + docker_init = _repo_text("docker_init.bash") + root_phase = docker_init[:docker_init.index("exec su -s /bin/bash")] + + assert "for gid in $(id -G)" in root_phase + assert "groupadd -g \"$gid\"" in root_phase + assert "Could not create supplemental group for GID $gid" in root_phase + assert "usermod -a -G \"$group_name\" hermeswebui" in root_phase + assert "Docker --group-add supplemental groups" in root_phase + + +def test_changelog_mentions_optional_gpu_runtime_path(): + changelog = _repo_text("CHANGELOG.md") + unreleased = changelog[changelog.index("## [Unreleased]"):changelog.index("## [v0.51.293]")] + + assert "Optional GPU runtime image path" in unreleased + assert "INSTALL_GPU_LIBS=1" in unreleased + assert "supplemental device groups" in unreleased diff --git a/tests/test_terminal_zombie_reaper.py b/tests/test_terminal_zombie_reaper.py new file mode 100644 index 000000000..aeee46106 --- /dev/null +++ b/tests/test_terminal_zombie_reaper.py @@ -0,0 +1,124 @@ +import os +import sys +import time + +import pytest + +pytestmark = pytest.mark.skipif( + os.name == "nt" + or not sys.platform.startswith("linux") + or not getattr(__import__("api.terminal", fromlist=["_TERMINAL_SUPPORTED"]), "_TERMINAL_SUPPORTED", False), + reason="Linux-only terminal zombie reaper coverage", +) + +import api.terminal as terminal + + +def _wait_until_waitable(pid: int, timeout: float = 2.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if result is not None and result.si_pid == pid: + return + time.sleep(0.01) + raise AssertionError(f"child {pid} did not exit before timeout") + + +def test_reap_terminal_descendants_reaps_exited_child(): + ready_r, ready_w = os.pipe() + pid = os.fork() + if pid == 0: + os.close(ready_r) + os.setpgid(0, 0) + os.write(ready_w, b"1") + os.close(ready_w) + os._exit(0) + + reaped = False + try: + os.close(ready_w) + assert os.read(ready_r, 1) == b"1" + os.close(ready_r) + _wait_until_waitable(pid) + + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + terminal._reap_terminal_descendants(pid) + try: + os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + except ChildProcessError: + reaped = True + break + time.sleep(0.01) + + assert reaped, "terminal descendant reaper did not reap the exited child" + finally: + if not reaped: + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + +def test_close_terminal_reaps_descendants_after_shell_wait(monkeypatch): + class FakeProc: + pid = 987654 + + def __init__(self): + self.wait_calls = [] + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + self.wait_calls.append(timeout) + self.returncode = -1 + return self.returncode + + proc = FakeProc() + term = terminal.TerminalSession( + session_id="term-descendant-reap", + workspace="/tmp", + proc=proc, + master_fd=12345, + ) + terminal._TERMINALS["term-descendant-reap"] = term + kills = [] + reaped = [] + + monkeypatch.setattr(terminal.os, "killpg", lambda pid, sig: kills.append((pid, sig))) + monkeypatch.setattr(terminal.os, "close", lambda fd: None) + monkeypatch.setattr(terminal, "_reap_terminal_descendants", lambda pgid: reaped.append(pgid) or 0) + + assert terminal.close_terminal("term-descendant-reap") is True + + assert kills == [(proc.pid, terminal.signal.SIGHUP)] + assert proc.wait_calls == [1.5] + assert reaped == [proc.pid] + + +def test_reap_terminal_descendants_ignores_expected_waitpid_errors(monkeypatch): + calls = [] + + def fake_waitpid(pid, flags): + calls.append((pid, flags)) + raise ChildProcessError() + + monkeypatch.setattr(terminal.os, "waitpid", fake_waitpid) + + assert terminal._reap_terminal_descendants(123) == 0 + assert calls == [(-123, os.WNOHANG)] + + +def test_reap_terminal_descendants_is_bounded(monkeypatch): + calls = [] + + def fake_waitpid(pid, flags): + calls.append((pid, flags)) + return (len(calls), 0) + + monkeypatch.setattr(terminal.os, "waitpid", fake_waitpid) + + assert terminal._reap_terminal_descendants(123, limit=3) == 3 + assert calls == [(-123, os.WNOHANG), (-123, os.WNOHANG), (-123, os.WNOHANG)]