From 0953e7ca8e22f86cbc636d32a204efc674d65b20 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Mon, 22 Jun 2026 22:41:42 -0700 Subject: [PATCH] test(#4740): fix two order/load-dependent full-suite flakes (test-only) (#4762) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(#4740): pin HERMES_HOME in readonly-scope pool test to fix order-dependent flake test_active_request_readonly_scope_blocks_pool_env_seed failed only under full-suite ordering (passed in isolation). Root cause, captured via an assertion-time diagnostic: the readonly scope correctly blocks the process-env credential fallback (branch 2 returns ''), but _has_explicit_pool_credentials (branch 3) calls read_credential_pool, which falls back to the GLOBAL-root auth.json via get_default_hermes_root() — and that reads os.environ['HERMES_HOME'] DIRECTLY, not the thread-local/scoped home. Under suite ordering HERMES_HOME pointed at the shared persistent test state dir, where an earlier test had persisted an openrouter credential_pool entry (source: env:OPENROUTER_API_KEY) into auth.json. The global-root fallback found that leaked entry, so _provider_has_key('openrouter') returned True and the assertion failed. Fix: monkeypatch.setenv('HERMES_HOME', ) so the global-root fallback resolves to this test's own empty tmp dir with no leaked auth.json. Test-only; no product change. Verified: passes in isolation AND the target test now passes under the full suite (the only remaining full-suite reds were unrelated subprocess-timeout SIGKILLs in test_tls_support under load, which pass on re-run). * test(#4740): retry config-probe subprocesses in test_tls_support under load The 3 TestTLSConfigFlag tests spawn a 'python -c' subprocess that imports api.config (heavyweight) with a 10s timeout. Under the fully parallel full suite the box can be saturated enough that the import exceeds 10s and the runner SIGKILLs the child (returncode -9) — a pure resource-contention flake (passes in isolation). Add _run_config_probe(): 60s timeout + retry on TimeoutExpired/SIGKILL, returning a real non-zero exit immediately so genuine failures still surface. Zero-flake-tolerance sweep (#4740). --------- Co-authored-by: nesquena-hermes --- ...test_issue3957_profile_providers_models.py | 13 +++++ tests/test_tls_support.py | 50 +++++++++++++------ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tests/test_issue3957_profile_providers_models.py b/tests/test_issue3957_profile_providers_models.py index d77a76c3d..27c2d1797 100644 --- a/tests/test_issue3957_profile_providers_models.py +++ b/tests/test_issue3957_profile_providers_models.py @@ -331,6 +331,19 @@ def test_active_request_readonly_scope_blocks_pool_env_seed(monkeypatch, tmp_pat work_home = base / "profiles" / "work" work_home.mkdir(parents=True) monkeypatch.setattr(profiles, "_DEFAULT_HERMES_HOME", base) + # Point HERMES_HOME at this test's own tmp base too (#4740). The readonly scope + # blocks the process-env credential fallback correctly, but the credential pool's + # global-root fallback (read_credential_pool -> _load_global_auth_store) resolves + # the root via get_default_hermes_root(), which reads os.environ["HERMES_HOME"] + # DIRECTLY — not the thread-local/scoped home. Under full-suite ordering that env + # var points at the shared persistent test state dir, where an EARLIER test + # persisted an openrouter credential_pool entry (source: env:OPENROUTER_API_KEY) + # into auth.json. read_credential_pool then falls back to that global store, finds + # the leaked openrouter entry, and _has_explicit_pool_credentials returns True — + # an order-dependent false failure that doesn't reproduce in isolation. Pinning + # HERMES_HOME to this test's empty tmp base makes the global-root fallback resolve + # to a clean dir with no leaked auth.json. + monkeypatch.setenv("HERMES_HOME", str(base)) monkeypatch.setenv("OPENROUTER_API_KEY", "from-process-env") profiles.set_request_profile("work") diff --git a/tests/test_tls_support.py b/tests/test_tls_support.py index d32205706..e3f28d6df 100644 --- a/tests/test_tls_support.py +++ b/tests/test_tls_support.py @@ -23,6 +23,38 @@ from conftest import requires_fcntl ROOT = Path(__file__).parent.parent +def _run_config_probe(code: str, env=None, *, attempts: int = 3, timeout: int = 60): + """Run a short `python -c` probe that imports api.config, with retries. + + Importing api.config in a fresh subprocess is heavyweight, and under a fully + parallel test suite the box can be saturated enough that a 10s timeout trips + and the runner SIGKILLs the child (returncode -9) — a pure resource-contention + flake, not a logic failure. Use a generous timeout and retry the spawn on + TimeoutExpired / SIGKILL so the full-suite run is deterministic (#4740 sweep: + zero tolerance for load-dependent flakes). A genuine non-zero exit with output + is returned immediately (no retry) so real failures still surface fast. + """ + last_exc = None + for attempt in range(attempts): + try: + r = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, text=True, timeout=timeout, + cwd=str(ROOT), env=env, + ) + except subprocess.TimeoutExpired as exc: + last_exc = exc + continue # contention — respawn + # returncode -9 == SIGKILL (OOM/oversubscription under load): retry. + if r.returncode == -9 and attempt < attempts - 1: + continue + return r + raise AssertionError( + f"config probe subprocess did not complete after {attempts} attempts " + f"(timeout={timeout}s each); last error: {last_exc!r}" + ) + + def _gen_test_cert(tmpdir: Path) -> tuple[str, str]: """Generate a self-signed cert and key pair for testing.""" cert = str(tmpdir / "test_cert.pem") @@ -149,11 +181,7 @@ class TestTLSConfigFlag(unittest.TestCase): from api.config import TLS_ENABLED print(TLS_ENABLED) """) - r = subprocess.run( - [os.sys.executable, "-c", code], - capture_output=True, text=True, timeout=10, - cwd=str(ROOT), - ) + r = _run_config_probe(code) self.assertEqual(r.stdout.strip(), "True") def test_tls_enabled_false_when_env_absent(self): @@ -166,11 +194,7 @@ class TestTLSConfigFlag(unittest.TestCase): from api.config import TLS_ENABLED print(TLS_ENABLED) """) - r = subprocess.run( - [os.sys.executable, "-c", code], - capture_output=True, text=True, timeout=10, - cwd=str(ROOT), env=env, - ) + r = _run_config_probe(code, env=env) self.assertEqual(r.stdout.strip(), "False") def test_tls_enabled_false_when_only_cert_set(self): @@ -181,11 +205,7 @@ class TestTLSConfigFlag(unittest.TestCase): from api.config import TLS_ENABLED print(TLS_ENABLED) """) - r = subprocess.run( - [os.sys.executable, "-c", code], - capture_output=True, text=True, timeout=10, - cwd=str(ROOT), env=env, - ) + r = _run_config_probe(code, env=env) self.assertEqual(r.stdout.strip(), "False")