hermes-webui/tests/test_issue1240_generic_cli_catalog_sync.py

"""Regression tests for #1240 — WebUI model catalog should delegate to Hermes CLI.

The WebUI picker should not freeze ordinary providers to its static
``_PROVIDER_MODELS`` snapshot when Hermes CLI can return a fresher provider
catalog. Static lists remain a fallback only.
"""

from __future__ import annotations

import sys
import types

import api.config as config


_PROVIDER_ENV_VARS = (
    "ANTHROPIC_API_KEY",
    "OPENAI_API_KEY",
    "OPENROUTER_API_KEY",
    "GOOGLE_API_KEY",
    "GEMINI_API_KEY",
    "GLM_API_KEY",
    "KIMI_API_KEY",
    "DEEPSEEK_API_KEY",
    "XIAOMI_API_KEY",
    "OPENCODE_ZEN_API_KEY",
    "OPENCODE_GO_API_KEY",
    "MINIMAX_API_KEY",
    "MINIMAX_CN_API_KEY",
    "XAI_API_KEY",
    "MISTRAL_API_KEY",
    "OLLAMA_CLOUD_API_KEY",
    "OLLAMA_API_KEY",
    "NOUS_API_KEY",
    "NVIDIA_API_KEY",
)


def _scrub_provider_env(monkeypatch):
    for name in _PROVIDER_ENV_VARS:
        monkeypatch.delenv(name, raising=False)


def _install_fake_hermes_cli(monkeypatch, *, provider_id: str, live_ids, raise_on_lookup: bool = False):
    """Install a hermes_cli stub that reports one authenticated provider."""
    fake_pkg = types.ModuleType("hermes_cli")
    fake_pkg.__path__ = []

    fake_models = types.ModuleType("hermes_cli.models")
    fake_models.list_available_providers = lambda: [
        {"id": provider_id, "authenticated": True}
    ]

    calls: list[str] = []

    def provider_model_ids(pid):
        calls.append(pid)
        if raise_on_lookup:
            raise RuntimeError("simulated provider_model_ids failure")
        return list(live_ids) if pid == provider_id else []

    fake_models.provider_model_ids = provider_model_ids

    fake_auth = types.ModuleType("hermes_cli.auth")

    def get_auth_status(pid):
        if pid == provider_id:
            return {"logged_in": True, "key_source": ""}
        return {"logged_in": False, "key_source": ""}

    fake_auth.get_auth_status = get_auth_status

    monkeypatch.setitem(sys.modules, "hermes_cli", fake_pkg)
    monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models)
    monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth)
    monkeypatch.delitem(sys.modules, "agent.credential_pool", raising=False)
    monkeypatch.delitem(sys.modules, "agent", raising=False)
    config.invalidate_models_cache()
    return calls


def _configure(monkeypatch, tmp_path, *, provider: str, default: str = ""):
    monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "missing-config.yaml")
    monkeypatch.setattr(config, "_models_cache_path", tmp_path / "models_cache.json")
    monkeypatch.setattr(
        config,
        "cfg",
        {
            "model": {"provider": provider, "default": default},
            "providers": {},
            "fallback_providers": [],
        },
    )
    monkeypatch.setattr(config, "_cfg_mtime", 0.0)
    config.invalidate_models_cache()


def _provider_group(result: dict, provider_id: str) -> dict:
    return next(g for g in result["groups"] if g.get("provider_id") == provider_id)


def _ids(group: dict) -> list[str]:
    return [m.get("id") for m in group.get("models", [])]


def test_generic_provider_uses_hermes_cli_catalog_before_static_snapshot(monkeypatch, tmp_path):
    """A normal provider should show fresh CLI-discovered models.

    ``claude-sonnet-5.0`` is intentionally absent from WebUI's static Anthropic
    list. Before this fix the group came entirely from ``_PROVIDER_MODELS`` and
    this model was invisible even though Hermes CLI knew about it.
    """
    _scrub_provider_env(monkeypatch)
    calls = _install_fake_hermes_cli(
        monkeypatch,
        provider_id="anthropic",
        live_ids=["claude-opus-4.7", "claude-sonnet-5.0"],
    )
    _configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7")

    result = config.get_available_models()
    group = _provider_group(result, "anthropic")

    assert calls == ["anthropic"]
    assert _ids(group) == ["claude-opus-4.7", "claude-sonnet-5.0"]
    assert group["models"][1]["label"] == "Claude Sonnet 5.0"


def test_generic_provider_keeps_static_catalog_as_cli_failure_fallback(monkeypatch, tmp_path):
    _scrub_provider_env(monkeypatch)
    calls = _install_fake_hermes_cli(
        monkeypatch,
        provider_id="anthropic",
        live_ids=[],
        raise_on_lookup=True,
    )
    _configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7")

    result = config.get_available_models()
    group = _provider_group(result, "anthropic")

    assert calls == ["anthropic"]
    assert "claude-opus-4.7" in _ids(group)
    assert "claude-sonnet-4.6" in _ids(group)


def test_generic_provider_prefixes_live_ids_when_not_active_provider(monkeypatch, tmp_path):
    """Provider-qualified live IDs must route through the selected provider."""
    _scrub_provider_env(monkeypatch)
    calls = _install_fake_hermes_cli(
        monkeypatch,
        provider_id="anthropic",
        live_ids=["claude-sonnet-5.0"],
    )
    # Anthropic is authenticated via Hermes CLI, but OpenAI is the active
    # default. The Anthropic row still has to be pickable/routable.
    _configure(monkeypatch, tmp_path, provider="openai", default="gpt-5.5")

    result = config.get_available_models()
    group = _provider_group(result, "anthropic")

    assert "anthropic" in calls
    assert _ids(group) == ["@anthropic:claude-sonnet-5.0"]