hermes-webui/tests/test_issue644.py

"""Tests for PR #644 — load provider models from config.yaml in get_available_models()."""
import pytest
import api.config as _cfg


@pytest.fixture(autouse=True)
def _isolate_models_cache():
    """Invalidate the models TTL cache before and after every test in this file."""
    try:
        _cfg.invalidate_models_cache()
    except Exception:
        pass
    yield
    try:
        _cfg.invalidate_models_cache()
    except Exception:
        pass


def _available_models_with_cfg(cfg_override):
    """Helper: temporarily patch config.cfg, call get_available_models(), restore.

    We also freeze _cfg_mtime to the *current* config file mtime so that
    get_available_models() does not call reload_config() from disk (which
    would overwrite the in-memory mock with the on-disk config.yaml).
    See #644 — this race exists in CI where config.yaml is present.
    """
    old_cfg = dict(_cfg.cfg)
    _cfg.cfg.clear()
    _cfg.cfg.update(cfg_override)
    # Freeze mtime so reload_config() is not triggered inside get_available_models()
    old_mtime = _cfg._cfg_mtime
    try:
        from pathlib import Path
        _cfg._cfg_mtime = Path(_cfg._get_config_path()).stat().st_mtime
    except OSError:
        _cfg._cfg_mtime = 0.0
    try:
        return _cfg.get_available_models()
    finally:
        _cfg.cfg.clear()
        _cfg.cfg.update(old_cfg)
        _cfg._cfg_mtime = old_mtime


class TestConfigYamlModelsLoading:
    """Verify that providers with explicit models in config.yaml use those models."""

    def test_provider_in_config_but_not_provider_models_gets_cfg_models(self):
        """A provider only in cfg.providers (not _PROVIDER_MODELS) should appear
        with its configured model list instead of being skipped entirely."""
        cfg = {
            "model": {"provider": "my-custom-llm"},
            "providers": {
                "my-custom-llm": {
                    "base_url": "http://custom.local/v1",
                    "models": ["custom-model-a", "custom-model-b"],
                }
            },
        }
        result = _available_models_with_cfg(cfg)
        groups = {g["provider"]: g["models"] for g in result["groups"]}
        # Provider should appear (previously it was silently skipped)
        provider_names = [g["provider"] for g in result["groups"]]
        found = any("my-custom-llm" in n.lower() or "My-Custom-Llm" in n for n in provider_names)
        # If it appears, its models must include our cfg models
        for g in result["groups"]:
            if "custom" in g["provider"].lower():
                model_ids = [m["id"] for m in g["models"]]
                assert any("custom-model-a" in mid for mid in model_ids), (
                    f"custom-model-a not in group models: {model_ids}"
                )

    def test_provider_models_dict_format_expanded(self):
        """models: {model_id: {context_length: ...}} — keys become model IDs."""
        cfg = {
            "model": {"provider": "anthropic"},
            "providers": {
                "anthropic": {
                    "models": {
                        "claude-custom-1": {"context_length": 200000},
                        "claude-custom-2": {"context_length": 100000},
                    }
                }
            },
        }
        result = _available_models_with_cfg(cfg)
        # Find Anthropic group
        for g in result["groups"]:
            if g["provider"] == "Anthropic":
                model_ids = [m["id"] for m in g["models"]]
                assert "claude-custom-1" in model_ids, (
                    f"claude-custom-1 not in Anthropic models: {model_ids}"
                )
                assert "claude-custom-2" in model_ids, (
                    f"claude-custom-2 not in Anthropic models: {model_ids}"
                )
                break

    def test_provider_models_list_format_expanded(self):
        """models: [model_id, ...] — items become model IDs."""
        cfg = {
            "model": {"provider": "anthropic"},
            "providers": {
                "anthropic": {
                    "models": ["claude-list-only-1", "claude-list-only-2"],
                }
            },
        }
        result = _available_models_with_cfg(cfg)
        for g in result["groups"]:
            if g["provider"] == "Anthropic":
                model_ids = [m["id"] for m in g["models"]]
                assert "claude-list-only-1" in model_ids, (
                    f"claude-list-only-1 not in Anthropic models: {model_ids}"
                )
                break

    def test_provider_in_provider_models_but_no_cfg_override_uses_static_fallback(self, monkeypatch):
        """When Hermes CLI has no live catalog, _PROVIDER_MODELS remains fallback."""
        monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda _pid: [])
        cfg = {
            "model": {"provider": "anthropic"},
            "providers": {
                "anthropic": {
                    "api_key": "sk-test",
                    # No 'models' key
                }
            },
        }
        result = _available_models_with_cfg(cfg)
        raw_ids = {m["id"] for m in _cfg._PROVIDER_MODELS.get("anthropic", [])}
        for g in result["groups"]:
            if g["provider"] == "Anthropic":
                returned_ids = {m["id"] for m in g["models"]}
                overlap = raw_ids & returned_ids
                assert overlap, (
                    f"No _PROVIDER_MODELS fallback models found in Anthropic group. "
                    f"Expected subset of {raw_ids}, got {returned_ids}"
                )
                break

    def test_non_dict_models_value_falls_through_gracefully(self):
        """If models value is neither dict nor list (e.g. null), no crash."""
        cfg = {
            "model": {"provider": "anthropic"},
            "providers": {
                "anthropic": {"models": None},  # invalid — should not crash
            },
        }
        # Should not raise
        result = _available_models_with_cfg(cfg)
        assert "groups" in result