hermes-webui/tests/test_provider_mismatch.py

"""
Tests for issue #266 — provider/model mismatch warning.

Covers:
  1. streaming.py: auth errors detected and classified as 'auth_mismatch'
  2. static/ui.js: _checkProviderMismatch() helper exists and logic is correct
  3. static/messages.js: apperror handler has auth_mismatch branch
  4. static/i18n.js: provider_mismatch_warning and provider_mismatch_label keys
     present in all locales (en, es, de, ru, zh, zh-Hant)
  5. static/boot.js: modelSelect.onchange calls _checkProviderMismatch
  6. /api/models: response includes active_provider field
"""
import json
import pathlib
import re
import urllib.request
from tests.conftest import TEST_STATE_DIR

REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve()
from tests._pytest_port import BASE


def _read(rel_path: str) -> str:
    return (REPO_ROOT / rel_path).read_text(encoding="utf-8")


def _post(path, body=None):
    data = json.dumps(body or {}).encode()
    req = urllib.request.Request(
        BASE + path, data=data, headers={"Content-Type": "application/json"}
    )
    with urllib.request.urlopen(req, timeout=10) as r:
        return json.loads(r.read()), r.status


# ── 1. streaming.py: auth error detection ───────────────────────────────────

class TestStreamingAuthErrorDetection:
    """streaming.py must classify auth/401 errors as auth_mismatch."""

    def test_auth_mismatch_type_defined_in_streaming(self):
        """'auth_mismatch' type must be emitted for auth errors."""
        src = _read("api/streaming.py")
        assert "auth_mismatch" in src, (
            "auth_mismatch type not found in streaming.py — "
            "401/auth errors will not be surfaced with a helpful message"
        )

    def test_is_auth_error_flag_defined(self):
        """auth error variable must exist in the error handler (exception path and silent-failure path)."""
        src = _read("api/streaming.py")
        # Variable renamed to _exc_is_auth in exception path, _is_auth in silent-failure path
        assert "_exc_is_auth" in src or "_is_auth" in src, (
            "auth error flag not found in streaming.py"
        )

    def test_auth_error_detects_401(self):
        """'401' must be part of the auth error detection logic."""
        src = _read("api/streaming.py")
        # Find the is_auth_error block
        # Variable renamed to _exc_is_auth in exception path, _is_auth in silent-failure path
        idx = src.find("_exc_is_auth")
        assert idx != -1
        block = src[idx:idx + 500]
        assert "'401'" in block or '"401"' in block, (
            "'401' not in auth error detection block"
        )

    def test_auth_error_detects_unauthorized(self):
        """'unauthorized' must be part of the auth error detection logic."""
        src = _read("api/streaming.py")
        # Variable renamed to _exc_is_auth in exception path
        idx = src.find("_exc_is_auth")
        block = src[idx:idx + 500]
        assert "unauthorized" in block.lower(), (
            "'unauthorized' not in auth error detection block"
        )

    def test_auth_error_hint_mentions_hermes_model(self):
        """The auth_mismatch hint must mention 'hermes model' command."""
        src = _read("api/streaming.py")
        # Find the auth_mismatch apperror block
        idx = src.find("auth_mismatch")
        block = src[idx:idx + 500]
        assert "hermes model" in block, (
            "auth_mismatch hint must mention 'hermes model' command "
            "so users know how to fix provider mismatch"
        )

    def test_auth_error_does_not_catch_rate_limit(self):
        """Rate limit errors must not be reclassified as auth_mismatch."""
        src = _read("api/streaming.py")
        # Variables renamed: _exc_is_rate_limit / _exc_is_auth in exception path
        # Quota check comes first (before rate limit), then rate limit, then auth
        rl_idx = src.find("_exc_is_rate_limit")
        ae_idx = src.find("_exc_is_auth")
        assert rl_idx != -1, "_exc_is_rate_limit not found in streaming.py exception path"
        assert ae_idx != -1, "_exc_is_auth not found in streaming.py exception path"
        assert rl_idx < ae_idx, (
            "_exc_is_rate_limit check should precede _exc_is_auth — "
            "rate limit errors must not be mistaken for auth errors"
        )


# ── 2. static/ui.js: _checkProviderMismatch() ───────────────────────────────

class TestCheckProviderMismatch:
    """ui.js must expose _checkProviderMismatch() helper."""

    def test_function_defined(self):
        """_checkProviderMismatch function must be defined in ui.js."""
        src = _read("static/ui.js")
        assert "function _checkProviderMismatch" in src, (
            "_checkProviderMismatch not defined in ui.js"
        )

    def test_uses_window_active_provider(self):
        """Function must read window._activeProvider."""
        src = _read("static/ui.js")
        idx = src.find("function _checkProviderMismatch")
        block = src[idx:idx + 800]
        assert "_activeProvider" in block, (
            "_checkProviderMismatch must read window._activeProvider"
        )

    def test_skips_check_for_openrouter(self):
        """OpenRouter can route to any provider — skip the warning."""
        src = _read("static/ui.js")
        idx = src.find("function _checkProviderMismatch")
        block = src[idx:idx + 800]
        assert "openrouter" in block.lower(), (
            "_checkProviderMismatch must skip the check for openrouter"
        )

    def test_skips_check_for_custom(self):
        """Custom endpoints can serve any model — skip the warning."""
        src = _read("static/ui.js")
        idx = src.find("function _checkProviderMismatch")
        block = src[idx:idx + 800]
        assert "custom" in block.lower(), (
            "_checkProviderMismatch must skip the check for custom provider"
        )

    def test_active_provider_stored_on_model_load(self):
        """populateModelDropdown must store active_provider from /api/models."""
        src = _read("static/ui.js")
        # Find the function definition (skip the comment that also mentions the name)
        idx = src.find("async function populateModelDropdown")
        assert idx != -1, "async function populateModelDropdown not found"
        block = src[idx:idx + 800]
        assert "_activeProvider" in block, (
            "populateModelDropdown must set window._activeProvider "
            "from the /api/models response"
        )


# ── 3. static/messages.js: apperror handler ─────────────────────────────────

class TestApperrorHandler:
    """messages.js apperror handler must handle auth_mismatch type."""

    def test_auth_mismatch_type_handled(self):
        """apperror handler must check for type='auth_mismatch'."""
        src = _read("static/messages.js")
        assert "auth_mismatch" in src, (
            "auth_mismatch type not handled in messages.js apperror handler"
        )

    def test_provider_mismatch_label(self):
        """'Provider mismatch' label must appear in the error handling."""
        src = _read("static/messages.js")
        assert "Provider mismatch" in src, (
            "'Provider mismatch' label not found in messages.js"
        )

    def test_is_auth_mismatch_variable(self):
        """isAuthMismatch variable must be defined."""
        src = _read("static/messages.js")
        assert "isAuthMismatch" in src, (
            "isAuthMismatch variable not found in messages.js apperror handler"
        )


# ── 4. static/i18n.js: all locales ───────────────────────────────────────────

class TestI18nProviderMismatch:
    """All locales must have provider_mismatch_warning and provider_mismatch_label."""

    REQUIRED_KEYS = ["provider_mismatch_warning", "provider_mismatch_label"]

    def _locale_names(self, src: str) -> list[str]:
        pattern = re.compile(
            r"^\s{2}(?:'(?P<quoted>[A-Za-z0-9-]+)'|(?P<plain>[A-Za-z0-9-]+))\s*:\s*\{",
            re.MULTILINE,
        )
        names = []
        for match in pattern.finditer(src):
            names.append(match.group("quoted") or match.group("plain"))
        return names

    def _count_key(self, src: str, key: str) -> int:
        return len(re.findall(r'\b' + re.escape(key) + r'\b', src))

    def test_all_locales_have_warning_key(self):
        """provider_mismatch_warning must appear in all locales."""
        src = _read("static/i18n.js")
        locale_count = len(self._locale_names(src))
        count = self._count_key(src, "provider_mismatch_warning")
        assert count >= locale_count, (
            f"provider_mismatch_warning found {count} times, expected >= {locale_count} "
            f"(one per locale)"
        )

    def test_all_locales_have_label_key(self):
        """provider_mismatch_label must appear in all locales."""
        src = _read("static/i18n.js")
        locale_count = len(self._locale_names(src))
        count = self._count_key(src, "provider_mismatch_label")
        assert count >= locale_count, (
            f"provider_mismatch_label found {count} times, expected >= {locale_count}"
        )

    def test_warning_is_function_in_en(self):
        """English provider_mismatch_warning must be a function (m, p) => ..."""
        src = _read("static/i18n.js")
        # Find the en block
        en_start = src.find("\n  en: {")
        es_start = src.find("\n  es: {")
        en_block = src[en_start:es_start]
        assert "provider_mismatch_warning" in en_block, "Key not in en block"
        idx = en_block.find("provider_mismatch_warning")
        line = en_block[idx:idx + 200]
        # Must be a function, not a plain string
        assert "=>" in line, (
            "provider_mismatch_warning in en locale must be an arrow function "
            "that takes (m, p) parameters for model and provider interpolation"
        )

    def test_spanish_locale_key_coverage(self):
        """Spanish locale must have the new keys (parity with English)."""
        src = _read("static/i18n.js")
        es_start = src.find("\n  es: {")
        de_start = src.find("\n  de: {")
        es_block = src[es_start:de_start]
        for key in self.REQUIRED_KEYS:
            assert key in es_block, f"Key '{key}' missing from Spanish locale"


# ── 5. static/boot.js: dropdown change handler ──────────────────────────────

class TestBootModelSelectChange:
    """boot.js modelSelect.onchange must call _checkProviderMismatch."""

    def test_onchange_calls_check_function(self):
        """modelSelect.onchange must invoke _checkProviderMismatch."""
        src = _read("static/boot.js")
        assert "_checkProviderMismatch" in src, (
            "boot.js modelSelect.onchange must call _checkProviderMismatch "
            "to warn users about provider/model mismatches"
        )
        # Verify it's called from the onchange handler (near modelSelect.onchange)
        idx = src.find("'modelSelect').onchange") or src.find('"modelSelect").onchange')
        if idx == -1:
            # Try alternate patterns
            idx = src.find("modelSelect")
        block_start = src.rfind("\n", 0, src.find("_checkProviderMismatch")) or 0
        surrounding = src[max(0, block_start - 200):block_start + 400]
        assert "modelSelect" in surrounding or "selectedModel" in surrounding, (
            "_checkProviderMismatch must be called in the context of model selection"
        )

    def test_onchange_shows_toast_on_mismatch(self):
        """The warning must be shown via showToast, not alert()."""
        src = _read("static/boot.js")
        # Both _checkProviderMismatch call and showToast must be near each other
        idx = src.find("_checkProviderMismatch")
        assert idx != -1, "_checkProviderMismatch not found in boot.js"
        block = src[idx:idx + 300]
        assert "showToast" in block, (
            "Provider mismatch warning must be shown via showToast(), not alert()"
        )


# ── 6. /api/models: active_provider in response ──────────────────────────────

def test_api_models_includes_active_provider():
    """/api/models must include 'active_provider' key in response."""
    with urllib.request.urlopen(BASE + "/api/models", timeout=10) as r:
        data = json.loads(r.read())
    # active_provider can be None/null but the key must exist
    assert "active_provider" in data, (
        "/api/models response missing 'active_provider' field — "
        "frontend needs this to detect provider mismatches"
    )


def test_codex_provider_qualified_model_routes_to_codex_not_openrouter():
    """@openai-codex:gpt-5.5 must route through OpenAI Codex, not OpenRouter."""
    import api.config as config

    old_cfg = dict(config.cfg)
    config.cfg["model"] = {
        "provider": "openrouter",
        "base_url": "https://openrouter.ai/api/v1",
    }
    try:
        model, provider, base_url = config.resolve_model_provider(
            "@openai-codex:gpt-5.5"
        )
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)

    assert model == "gpt-5.5"
    assert provider == "openai-codex"
    assert provider != "openrouter"
    assert base_url is None


def test_default_model_save_persists_codex_provider_for_qualified_model(tmp_path, monkeypatch):
    """Saving @openai-codex:gpt-5.5 must persist model.provider=openai-codex."""
    import yaml
    import api.config as config

    config_file = tmp_path / "config.yaml"
    config_file.write_text(
        "model:\n"
        "  provider: openrouter\n"
        "  default: openai/gpt-5.4\n"
        "  base_url: https://openrouter.ai/api/v1\n",
        encoding="utf-8",
    )
    old_cfg = dict(config.cfg)
    old_mtime = config._cfg_mtime
    monkeypatch.setattr(config, "_get_config_path", lambda: config_file)
    config.cfg["model"] = {
        "provider": "openrouter",
        "default": "openai/gpt-5.4",
        "base_url": "https://openrouter.ai/api/v1",
    }
    config._cfg_mtime = config_file.stat().st_mtime
    try:
        result = config.set_hermes_default_model("@openai-codex:gpt-5.5")
        saved = yaml.safe_load(config_file.read_text(encoding="utf-8"))
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)
        config._cfg_mtime = old_mtime
        config.invalidate_models_cache()

    assert result["ok"] is True
    assert result["model"] == "gpt-5.5"
    assert saved["model"]["default"] == "gpt-5.5"
    assert saved["model"]["provider"] == "openai-codex"
    assert saved["model"].get("base_url") != "https://openrouter.ai/api/v1"


def test_active_codex_at_provider_session_model_preserved(monkeypatch):
    """@openai-codex:gpt-5.5 session selections must keep their provider hint."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
                {
                    "provider": "OpenRouter",
                    "provider_id": "openrouter",
                    "models": [{"id": "openai/gpt-5.5", "label": "GPT-5.5"}],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@openai-codex:gpt-5.5"
    )

    assert changed is False
    assert effective == "@openai-codex:gpt-5.5"


def test_bare_codex_gpt_session_model_gets_separate_provider_context(monkeypatch):
    """A bare GPT model under active Codex stays bare and carries model_provider."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
                {
                    "provider": "OpenRouter",
                    "provider_id": "openrouter",
                    "models": [{"id": "openai/gpt-5.5", "label": "GPT-5.5"}],
                },
            ],
        },
    )

    effective, provider, changed = routes._resolve_compatible_session_model_state("gpt-5.5")

    assert changed is False
    assert effective == "gpt-5.5"
    assert provider == "openai-codex"


def test_session_model_normalizer_keeps_bare_codex_model_and_saves_provider(monkeypatch):
    """Write-path normalization must persist model_provider without adding @."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
            ],
        },
    )

    save_calls = []

    class DummySession:
        def __init__(self):
            self.model = "gpt-5.5"
            self.model_provider = None

        def save(self, touch_updated_at=True):
            save_calls.append(touch_updated_at)

    session = DummySession()
    effective = routes._normalize_session_model_in_place(session)

    assert effective == "gpt-5.5"
    assert session.model == "gpt-5.5"
    assert session.model_provider == "openai-codex"
    assert save_calls == [False]


def test_bare_codex_gpt_runtime_bridge_routes_to_codex(monkeypatch):
    """Bare model + model_provider=openai-codex must route Codex at runtime."""
    import api.config as config

    old_cfg = dict(config.cfg)
    config.cfg["model"] = {
        "provider": "openrouter",
        "default": "openai/gpt-5.4",
        "base_url": "https://openrouter.ai/api/v1",
    }
    try:
        runtime_model = config.model_with_provider_context(
            "gpt-5.5",
            "openai-codex",
        )
        model, provider, base_url = config.resolve_model_provider(runtime_model)
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)

    assert runtime_model == "@openai-codex:gpt-5.5"
    assert model == "gpt-5.5"
    assert provider == "openai-codex"
    assert base_url is None


def test_non_openrouter_slash_model_provider_context_stays_unqualified():
    """Portal/custom slash IDs must not be blindly wrapped as @provider:model."""
    import api.config as config

    runtime_model = config.model_with_provider_context(
        "anthropic/claude-sonnet-4.6",
        "nous",
    )

    assert runtime_model == "anthropic/claude-sonnet-4.6"


def test_api_session_new_persists_model_provider_context():
    """POST /api/session/new returns compact session model_provider metadata."""
    created, status = _post(
        "/api/session/new",
        {"model": "gpt-5.5", "model_provider": "openai-codex"},
    )

    assert status == 200
    assert created["session"]["model"] == "gpt-5.5"
    assert created["session"]["model_provider"] == "openai-codex"


def test_explicit_openrouter_selection_supported_with_codex_base_url():
    """OpenRouter slash and @openrouter selections must remain routable."""
    import api.config as config

    old_cfg = dict(config.cfg)
    config.cfg["model"] = {
        "provider": "openai-codex",
        "default": "gpt-5.5",
        "base_url": "https://chatgpt.com/backend-api/codex",
    }
    try:
        slash_model, slash_provider, slash_base_url = config.resolve_model_provider(
            "openai/gpt-5.5"
        )
        at_model, at_provider, at_base_url = config.resolve_model_provider(
            "@openrouter:openai/gpt-5.5"
        )
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)

    assert slash_model == "openai/gpt-5.5"
    assert slash_provider == "openrouter"
    assert slash_base_url is None
    assert at_model == "openai/gpt-5.5"
    assert at_provider == "openrouter"
    assert at_base_url is None


def test_real_provider_custom_base_url_slash_model_stays_on_configured_endpoint():
    """A real-provider proxy base_url must not be silently rerouted to OpenRouter."""
    import api.config as config

    old_cfg = dict(config.cfg)
    config.cfg["model"] = {
        "provider": "openai",
        "default": "google/gemma-4-26b-a4b",
        "base_url": "http://proxy.local/v1",
    }
    try:
        model, provider, base_url = config.resolve_model_provider(
            "google/gemma-4-26b-a4b"
        )
    finally:
        config.cfg.clear()
        config.cfg.update(old_cfg)

    assert model == "gemma-4-26b-a4b"
    assert provider == "openai"
    assert provider != "openrouter"
    assert base_url == "http://proxy.local/v1"


def test_bare_gemini_session_model_normalizes_to_active_provider_default(monkeypatch):
    """Persisted bare Gemini IDs must not survive a provider switch."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "gemini-3.1-pro-preview"
    )

    assert changed is True
    assert effective == "gpt-5.4-mini"


def test_prefixed_google_session_model_normalizes_to_active_provider_default(monkeypatch):
    """Persisted provider-prefixed Gemini IDs must normalize too."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "google/gemini-3.1-pro-preview"
    )

    assert changed is True
    assert effective == "gpt-5.4-mini"


def test_legacy_at_provider_session_model_normalizes_when_provider_hidden(monkeypatch):
    """Old @provider:model session values must not bypass stale-model recovery."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@copilot:gpt-5.5"
    )

    assert changed is True
    assert effective == "gpt-5.5"


def test_active_at_provider_session_model_preserved_with_hint(monkeypatch):
    """@active-provider:model must be preserved — stripping the prefix breaks duplicate-ID routing.

    Before #1253 was fixed, this path stripped the @provider: prefix and returned
    the bare model ID. That caused the picker to snap to the first matching provider
    (not the explicitly selected one) on the next send, and the agent to run on the
    wrong provider. The fix returns the full @provider:model unchanged so
    resolve_model_provider() can route through the correct provider.
    """
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.4-mini", "label": "GPT-5.4 Mini"}],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@openai-codex:gpt-5.4-mini"
    )

    # Must preserve the full @provider:model so resolve_model_provider() routes
    # through openai-codex, not through whatever provider happens to be first.
    assert changed is False
    assert effective == "@openai-codex:gpt-5.4-mini"


def test_routable_non_active_at_provider_session_model_is_preserved(monkeypatch):
    """Visible cross-provider dropdown selections must keep their provider hint."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
                {
                    "provider": "GitHub Copilot",
                    "provider_id": "copilot",
                    "models": [{"id": "@copilot:gpt-5.4", "label": "GPT-5.4"}],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@copilot:gpt-5.4"
    )

    assert changed is False
    assert effective == "@copilot:gpt-5.4"


def test_issue1253_duplicate_model_id_active_provider_hint_preserved(monkeypatch):
    """@provider:model where hint matches active provider must survive _resolve_compatible_session_model.

    Regression test for #1253: when two providers both expose the same bare model ID
    (e.g. both custom:edith and openai both expose 'gpt-5.4'), the picker stores the
    selection as @custom:gpt-5.4. On chat/start that value must be returned unchanged
    so resolve_model_provider() routes to 'custom', not to the default provider.

    Before the fix, hint_matches_active=True caused the prefix to be stripped:
      '@custom:gpt-5.4' → ('gpt-5.4', True)
    which then got written back to disk and sent as effective_model, snapping the
    picker to the first (wrong) provider.
    """
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "custom",
            "default_model": "gpt-5.4",
            "groups": [
                {
                    "provider": "Custom",
                    "provider_id": "custom",
                    "models": [{"id": "@custom:edith", "label": "Edith"}],
                },
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.4", "label": "GPT-5.4"}],
                },
            ],
        },
    )

    # User selected the custom:edith model — explicit @provider:model form.
    effective, changed = routes._resolve_compatible_session_model("@custom:edith")

    # Must NOT be stripped to 'edith' — that would route to the default provider.
    assert changed is False, (
        f"_resolve_compatible_session_model must not strip @custom:edith "
        f"(got effective='{effective}', changed={changed})"
    )
    assert effective == "@custom:edith", (
        f"expected '@custom:edith', got '{effective}'"
    )


def test_named_custom_provider_hint_with_colon_is_preserved(monkeypatch):
    """@custom:name:model must survive chat/start normalization for WebUI routing."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "deepseek",
            "default_model": "deepseek-v4-pro",
            "groups": [
                {
                    "provider": "sub2api",
                    "provider_id": "custom:sub2api",
                    "models": [
                        {
                            "id": "@custom:sub2api:gpt-5.4-mini",
                            "label": "GPT 5.4 Mini",
                        }
                    ],
                },
                {
                    "provider": "DeepSeek",
                    "provider_id": "deepseek",
                    "models": [
                        {
                            "id": "deepseek-v4-pro",
                            "label": "DeepSeek V4 Pro",
                        }
                    ],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@custom:sub2api:gpt-5.4-mini"
    )

    assert changed is False
    assert effective == "@custom:sub2api:gpt-5.4-mini"


def test_stale_at_provider_model_falls_back_when_family_mismatches(monkeypatch):
    """Unroutable @provider:model should not invent a bare model for another family."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.5",
            "groups": [
                {
                    "provider": "OpenAI Codex",
                    "provider_id": "openai-codex",
                    "models": [{"id": "gpt-5.5", "label": "GPT-5.5"}],
                },
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "@copilot:claude-opus-4.6"
    )

    assert changed is True
    assert effective == "gpt-5.5"


def test_google_active_provider_keeps_valid_gemini_session_model(monkeypatch):
    """A Google-configured session must keep its Gemini model."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "google",
            "default_model": "gemini-3.1-pro-preview",
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "gemini-3.1-pro-preview"
    )

    assert changed is False
    assert effective == "gemini-3.1-pro-preview"


def test_session_model_normalizer_persists_corrected_model(monkeypatch):
    """Write-path normalization should still persist corrected models."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    save_calls = []

    class DummySession:
        def __init__(self):
            self.model = "gemini-3.1-pro-preview"

        def save(self, touch_updated_at=True):
            save_calls.append(touch_updated_at)

    session = DummySession()
    effective = routes._normalize_session_model_in_place(session)

    assert effective == "gpt-5.4-mini"
    assert session.model == "gpt-5.4-mini"
    assert save_calls == [False]


def test_session_model_display_resolver_is_read_only(monkeypatch):
    """Read-path model resolution must not mutate or save the session."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    save_calls = []

    class DummySession:
        def __init__(self):
            self.model = "gemini-3.1-pro-preview"

        def save(self, touch_updated_at=True):
            save_calls.append(touch_updated_at)

    session = DummySession()
    effective = routes._resolve_effective_session_model_for_display(session)

    assert effective == "gpt-5.4-mini"
    assert session.model == "gemini-3.1-pro-preview"
    assert save_calls == []


def test_api_session_is_side_effect_free_for_stale_models():
    """GET /api/session must not rewrite the session file on first open (#845)."""
    created, status = _post("/api/session/new", {})
    assert status == 200
    sid = created["session"]["session_id"]

    session_path = TEST_STATE_DIR / "sessions" / f"{sid}.json"
    # POST /api/session/new no longer eagerly writes empty sessions to disk
    # (#1171 follow-up). Materialise the file from the API response so the
    # rest of this test, which checks that GET is side-effect-free against
    # an on-disk session with a stale model, has a file to work with.
    if not session_path.exists():
        session_path.parent.mkdir(parents=True, exist_ok=True)
        session_path.write_text(
            json.dumps(created["session"], ensure_ascii=False, indent=2),
            encoding="utf-8",
        )
    session_data = json.loads(session_path.read_text(encoding="utf-8"))
    stale_model = "google/gemini-3.1-pro-preview"
    session_data["model"] = stale_model
    before = json.dumps(session_data, ensure_ascii=False, indent=2)
    session_path.write_text(before, encoding="utf-8")

    with urllib.request.urlopen(
        BASE + f"/api/session?session_id={sid}", timeout=10
    ) as r:
        payload = json.loads(r.read())

    after = session_path.read_text(encoding="utf-8")
    assert payload["session"]["model"], "response should still expose an effective display model"
    assert payload["session"]["model"] != stale_model, (
        "response model should be compatibility-normalized on the read path"
    )
    assert after == before, (
        "GET /api/session must return an effective model for display without "
        "rewriting the session file on disk"
    )


# ── Model switch toast (#419) ─────────────────────────────────────────────────

class TestModelSwitchToast:
    """Toast appears when user switches the current conversation model."""

    def test_toast_in_model_select_onchange(self):
        """modelSelect.onchange must show a scope toast after selecting a model."""
        src = _read("static/boot.js")
        # Find the onchange block
        idx = src.find("modelSelect').onchange")
        assert idx != -1, "modelSelect.onchange not found in boot.js"
        block = src[idx:idx + 1100]
        assert "model_scope_toast" in block, (
            "modelSelect.onchange must show that the selected model applies to this conversation"
        )

    def test_toast_is_not_gated_on_messages_length(self):
        """Toast must fire for every model selection, not only sessions with messages."""
        src = _read("static/boot.js")
        idx = src.find("model_scope_toast")
        assert idx != -1
        surrounding = src[max(0, idx - 220):idx + 80]
        assert not ("S.messages" in surrounding and ".length" in surrounding), (
            "Model scope toast should not be gated on S.messages.length"
        )

    def test_toast_uses_show_toast_not_alert(self):
        """Toast must use showToast(), not alert()."""
        src = _read("static/boot.js")
        idx = src.find("model_scope_toast")
        assert idx != -1
        surrounding = src[max(0, idx - 50):idx + 100]
        assert "showToast" in surrounding, "Must use showToast() not alert()"
        assert "alert(" not in surrounding, "Must not use alert()"

    def test_toast_has_typeof_showtoast_guard(self):
        """Toast call must guard typeof showToast to be safe during boot."""
        src = _read("static/boot.js")
        idx = src.find("model_scope_toast")
        assert idx != -1
        surrounding = src[max(0, idx - 100):idx + 50]
        assert "typeof showToast" in surrounding, (
            "showToast call must be guarded with typeof check"
        )


class TestChatStartEffectiveModelRecovery:
    """messages.js must accept an effective_model correction from the backend."""

    def test_send_applies_effective_model_from_chat_start(self):
        src = _read("static/messages.js")
        assert "startData.effective_model" in src, (
            "send() must read effective_model from /api/chat/start so the UI can "
            "recover from stale persisted session models"
        )
        assert "localStorage.setItem('hermes-webui-model', startData.effective_model)" in src, (
            "effective_model correction must update the saved model preference"
        )
        assert "startData.effective_model_provider" in src, (
            "send() must preserve provider context returned by /api/chat/start"
        )


class TestFrontendModelProviderState:
    """Frontend model persistence should store provider separately."""

    def test_boot_session_update_sends_model_provider(self):
        src = _read("static/boot.js")
        assert "_modelStateForSelect" in src
        assert "model_provider:modelState.model_provider||null" in src

    def test_new_session_sends_model_provider(self):
        src = _read("static/sessions.js")
        assert "_modelStateForSelect(modelSel,selectedDefaultModel)" in src
        assert "model_provider:newModelState.model_provider||null" in src

    def test_ui_has_json_model_state_storage(self):
        src = _read("static/ui.js")
        assert "hermes-webui-model-state" in src
        assert "function _writePersistedModelState" in src
        assert "_providerQualifiedModelValueForSelect(sel, modelId)" in src
        assert "return _modelStateForSelect(sel,modelId).model" in src


def test_unknown_prefix_model_passes_through_unchanged(monkeypatch):
    """Models with unknown/custom prefixes must never be stripped — regression test for #751."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    for custom_model in (
        "custom-provider/test-model-999",
        "test/import-model",
        "my-local-llm/variant-1",
        "lmstudio-community/Qwen2.5-Coder-7B-Instruct-GGUF",
    ):
        effective, changed = routes._resolve_compatible_session_model(custom_model)
        assert changed is False, (
            f"Model '{custom_model}' has an unknown prefix and must pass through unchanged, "
            f"but _resolve_compatible_session_model returned changed=True (effective='{effective}')"
        )
        assert effective == custom_model, (
            f"Expected '{custom_model}', got '{effective}'"
        )


def test_empty_model_session_does_not_trigger_save(monkeypatch):
    """Sessions with no model stored must not trigger session.save() — index rebuild is expensive."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openai-codex",
            "default_model": "gpt-5.4-mini",
        },
    )

    save_calls = []

    class DummySession:
        def __init__(self):
            self.model = None  # no model stored

        def save(self, touch_updated_at=True):
            save_calls.append(touch_updated_at)

    session = DummySession()
    effective = routes._normalize_session_model_in_place(session)

    # Must return the default, but must NOT write to disk
    assert effective == "gpt-5.4-mini"
    assert save_calls == [], (
        "_normalize_session_model_in_place must not call session.save() when "
        "the session has no stored model — no correction needed, just a fallback."
    )


# ── Issue #829: stale cross-provider model on custom_providers-only setup ─────

def test_stale_openai_model_cleared_for_custom_only_provider(monkeypatch):
    """A stale openai/... session model must be cleared when active provider is
    'custom' and no catalog group can route the openai prefix (#829)."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "custom",
            "default_model": "",
            "groups": [
                {"provider": "Agent37", "provider_id": "custom:agent37",
                 "models": [{"id": "agent37/default", "label": "default"}]},
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "openai/gpt-5.4-mini"
    )

    # No routable group for openai/ — should clear to default (empty → model itself
    # only if no default available, which means changed=False when default_model="")
    # When default_model is empty, we can't clear — preserve and return False
    assert changed is False
    assert effective == "openai/gpt-5.4-mini"


def test_stale_openai_model_cleared_for_custom_provider_with_default(monkeypatch):
    """When active_provider='custom', no openrouter group, and default_model is
    configured, stale openai/... model should be cleared to default (#829)."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "custom",
            "default_model": "agent37/default",
            "groups": [
                {"provider": "Agent37", "provider_id": "custom:agent37",
                 "models": [{"id": "agent37/default", "label": "default"}]},
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "openai/gpt-5.4-mini"
    )

    assert changed is True
    assert effective == "agent37/default"


def test_openrouter_model_preserved_when_openrouter_group_present(monkeypatch):
    """When active_provider='openrouter' and openrouter group exists,
    openai/... model IDs must pass through unchanged — they are routable (#829)."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "openrouter",
            "default_model": "openai/gpt-5.4-mini",
            "groups": [
                {"provider": "OpenRouter", "provider_id": "openrouter",
                 "models": [{"id": "openai/gpt-5.4-mini", "label": "GPT-5.4 Mini"}]},
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "openai/gpt-5.4-mini"
    )

    assert changed is False
    assert effective == "openai/gpt-5.4-mini"


def test_custom_namespace_model_always_preserved_on_custom_provider(monkeypatch):
    """Model IDs with 'custom/' prefix must always pass through unchanged even
    when active_provider='custom' (#829)."""
    import api.routes as routes

    monkeypatch.setattr(
        routes,
        "get_available_models",
        lambda: {
            "active_provider": "custom",
            "default_model": "agent37/default",
            "groups": [
                {"provider": "Agent37", "provider_id": "custom:agent37",
                 "models": [{"id": "agent37/default", "label": "default"}]},
            ],
        },
    )

    effective, changed = routes._resolve_compatible_session_model(
        "custom/my-local-llm"
    )

    assert changed is False
    assert effective == "custom/my-local-llm"


def test_stale_ui_js_does_not_inject_unavailable_option():
    """renderSession() must no longer inject a bare (unavailable) option into
    modelSelect when the session model is not in the provider list (#829).
    It should silently reset to the first available model instead."""
    import os
    src = open(os.path.join(os.path.dirname(__file__), "..", "static", "ui.js"),
               encoding="utf-8").read()

    # The old pattern must be gone — both keys removed from ui.js
    assert "model_unavailable" not in src and "model_unavailable_title" not in src, (
        "renderSession() must not inject '(unavailable)' options — "
        "stale models should be silently reset to the first available model (#829)"
    )

    # The new silent-reset pattern must be present
    assert "first.value" in src and "S.session.model=first.value" in src, (
        "renderSession() must silently reset S.session.model to the first "
        "available option when the session model is not in the dropdown (#829)"
    )