gooneyryan/hermes-webui
1
0
Fork 0
mirror of https://github.com/nesquena/hermes-webui.git synced 2026-05-15 03:36:59 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
hermes-webui/tests/test_model_resolver.py
T
Frank Song f6115b78c6 Fix custom provider name slugs with ports
2026-05-11 17:24:53 +08:00

629 lines
26 KiB
Python
Raw Permalink Blame History

"""
Tests for resolve_model_provider() model routing logic.
Verifies that model IDs are correctly resolved to (model, provider, base_url)
tuples for different provider configurations.
"""
import pytest
import api.config as config
def _resolve_with_config(model_id, provider=None, base_url=None, default=None, custom_providers=None):
"""Helper: temporarily set config.cfg model/custom provider sections, call resolve, restore."""
old_cfg = dict(config.cfg)
model_cfg = {}
if provider:
model_cfg['provider'] = provider
if base_url:
model_cfg['base_url'] = base_url
if default:
model_cfg['default'] = default
config.cfg['model'] = model_cfg if model_cfg else {}
if custom_providers is not None:
config.cfg['custom_providers'] = custom_providers
try:
return config.resolve_model_provider(model_id)
finally:
config.cfg.clear()
config.cfg.update(old_cfg)
# ── OpenRouter prefix handling ────────────────────────────────────────────
def test_openrouter_free_keeps_full_path():
"""openrouter/free must NOT be stripped to 'free' when provider is openrouter."""
model, provider, base_url = _resolve_with_config(
'openrouter/free', provider='openrouter',
base_url='https://openrouter.ai/api/v1',
)
assert model == 'openrouter/free', f"Expected 'openrouter/free', got '{model}'"
assert provider == 'openrouter'
def test_openrouter_model_with_provider_prefix():
"""anthropic/claude-sonnet-4.6 via openrouter keeps full path."""
model, provider, base_url = _resolve_with_config(
'anthropic/claude-sonnet-4.6', provider='openrouter',
base_url='https://openrouter.ai/api/v1',
)
assert model == 'anthropic/claude-sonnet-4.6'
assert provider == 'openrouter'
# ── Direct provider prefix stripping ─────────────────────────────────────
def test_anthropic_prefix_stripped_for_direct_api():
"""anthropic/claude-sonnet-4.6 strips prefix when provider is anthropic."""
model, provider, base_url = _resolve_with_config(
'anthropic/claude-sonnet-4.6', provider='anthropic',
)
assert model == 'claude-sonnet-4.6'
assert provider == 'anthropic'
def test_openai_prefix_stripped_for_direct_api():
"""openai/gpt-5.4-mini strips prefix when provider is openai."""
model, provider, base_url = _resolve_with_config(
'openai/gpt-5.4-mini', provider='openai',
)
assert model == 'gpt-5.4-mini'
assert provider == 'openai'
# ── Cross-provider routing ───────────────────────────────────────────────
def test_cross_provider_routes_through_openrouter():
"""Picking openai model when config is anthropic routes via openrouter."""
model, provider, base_url = _resolve_with_config(
'openai/gpt-5.4-mini', provider='anthropic',
)
assert model == 'openai/gpt-5.4-mini'
assert provider == 'openrouter'
assert base_url is None # openrouter uses its own endpoint
# ── Bare model names ─────────────────────────────────────────────────────
def test_bare_model_uses_config_provider():
"""A model name without / uses the config provider and base_url."""
model, provider, base_url = _resolve_with_config(
'gemma-4-26B', provider='custom',
base_url='http://192.168.1.160:4000',
)
assert model == 'gemma-4-26B'
assert provider == 'custom'
assert base_url == 'http://192.168.1.160:4000'
def test_empty_model_returns_config_defaults():
"""Empty model string returns config provider and base_url."""
model, provider, base_url = _resolve_with_config(
'', provider='anthropic',
)
assert model == ''
assert provider == 'anthropic'
# ── @provider:model hint routing (Issue #138 v2) ────────────────────────
def test_provider_hint_routes_to_specific_provider():
"""@minimax:MiniMax-M2.7 routes to minimax provider directly."""
model, provider, base_url = _resolve_with_config(
'@minimax:MiniMax-M2.7', provider='anthropic',
)
assert model == 'MiniMax-M2.7'
assert provider == 'minimax'
assert base_url is None # resolve_runtime_provider will fill this
def test_provider_hint_zai():
"""@zai:GLM-5 routes to zai provider directly."""
model, provider, base_url = _resolve_with_config(
'@zai:GLM-5', provider='openai',
)
assert model == 'GLM-5'
assert provider == 'zai'
def test_provider_hint_deepseek():
"""@deepseek:deepseek-chat routes to deepseek provider."""
model, provider, base_url = _resolve_with_config(
'@deepseek:deepseek-chat', provider='anthropic',
)
assert model == 'deepseek-chat'
assert provider == 'deepseek'
def test_slash_prefix_non_default_still_routes_openrouter():
"""minimax/MiniMax-M2.7 (old format) still routes through openrouter."""
model, provider, base_url = _resolve_with_config(
'minimax/MiniMax-M2.7', provider='anthropic',
)
assert model == 'minimax/MiniMax-M2.7'
assert provider == 'openrouter'
def test_custom_provider_model_with_slash_routes_to_named_custom_provider():
"""Slash-containing custom endpoint model IDs must not be mistaken for OpenRouter models."""
model, provider, base_url = _resolve_with_config(
'google/gemma-4-26b-a4b',
provider='openrouter',
base_url='https://openrouter.ai/api/v1',
custom_providers=[{
'name': 'Local LM Studio',
'base_url': 'http://lmstudio.local:1234/v1',
'model': 'google/gemma-4-26b-a4b',
}],
)
assert model == 'google/gemma-4-26b-a4b'
assert provider == 'custom:local-lm-studio'
assert base_url == 'http://lmstudio.local:1234/v1'
def test_custom_provider_models_dict_routes_to_named_custom_provider():
"""Models listed only under custom_providers[].models still route to that endpoint."""
model, provider, base_url = _resolve_with_config(
'sensenova-6.7-flash-lite',
provider='xiaomi',
custom_providers=[{
'name': 'LiteLLM Proxy',
'base_url': 'http://127.0.0.1:8080/v1',
'model': 'deepseek-v4-flash',
'models': {
'deepseek-v4-flash': {},
'sensenova-6.7-flash-lite': {},
},
}],
)
assert model == 'sensenova-6.7-flash-lite'
assert provider == 'custom:litellm-proxy'
assert base_url == 'http://127.0.0.1:8080/v1'
# ── Issue #2047: parenthesized local provider names with ports ────────────
def test_custom_provider_name_with_parenthesized_port_uses_safe_slug():
"""Setup-generated names like 'Local (host:port)' must not leak ':' into slugs."""
model, provider, base_url = _resolve_with_config(
'deepseek-v4-flash',
provider='custom',
custom_providers=[{
'name': 'Local (127.0.0.1:15721)',
'base_url': 'http://127.0.0.1:15721/v1',
'model': 'deepseek-v4-flash',
}],
)
assert model == 'deepseek-v4-flash'
assert provider == 'custom:local-127.0.0.1-15721'
assert base_url == 'http://127.0.0.1:15721/v1'
def test_safe_custom_provider_hint_keeps_model_after_port_slug():
"""The safe slug emitted by the picker must parse back without corrupting the model."""
model, provider, base_url = _resolve_with_config(
'@custom:local-127.0.0.1-15721:deepseek-v4-flash',
provider='custom',
)
assert model == 'deepseek-v4-flash'
assert provider == 'custom:local-127.0.0.1-15721'
assert base_url is None
# ── Issue #1922: default model shadowed by overlapping custom_providers[] ──
def test_default_model_not_shadowed_by_overlapping_custom_provider():
r'''Regression test for #1922.
When the active provider is an explicit non-custom provider (e.g. ai-gateway,
openrouter, xiaomi) AND the requested model_id matches the configured default
model, the active provider's base_url must take precedence over an overlapping
custom_providers[] entry. Otherwise the WebUI routes to 'custom:<name>' with
the wrong endpoint, causing 401 errors.
This test mirrors the reported scenario:
- provider: ai-gateway
- base_url: https://api.ai-gateway.example/v1
- default: gpt-5.4
- An overlapping custom_providers[] entry with the same default model
Expected: active provider (ai-gateway) wins over custom provider.
'''
model, provider, base_url = _resolve_with_config(
'gpt-5.4',
provider='ai-gateway',
base_url='https://api.ai-gateway.example/v1',
default='gpt-5.4',
custom_providers=[{
'name': 'My Custom Endpoint',
'base_url': 'http://localhost:8080/v1',
'model': 'gpt-5.4',
}],
)
assert model == 'gpt-5.4', f'Expected model=gpt-5.4, got {model!r}'
assert provider == 'ai-gateway', f'Expected provider=ai-gateway, got {provider!r}'
assert base_url == 'https://api.ai-gateway.example/v1', f'Expected base_url from active provider, got {base_url!r}'
def test_default_model_shadowed_with_xiaomi_provider():
r'''Same regression test with provider=xiaomi instead of ai-gateway.'''
model, provider, base_url = _resolve_with_config(
'deepseek-v4-flash',
provider='xiaomi',
default='deepseek-v4-flash',
custom_providers=[{
'name': 'LiteLLM Proxy',
'base_url': 'http://127.0.0.1:8080/v1',
'model': 'deepseek-v4-flash',
}],
)
assert model == 'deepseek-v4-flash'
assert provider == 'xiaomi'
assert base_url is None # xiaomi has no config base_url in this test
# ── get_available_models() @provider: hint behaviour ──────────────────────
@pytest.fixture(autouse=True)
def _isolate_models_cache():
"""Invalidate the models TTL cache before and after every test in this file.
Several helpers here mutate ``config.cfg`` in-memory and call
``get_available_models()``. Without this guard, a prior test that called
``get_available_models()`` leaves a 60-second TTL cache entry; the next
test that mutates cfg and calls the function gets a cache hit instead of
running the function body, causing silently wrong results (e.g. the
``test_custom_endpoint_uses_model_config_api_key_for_model_discovery``
``KeyError: 'auth'`` on CI where ``urlopen`` is never reached).
"""
try:
config.invalidate_models_cache()
except Exception:
pass
yield
try:
config.invalidate_models_cache()
except Exception:
pass
def _available_models_with_provider(provider):
"""Helper: temporarily set active_provider in config."""
old_cfg = dict(config.cfg)
config.cfg['model'] = {'provider': provider}
try:
return config.get_available_models()
finally:
config.cfg.clear()
config.cfg.update(old_cfg)
def test_non_default_provider_models_use_hint_prefix():
"""With anthropic as default, minimax model IDs should use @minimax: prefix."""
result = _available_models_with_provider('anthropic')
groups = {g['provider']: g['models'] for g in result['groups']}
if 'MiniMax' in groups:
for m in groups['MiniMax']:
assert m['id'].startswith('@minimax:'), (
f"Expected @minimax: prefix, got: {m['id']!r}"
)
def test_no_duplicate_when_default_model_is_prefixed():
"""Issue #147 Bug 2: 'anthropic/claude-opus-4.6' as default_model must not
inject a duplicate alongside the existing bare 'claude-opus-4.6' entry in
the same provider group."""
import api.config as _cfg
old_cfg = dict(_cfg.cfg)
_cfg.cfg['model'] = {
'provider': 'anthropic',
'default': 'anthropic/claude-opus-4.6',
}
try:
result = _cfg.get_available_models()
norm = lambda mid: mid.split('/', 1)[-1] if '/' in mid else mid
# Check each group individually: no group should have two entries that
# normalize to the same bare model name
for g in result['groups']:
bare_ids = [norm(m['id']) for m in g['models']]
duplicates = [mid for mid in set(bare_ids) if bare_ids.count(mid) > 1]
assert not duplicates, (
f"Provider group '{g['provider']}' has duplicate models after normalization: "
f"{duplicates}\nFull group: {[m['id'] for m in g['models']]}"
)
finally:
_cfg.cfg.clear()
_cfg.cfg.update(old_cfg)
def test_default_provider_models_not_prefixed(monkeypatch):
"""The active provider's models remain bare (no @prefix added)."""
import api.config as _cfg
monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda pid: ["claude-sonnet-5.0"] if pid == "anthropic" else [])
result = _available_models_with_provider('anthropic')
groups = {g['provider']: g['models'] for g in result['groups']}
if 'Anthropic' in groups:
returned_ids = {m['id'] for m in groups['Anthropic']}
assert "claude-sonnet-5.0" in returned_ids
assert not any(mid.startswith('@anthropic:') for mid in returned_ids), returned_ids
# ── get_available_models(): phantom "Custom" group regression ─────────────
#
# When the user has model.provider set to a real provider (e.g. openai-codex)
# AND a model.base_url set, hermes_cli reports the 'custom' pseudo-provider as
# authenticated. The WebUI picker must NOT build a separate "Custom" group in
# that case — the base_url belongs to the active provider.
def _available_models_with_full_cfg(provider, default, base_url):
"""Helper: set model.provider, model.default, model.base_url at once.
Clears model-override env vars (HERMES_MODEL, OPENAI_MODEL, LLM_MODEL)
during the call so the real hermes profile environment doesn't leak into
the test and override the fixture's default model.
"""
import os
import api.config as _cfg
old_cfg = dict(_cfg.cfg)
_cfg.cfg['model'] = {
'provider': provider,
'default': default,
'base_url': base_url,
}
try:
_cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime
except Exception:
# No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check
# inside get_available_models() sees 0.0 == 0.0 and doesn't call reload_config(),
# which would overwrite the in-memory cfg we just set up.
_cfg._cfg_mtime = 0.0
# Clear model-override env vars to prevent the real profile from leaking in
_model_env_keys = ('HERMES_MODEL', 'OPENAI_MODEL', 'LLM_MODEL')
_saved_env = {k: os.environ.pop(k, None) for k in _model_env_keys}
try:
return _cfg.get_available_models()
finally:
_cfg.cfg.clear()
_cfg.cfg.update(old_cfg)
for k, v in _saved_env.items():
if v is not None:
os.environ[k] = v
def test_no_phantom_custom_group_when_active_provider_is_set(monkeypatch):
"""Issue: with provider=openai-codex + base_url set, gpt-5.4 was landing
under a phantom "Custom" group instead of the "OpenAI Codex" group."""
import sys, types
# Force hermes_cli to report both the real provider and the phantom
# 'custom' as authenticated, simulating what list_available_providers()
# returns when base_url is configured.
fake_mod = types.ModuleType('hermes_cli.models')
fake_mod.list_available_providers = lambda: [
{'id': 'openai-codex', 'authenticated': True},
{'id': 'custom', 'authenticated': True},
]
fake_auth = types.ModuleType('hermes_cli.auth')
fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}
monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)
monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)
result = _available_models_with_full_cfg(
provider='openai-codex',
default='gpt-5.4',
base_url='https://chatgpt.com/backend-api/codex',
)
group_names = [g['provider'] for g in result['groups']]
assert 'Custom' not in group_names, (
f"Phantom 'Custom' group present; full groups: {group_names}"
)
def test_default_model_lands_under_active_provider_group(monkeypatch):
"""The configured default_model must appear under the active provider's
display group, even when the model isn't in _PROVIDER_MODELS[provider]
AND the active provider isn't the alphabetical first detected provider.
Regression guard for a hyphen-vs-space bug in the "ensure default_model
appears" post-pass: the substring check `active_provider.lower() in
g.get('provider', '').lower()` was failing for 'openai-codex' vs
display name 'OpenAI Codex' (hyphen vs. space), silently falling back
to groups[0] — which, when another provider sorted earlier
alphabetically (e.g. 'anthropic'), placed gpt-5.4 in the WRONG group.
"""
import sys, types
fake_mod = types.ModuleType('hermes_cli.models')
fake_mod.list_available_providers = lambda: [
{'id': 'anthropic', 'authenticated': True}, # sorts before openai-codex
{'id': 'openai-codex', 'authenticated': True},
{'id': 'custom', 'authenticated': True},
]
fake_auth = types.ModuleType('hermes_cli.auth')
fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}
monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)
monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)
result = _available_models_with_full_cfg(
provider='openai-codex',
default='gpt-5.4',
base_url='https://chatgpt.com/backend-api/codex',
)
groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}
assert 'OpenAI Codex' in groups, f"OpenAI Codex group missing: {list(groups)}"
norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1]
assert 'gpt-5.4' in {norm(mid) for mid in groups['OpenAI Codex']}, (
f"gpt-5.4 not in OpenAI Codex group; contents: {groups['OpenAI Codex']}"
)
# And crucially, it must NOT have landed in the alphabetically-first
# group (Anthropic) via the fallback path.
assert 'gpt-5.4' not in {norm(mid) for mid in groups.get('Anthropic', [])}, (
f"gpt-5.4 leaked into Anthropic group via fallback: {groups.get('Anthropic')}"
)
def test_unknown_providers_do_not_inherit_default_model(monkeypatch):
"""Detected providers without their own model catalog must not be filled
with the global default_model placeholder.
Regression guard for the bug where unknown providers ended up showing
gpt-5.4-mini even though those providers do not serve it. Minimax-Cn is
now known and should show its own catalog instead.
"""
import sys, types
fake_mod = types.ModuleType('hermes_cli.models')
fake_mod.list_available_providers = lambda: [
{'id': 'openai-codex', 'authenticated': True},
{'id': 'alibaba', 'authenticated': True},
{'id': 'minimax-cn', 'authenticated': True},
]
fake_auth = types.ModuleType('hermes_cli.auth')
fake_auth.get_auth_status = lambda pid: {'key_source': 'env'}
monkeypatch.setitem(sys.modules, 'hermes_cli.models', fake_mod)
monkeypatch.setitem(sys.modules, 'hermes_cli.auth', fake_auth)
result = _available_models_with_full_cfg(
provider='openai-codex',
default='gpt-5.4-mini',
base_url='',
)
groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}
norm = lambda mid: mid.split('/', 1)[-1].split(':', 1)[-1]
assert 'Alibaba' not in groups, (
f"Alibaba should not inherit the default model placeholder: {groups}"
)
assert 'MiniMax (China)' in groups, (
f"Minimax-Cn should render its own static catalog: {groups}"
)
assert not any(
norm(mid) == 'gpt-5.4-mini'
for mid in groups.get('Alibaba', []) + groups.get('MiniMax (China)', [])
), (
f"Unknown provider groups still inherited the default model: {groups}"
)
def test_custom_endpoint_uses_model_config_api_key_for_model_discovery(monkeypatch):
"""Custom endpoint model discovery must use model.api_key from config.yaml,
not only environment variables, otherwise the dropdown collapses to the
default model when /v1/models requires auth."""
import json as _json
import api.config as _cfg
old_cfg = dict(_cfg.cfg)
_cfg.cfg['model'] = {
'provider': 'custom',
'default': 'gpt-5.4',
'base_url': 'https://example.test/v1',
'api_key': 'sk-test-model-key',
}
try:
_cfg._cfg_mtime = _cfg.Path(_cfg._get_config_path()).stat().st_mtime
except Exception:
# No config.yaml on this machine (e.g. CI); pin to 0.0 so the mtime check
# inside get_available_models() sees 0.0 == 0.0 and skips reload_config().
_cfg._cfg_mtime = 0.0
_cfg.cfg.pop('providers', None)
captured = {}
class _Resp:
def read(self):
return _json.dumps({'data': [{'id': 'gpt-5.2', 'name': 'GPT-5.2'}]}).encode('utf-8')
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def _fake_urlopen(req, timeout=10):
url = getattr(req, 'full_url', '')
if 'example.test' in url:
captured['auth'] = req.get_header('Authorization')
captured['ua'] = req.get_header('User-agent')
return _Resp()
monkeypatch.setattr('urllib.request.urlopen', _fake_urlopen)
monkeypatch.setattr('socket.getaddrinfo', lambda *a, **k: [])
monkeypatch.delenv('OPENAI_API_KEY', raising=False)
monkeypatch.delenv('HERMES_API_KEY', raising=False)
monkeypatch.delenv('HERMES_OPENAI_API_KEY', raising=False)
monkeypatch.delenv('LOCAL_API_KEY', raising=False)
monkeypatch.delenv('OPENROUTER_API_KEY', raising=False)
monkeypatch.delenv('API_KEY', raising=False)
try:
result = _cfg.get_available_models()
finally:
_cfg.cfg.clear()
_cfg.cfg.update(old_cfg)
assert captured['auth'] == 'Bearer sk-test-model-key'
assert captured['ua'] == 'OpenAI/Python 1.0'
groups = {g['provider']: [m['id'] for m in g['models']] for g in result['groups']}
assert 'Custom' in groups
# Model ID may be prefixed with @provider: due to cross-provider dedup (#1228)
assert any('gpt-5.2' in m for m in groups['Custom']), f'gpt-5.2 not found in Custom: {groups}'
# -- Issue #230: custom provider with slash model name -----------------------
def test_custom_endpoint_slash_model_routes_to_custom_not_openrouter():
"""Regression test for #230, updated for #1625.
When provider=custom (or any non-openrouter provider) and base_url is set,
a model name containing a slash (e.g. google/gemma-4-26b-a4b) must NOT be
rerouted to OpenRouter -- it should stay on the configured custom endpoint.
#1625 layered an additional rule on top: a base_url pointing at a loopback
or private-IP host is treated as a local model server (LM Studio, Ollama,
llama.cpp, vLLM, TabbyAPI), which register models under their full
HuggingFace path. On such hosts the prefix is now PRESERVED. The original
#433 strip behaviour still applies on public hosts (real OpenAI-compatible
proxies like LiteLLM at https://litellm.example.com/v1).
"""
# --- custom provider with slash model name should NOT go to openrouter ---
model, provider, base_url = _resolve_with_config(
'google/gemma-4-26b-a4b',
provider='custom',
base_url='http://127.0.0.1:1234/v1',
default='google/gemma-4-26b-a4b',
)
assert provider.startswith('custom'), (
"Expected provider starting with 'custom', got '{}'. "
"Slash in model name should NOT trigger OpenRouter rerouting when base_url is set.".format(provider)
)
assert base_url == 'http://127.0.0.1:1234/v1', (
"Expected base_url 'http://127.0.0.1:1234/v1', got '{}'.".format(base_url)
)
# #1625 (supersedes the v0.50 #433 strip-on-custom rule for loopback hosts):
# 127.0.0.1 base_url is almost certainly a local LM Studio / Ollama / etc.,
# which keys models on the full HuggingFace path. Preserve the prefix.
assert model == 'google/gemma-4-26b-a4b', (
"Model name prefix must be PRESERVED on loopback base_url (#1625), got '{}'.".format(model)
)
# --- public-host openai-compatible proxy STILL strips per #433 ----------
model2, provider2, base_url2 = _resolve_with_config(
'google/gemma-4-26b-a4b',
provider='openai',
base_url='https://litellm.example.com/v1',
default='google/gemma-4-26b-a4b',
)
assert model2 == 'gemma-4-26b-a4b', (
"Public-host OpenAI-compat proxy must still strip prefix per #433, got '{}'.".format(model2)
)
# --- openrouter with slash model name MUST still route to openrouter -----
model_or, provider_or, _ = _resolve_with_config(
'google/gemma-4-26b-a4b',
provider='openrouter',
base_url='https://openrouter.ai/api/v1',
default='google/gemma-4-26b-a4b',
)
assert provider_or == 'openrouter', (
"Expected provider 'openrouter', got '{}'. "
"Slash model via openrouter provider must still resolve to openrouter.".format(provider_or)
)
assert model_or == 'google/gemma-4-26b-a4b', (
"Model name should be preserved for openrouter, got '{}'.".format(model_or)
)
Reference in New Issue View Git Blame Copy Permalink