Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions hub/agents/gaia/python/gaia_agent/stdio.py
Original file line number Diff line number Diff line change
Expand Up @@ -636,10 +636,19 @@ def _apply_local_switch(agent: Any, target: str) -> str:
base_url=chat.config.base_url,
system_prompt=chat.config.system_prompt,
)
# Warm the catalog metadata now, before this client answers anything.
# cloud_model_provider() only recognises a runtime-discovered provider
# (an id outside the fireworks./amd. prefixes) once list_models() has
# read it — and both the switch message below and the system prompt
# _apply_switch is about to rebuild call it on this same client
# (#4365).
refresh = getattr(new_client, "refresh_model_catalog", None)
if callable(refresh):
refresh()
except Exception as exc: # pylint: disable=broad-exception-caught
# Reachability was already confirmed above — this covers whatever else
# LemonadeClient's constructor could still reject (a malformed
# base_url, mostly). Nothing on agent/chat has moved yet.
# LemonadeClient's constructor (or the catalog warm-up) could still
# reject. Nothing on agent/chat has moved yet.
raise RuntimeError(f"{type(exc).__name__}: {exc}") from exc

_apply_switch(
Expand Down Expand Up @@ -745,8 +754,16 @@ def run_model_command(agent: Any, query: str, out) -> None:

logger.info("switched model to %s (%s)", arg, display)
_write(_model_state_event(agent), out)
# Pass the live client's classifier, as the system prompt does — the id
# prefix knows only two providers, so a runtime-discovered one would be
# called local here while the prompt calls it cloud.
lookup = getattr(
getattr(agent.chat, "llm_client", None), "cloud_model_provider", None
)
location = resolve_inference_location(
agent.chat.effective_model, use_claude=bool(agent._use_claude)
agent.chat.effective_model,
use_claude=bool(agent._use_claude),
cloud_provider_lookup=lookup if callable(lookup) else None,
)
_write(
{
Expand Down
108 changes: 108 additions & 0 deletions hub/agents/gaia/python/tests/test_stdio.py
Original file line number Diff line number Diff line change
Expand Up @@ -1145,6 +1145,114 @@ def test_cloud_model_switch_preserves_session_and_reports_remote(
assert agent._use_claude is False


def test_switch_message_classifies_from_the_client_not_the_id_prefix(
monkeypatch, stub_lemonade
):
"""A cloud model whose id prefix is not a known provider must not be
announced as local.

The switchable-model filter calls ``cloud_model_provider(id, metadata)``,
which honours the catalog's own ``cloud_provider`` field — so a model can
be switchable under any prefix. The switch message called the one-argument
form, which knows only the ``fireworks.``/``amd.`` prefixes and returns
None for anything else, so this model was described with the LOCAL
sentence: an affirmative "not sending this conversation to any cloud
provider" about a cloud model, contradicting the system prompt, which does
consult the client.

The stub starts COLD, like the real ``LemonadeProvider`` create_client
just built: ``cloud_model_provider`` answers from empty metadata until
``refresh_model_catalog`` — the warm-up ``_apply_local_switch`` now runs
right after construction — has read the catalog. A stub that answers
correctly from birth (the pre-fix version of this test) can't fail even
when the production code forgets to warm the client.
"""
model = "custom.gemma-4-31b-it"
stub_lemonade.catalog = {
"data": [
{
"id": model,
"recipe": "cloud",
"cloud_provider": "fireworks",
"downloaded": False,
}
]
}

class _CatalogAwareClient:
"""Mirrors LemonadeProvider: classifies a discovered cloud model only
after refresh_model_catalog (== list_models) has run."""

def __init__(self):
self._metadata = {}
self.refresh_calls = 0

def refresh_model_catalog(self, show_all=True):
self.refresh_calls += 1
assert show_all is True, "labels/downloaded are only in the full catalog"
for entry in stub_lemonade.catalog.get("data", []):
if entry.get("id"):
self._metadata.setdefault(entry["id"], {}).update(entry)

def cloud_model_provider(self, model_id):
meta = self._metadata.get(model_id)
if meta is None:
return None
return meta.get("cloud_provider")

created = []

def _fake_create_client(**_kwargs):
client = _CatalogAwareClient()
created.append(client)
return client

monkeypatch.setattr(stdio, "create_client", _fake_create_client)

answer = _events(_model_run(_ModelSwitchAgent(), f"/model {model}"))[-1]["answer"]

assert created[0].refresh_calls == 1
assert "Fireworks AI" in answer
assert "a cloud provider" in answer
assert "not sending this conversation to any cloud provider" not in answer


def test_switch_message_falls_back_to_prefix_rule_without_refresh_model_catalog(
monkeypatch, stub_lemonade
):
"""A client with no ``refresh_model_catalog`` (a fake without the new
method, or a future non-Lemonade backend) must not crash the switch —
``_apply_local_switch`` just skips the warm-up, same as before this fix
existed, and the message falls back to the id-prefix rule."""
model = "custom.gemma-4-31b-it"
stub_lemonade.catalog = {
"data": [
{
"id": model,
"recipe": "cloud",
"cloud_provider": "fireworks",
"downloaded": False,
}
]
}

class _UnwarmableClient:
"""Has the classifier but no way to populate it — represents any
future backend that never learns catalog metadata."""

def cloud_model_provider(self, model_id):
return None

monkeypatch.setattr(stdio, "create_client", lambda **kwargs: _UnwarmableClient())

answer = _events(_model_run(_ModelSwitchAgent(), f"/model {model}"))[-1]["answer"]

# No catalog metadata available: falls back to the id-prefix rule, which
# does not know "custom." — described as local, the pre-existing (and
# documented) fallback behaviour.
assert "not sending this conversation to any cloud provider" in answer


def test_model_list_groups_discovered_cloud_without_downloads(stub_lemonade):
stub_lemonade.catalog = {
"data": [
Expand Down
16 changes: 15 additions & 1 deletion src/gaia/llm/providers/lemonade.py
Original file line number Diff line number Diff line change
Expand Up @@ -504,12 +504,26 @@ def cloud_model_provider(self, model: Optional[str] = None) -> Optional[str]:

Goes through the backend, which carries the catalog metadata, so a
provider discovered at runtime is recognised as well as the two whose
id prefixes are known up front.
id prefixes are known up front. Answers ``None`` for a runtime-
discovered provider until ``refresh_model_catalog`` (or any call that
reads ``list_models``) has populated that metadata at least once.
"""
return self._backend.cloud_model_provider(
model or self._last_model or self._model or DEFAULT_MODEL_NAME
)

def refresh_model_catalog(self, show_all: bool = True) -> None:
"""Read the Lemonade catalog once so ``cloud_model_provider`` can
recognise a runtime-discovered provider right away.

A freshly constructed client has no catalog metadata yet —
``cloud_model_provider`` only reads what ``list_models`` has cached —
so anything composed from it immediately after construction (a
switch message, a rebuilt system prompt) would call a runtime-
discovered cloud model local (#4365).
"""
self._backend.list_models(show_all=show_all)

def generate(
self,
prompt: str,
Expand Down
Loading