feat(openrouter): trace GA and beta Responses APIs

py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py

@@ -29,4 +29,26 @@
     assert span["metadata"]["model"] == "gpt-4o-mini"
     assert "4" in span["output"][0]["message"]["content"]
 
+with autoinstrument_test_context(
+    "test_wrap_openrouter_beta_responses_send", integration="openrouter"
+) as memory_logger:
+    client = openrouter.OpenRouter(api_key=os.environ.get("OPENROUTER_API_KEY"))
+    response = client.beta.responses.send(
+        model="openai/gpt-4o-mini",
+        input="Say one short sentence about observability.",
+        max_output_tokens=64,
+        temperature=0,
+    )
+    assert response.output
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1, f"Expected 1 beta Responses span, got {len(spans)}"
+    span = spans[0]
+    assert span["span_attributes"]["name"] == "openrouter.beta.responses.send"
+    assert span["input"] == "Say one short sentence about observability."
+    assert span["metadata"]["model"] == "gpt-4o-mini"
+    assert span["metadata"]["provider"] == "openai"
+    assert span["output"][0]["type"] == "message"
+    assert span["metrics"]["tokens"] > 0
+
 print("SUCCESS")

py/src/braintrust/integrations/openrouter/integration.py

@@ -2,7 +2,7 @@
 
 from braintrust.integrations.base import BaseIntegration
 
-from .patchers import ChatPatcher, EmbeddingsPatcher, ResponsesPatcher
+from .patchers import BetaResponsesPatcher, ChatPatcher, EmbeddingsPatcher, ResponsesPatcher
 
 
 class OpenRouterIntegration(BaseIntegration):
@@ -15,4 +15,5 @@ class OpenRouterIntegration(BaseIntegration):
         ChatPatcher,
         EmbeddingsPatcher,
         ResponsesPatcher,
+        BetaResponsesPatcher,
     )

py/src/braintrust/integrations/openrouter/patchers.py

@@ -1,5 +1,7 @@
 """OpenRouter patchers."""
 
+from functools import partial
+
 from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher
 
 from .tracing import (
@@ -51,19 +53,38 @@ class EmbeddingsPatcher(CompositeFunctionWrapperPatcher):
 
 
 class ResponsesSendPatcher(FunctionWrapperPatcher):
-    name = "openrouter.beta.responses.send"
+    name = "openrouter.responses.send"
     target_module = "openrouter.responses"
     target_path = "Responses.send"
     wrapper = _responses_send_wrapper
 
 
 class ResponsesSendAsyncPatcher(FunctionWrapperPatcher):
-    name = "openrouter.beta.responses.send_async"
+    name = "openrouter.responses.send_async"
     target_module = "openrouter.responses"
     target_path = "Responses.send_async"
     wrapper = _responses_send_async_wrapper
 
 
 class ResponsesPatcher(CompositeFunctionWrapperPatcher):
-    name = "openrouter.beta.responses"
+    name = "openrouter.responses"
     sub_patchers = (ResponsesSendPatcher, ResponsesSendAsyncPatcher)
+
+
+class BetaResponsesSendPatcher(FunctionWrapperPatcher):
+    name = "openrouter.beta.responses.send"
+    target_module = "openrouter.beta_responses"
+    target_path = "BetaResponses.send"
+    wrapper = partial(_responses_send_wrapper, span_name="openrouter.beta.responses.send")
+
+
+class BetaResponsesSendAsyncPatcher(FunctionWrapperPatcher):
+    name = "openrouter.beta.responses.send_async"
+    target_module = "openrouter.beta_responses"
+    target_path = "BetaResponses.send_async"
+    wrapper = partial(_responses_send_async_wrapper, span_name="openrouter.beta.responses.send")
+
+
+class BetaResponsesPatcher(CompositeFunctionWrapperPatcher):
+    name = "openrouter.beta.responses"
+    sub_patchers = (BetaResponsesSendPatcher, BetaResponsesSendAsyncPatcher)

py/src/braintrust/integrations/openrouter/test_openrouter.py

@@ -153,7 +153,7 @@ def test_wrap_openrouter_embeddings_generate(memory_logger):
 
 
 @pytest.mark.vcr
-def test_wrap_openrouter_responses_send(memory_logger):
+def test_wrap_openrouter_beta_responses_send(memory_logger):
     assert not memory_logger.pop()
 
     client = wrap_openrouter(_get_client())
@@ -185,6 +185,34 @@ def test_wrap_openrouter_responses_send(memory_logger):
     assert_metrics_are_valid(span["metrics"], start, end)
 
 
+@pytest.mark.vcr
+def test_wrap_openrouter_responses_send(memory_logger):
+    client = _get_client()
+    if not hasattr(client, "responses"):
+        pytest.skip("GA Responses is unavailable in this OpenRouter version")
+
+    wrap_openrouter(client)
+    response = client.responses.send(
+        model=CHAT_MODEL,
+        input="Say one short sentence about observability.",
+        max_output_tokens=64,
+        temperature=0,
+    )
+    assert response.output
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+    assert span["span_attributes"]["name"] == "openrouter.responses.send"
+    assert span["span_attributes"]["type"] == "llm"
+    assert span["input"] == "Say one short sentence about observability."
+    assert span["metadata"]["model"] == "gpt-4o-mini"
+    assert span["metadata"]["provider"] == "openai"
+    assert span["output"][0]["type"] == "message"
+    assert span["metrics"]["tokens"] > 0
+    assert span["context"]["span_origin"]["instrumentation"]["name"] == "openrouter-auto"
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_wrap_openrouter_responses_send_async_stream(memory_logger):

py/src/braintrust/integrations/openrouter/tracing.py

@@ -1,5 +1,6 @@
 """OpenRouter-specific tracing helpers."""
 
+import importlib.util
 import logging
 import time
 from collections.abc import AsyncIterator, Iterator
@@ -625,9 +626,18 @@ async def _embeddings_generate_async_wrapper(wrapped, instance, args, kwargs):
     return result
 
 
-def _responses_send_wrapper(wrapped, instance, args, kwargs):
+def _responses_span_name(span_name: str | None) -> str:
+    if span_name is not None:
+        return span_name
+    # In older releases, openrouter.responses.Responses backs client.beta.responses.
+    if importlib.util.find_spec("openrouter.beta_responses") is None:
+        return "openrouter.beta.responses.send"
+    return "openrouter.responses.send"
+
+
+def _responses_send_wrapper(wrapped, instance, args, kwargs, *, span_name: str | None = None):
     request_metadata = _build_request_metadata(kwargs, _RESPONSES_REQUEST_KEYS)
-    span = _start_span("openrouter.beta.responses.send", kwargs.get("input"), request_metadata)
+    span = _start_span(_responses_span_name(span_name), kwargs.get("input"), request_metadata)
     start_time = time.time()
 
     try:
@@ -650,9 +660,9 @@ def _responses_send_wrapper(wrapped, instance, args, kwargs):
     return result
 
 
-async def _responses_send_async_wrapper(wrapped, instance, args, kwargs):
+async def _responses_send_async_wrapper(wrapped, instance, args, kwargs, *, span_name: str | None = None):
     request_metadata = _build_request_metadata(kwargs, _RESPONSES_REQUEST_KEYS)
-    span = _start_span("openrouter.beta.responses.send", kwargs.get("input"), request_metadata)
+    span = _start_span(_responses_span_name(span_name), kwargs.get("input"), request_metadata)
     start_time = time.time()
 
     try:
@@ -677,7 +687,7 @@ async def _responses_send_async_wrapper(wrapped, instance, args, kwargs):
 
 def wrap_openrouter(client: Any) -> Any:
     """Wrap a single OpenRouter client instance for tracing."""
-    from .patchers import ChatPatcher, EmbeddingsPatcher, ResponsesPatcher
+    from .patchers import BetaResponsesPatcher, ChatPatcher, EmbeddingsPatcher, ResponsesPatcher
 
     chat = getattr(client, "chat", None)
     if chat is not None:
@@ -687,9 +697,13 @@ def wrap_openrouter(client: Any) -> Any:
     if embeddings is not None:
         EmbeddingsPatcher.wrap_target(embeddings)
 
+    responses = getattr(client, "responses", None)
+    if responses is not None:
+        ResponsesPatcher.wrap_target(responses)
+
     beta = getattr(client, "beta", None)
     responses = getattr(beta, "responses", None) if beta is not None else None
     if responses is not None:
-        ResponsesPatcher.wrap_target(responses)
+        BetaResponsesPatcher.wrap_target(responses)
 
     return client