feat(webeval): add OrcaRouter as a first-class chat completion provider

webeval/src/webeval/oai_clients/__init__.py

@@ -1,4 +1,4 @@
-"""Chat completion clients used by webeval (OpenAI / Azure OpenAI / Azure ML)."""
+"""Chat completion clients used by webeval (OpenAI / OrcaRouter / Azure OpenAI / Azure ML)."""
 
 from .create_utils import (
     ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON,
@@ -32,6 +32,7 @@
     ClientWrapper,
     ModelCapabilities,
     OpenAIClientWrapper,
+    OrcaRouterClientWrapper,
 )
 
 __all__ = [
@@ -50,6 +51,7 @@
     "LLMMessage",
     "ModelCapabilities",
     "OpenAIClientWrapper",
+    "OrcaRouterClientWrapper",
     "RequestUsage",
     "ResponsesGracefulRetryClient",
     "SystemMessage",

webeval/src/webeval/oai_clients/create_utils.py

@@ -14,6 +14,7 @@
     AzureOpenAIResponsesWrapper,
     ChatCompletionClient,
     OpenAIClientWrapper,
+    OrcaRouterClientWrapper,
 )
 
 ENVIRON_KEY_CHAT_COMPLETION_PROVIDER = "CHAT_COMPLETION_PROVIDER"
@@ -32,8 +33,8 @@ def create_completion_client_from_env(
 ) -> ChatCompletionClient:
     """Construct a client from a config dict.
 
-    The dict must contain ``CHAT_COMPLETION_PROVIDER`` ("openai", "azure",
-    "trapi", "azure_ml", or "graceful_retry") and
+    The dict must contain ``CHAT_COMPLETION_PROVIDER`` ("openai",
+    "orcarouter", "azure", "trapi", "azure_ml", or "graceful_retry") and
     ``CHAT_COMPLETION_KWARGS_JSON`` (the kwargs forwarded to the
     underlying SDK client constructor).
     """
@@ -50,6 +51,9 @@ def create_completion_client_from_env(
     if _provider == "openai":
         _kwargs.pop("proxies", None)
         return OpenAIClientWrapper(**_kwargs)
+    if _provider == "orcarouter":
+        _kwargs.pop("proxies", None)
+        return OrcaRouterClientWrapper(**_kwargs)
     if _provider in ("azure", "trapi"):
         model = _kwargs.get("model", _kwargs.get("azure_deployment", ""))
         if "codex" in model or "o3-pro" in model or use_responses_api:

webeval/src/webeval/oai_clients/wrapper.py

@@ -10,6 +10,7 @@
 import json
 import logging
 import math
+import os
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Union
 
@@ -293,6 +294,25 @@ async def create(
         )
 
 
+class OrcaRouterClientWrapper(OpenAIClientWrapper):
+    """OpenAI-compatible client for the OrcaRouter gateway.
+
+    OrcaRouter exposes many upstream models behind a single OpenAI-compatible
+    endpoint (``https://api.orcarouter.ai/v1``) using ``vendor/model`` names,
+    plus adaptive routing via ``orcarouter/auto``. When no ``base_url`` or
+    ``api_key`` is supplied, it defaults to the public endpoint and the
+    ``ORCAROUTER_API_KEY`` environment variable.
+    """
+
+    DEFAULT_BASE_URL = "https://api.orcarouter.ai/v1"
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault("base_url", self.DEFAULT_BASE_URL)
+        kwargs.setdefault("api_key", os.environ.get("ORCAROUTER_API_KEY"))
+        super().__init__(**kwargs)
+        self.metadata = {"model": self.model, "provider": "orcarouter"}
+
+
 class AzureOpenAIClientWrapper(ChatCompletionClient):
     """Wrapper around Azure OpenAI Chat Completions."""
 

webeval/tests/test_oai_clients.py

@@ -269,6 +269,48 @@ async def fake_chat_create(**kwargs):
     assert captured["kwargs"]["model"] == "gpt-4o"
 
 
+def test_orcarouter_provider_round_trip(monkeypatch):
+    """The ``orcarouter`` provider resolves to an OpenAI-compatible client
+    pointed at the OrcaRouter gateway and marks itself as orcarouter."""
+    from webeval.oai_clients import (
+        ChatCompletionClient,
+        OrcaRouterClientWrapper,
+        UserMessage,
+        create_client_from_config,
+    )
+
+    captured = {}
+
+    async def fake_chat_create(**kwargs):
+        captured["kwargs"] = kwargs
+        message = SimpleNamespace(content="hello from orca", tool_calls=None)
+        choice = SimpleNamespace(message=message, finish_reason="stop")
+        usage = SimpleNamespace(
+            prompt_tokens=7,
+            completion_tokens=2,
+            completion_tokens_details=None,
+        )
+        return SimpleNamespace(choices=[choice], usage=usage)
+
+    client = create_client_from_config(
+        {
+            "CHAT_COMPLETION_PROVIDER": "orcarouter",
+            "CHAT_COMPLETION_KWARGS_JSON": {"model": "openai/gpt-4o"},
+        }
+    )
+    assert isinstance(client, OrcaRouterClientWrapper)
+    assert isinstance(client, ChatCompletionClient)
+    assert client.metadata["provider"] == "orcarouter"
+    assert str(client.client.base_url).rstrip("/") == "https://api.orcarouter.ai/v1"
+
+    monkeypatch.setattr(
+        client.client.chat.completions, "create", fake_chat_create, raising=True
+    )
+    result = asyncio.run(client.create(messages=[UserMessage(content="ping")]))
+    assert result.content == "hello from orca"
+    assert captured["kwargs"]["model"] == "openai/gpt-4o"
+
+
 def test_client_wrapper_from_config_returns_chat_client(monkeypatch):
     """Backwards-compat alias for callers that still use ``ClientWrapper.from_config``."""
     from webeval.oai_clients import ChatCompletionClient, ClientWrapper