diff --git a/webeval/src/webeval/oai_clients/__init__.py b/webeval/src/webeval/oai_clients/__init__.py index 0022fa7..962195a 100644 --- a/webeval/src/webeval/oai_clients/__init__.py +++ b/webeval/src/webeval/oai_clients/__init__.py @@ -1,4 +1,4 @@ -"""Chat completion clients used by webeval (OpenAI / Azure OpenAI / Azure ML).""" +"""Chat completion clients used by webeval (OpenAI / OrcaRouter / Azure OpenAI / Azure ML).""" from .create_utils import ( ENVIRON_KEY_CHAT_COMPLETION_KWARGS_JSON, @@ -32,6 +32,7 @@ ClientWrapper, ModelCapabilities, OpenAIClientWrapper, + OrcaRouterClientWrapper, ) __all__ = [ @@ -50,6 +51,7 @@ "LLMMessage", "ModelCapabilities", "OpenAIClientWrapper", + "OrcaRouterClientWrapper", "RequestUsage", "ResponsesGracefulRetryClient", "SystemMessage", diff --git a/webeval/src/webeval/oai_clients/create_utils.py b/webeval/src/webeval/oai_clients/create_utils.py index 3098639..9503645 100644 --- a/webeval/src/webeval/oai_clients/create_utils.py +++ b/webeval/src/webeval/oai_clients/create_utils.py @@ -14,6 +14,7 @@ AzureOpenAIResponsesWrapper, ChatCompletionClient, OpenAIClientWrapper, + OrcaRouterClientWrapper, ) ENVIRON_KEY_CHAT_COMPLETION_PROVIDER = "CHAT_COMPLETION_PROVIDER" @@ -32,8 +33,8 @@ def create_completion_client_from_env( ) -> ChatCompletionClient: """Construct a client from a config dict. - The dict must contain ``CHAT_COMPLETION_PROVIDER`` ("openai", "azure", - "trapi", "azure_ml", or "graceful_retry") and + The dict must contain ``CHAT_COMPLETION_PROVIDER`` ("openai", + "orcarouter", "azure", "trapi", "azure_ml", or "graceful_retry") and ``CHAT_COMPLETION_KWARGS_JSON`` (the kwargs forwarded to the underlying SDK client constructor). """ @@ -50,6 +51,9 @@ def create_completion_client_from_env( if _provider == "openai": _kwargs.pop("proxies", None) return OpenAIClientWrapper(**_kwargs) + if _provider == "orcarouter": + _kwargs.pop("proxies", None) + return OrcaRouterClientWrapper(**_kwargs) if _provider in ("azure", "trapi"): model = _kwargs.get("model", _kwargs.get("azure_deployment", "")) if "codex" in model or "o3-pro" in model or use_responses_api: diff --git a/webeval/src/webeval/oai_clients/wrapper.py b/webeval/src/webeval/oai_clients/wrapper.py index d7a2cc4..191e9c3 100644 --- a/webeval/src/webeval/oai_clients/wrapper.py +++ b/webeval/src/webeval/oai_clients/wrapper.py @@ -10,6 +10,7 @@ import json import logging import math +import os from dataclasses import dataclass from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Union @@ -293,6 +294,25 @@ async def create( ) +class OrcaRouterClientWrapper(OpenAIClientWrapper): + """OpenAI-compatible client for the OrcaRouter gateway. + + OrcaRouter exposes many upstream models behind a single OpenAI-compatible + endpoint (``https://api.orcarouter.ai/v1``) using ``vendor/model`` names, + plus adaptive routing via ``orcarouter/auto``. When no ``base_url`` or + ``api_key`` is supplied, it defaults to the public endpoint and the + ``ORCAROUTER_API_KEY`` environment variable. + """ + + DEFAULT_BASE_URL = "https://api.orcarouter.ai/v1" + + def __init__(self, **kwargs): + kwargs.setdefault("base_url", self.DEFAULT_BASE_URL) + kwargs.setdefault("api_key", os.environ.get("ORCAROUTER_API_KEY")) + super().__init__(**kwargs) + self.metadata = {"model": self.model, "provider": "orcarouter"} + + class AzureOpenAIClientWrapper(ChatCompletionClient): """Wrapper around Azure OpenAI Chat Completions.""" diff --git a/webeval/tests/test_oai_clients.py b/webeval/tests/test_oai_clients.py index c57b225..a88b172 100644 --- a/webeval/tests/test_oai_clients.py +++ b/webeval/tests/test_oai_clients.py @@ -269,6 +269,48 @@ async def fake_chat_create(**kwargs): assert captured["kwargs"]["model"] == "gpt-4o" +def test_orcarouter_provider_round_trip(monkeypatch): + """The ``orcarouter`` provider resolves to an OpenAI-compatible client + pointed at the OrcaRouter gateway and marks itself as orcarouter.""" + from webeval.oai_clients import ( + ChatCompletionClient, + OrcaRouterClientWrapper, + UserMessage, + create_client_from_config, + ) + + captured = {} + + async def fake_chat_create(**kwargs): + captured["kwargs"] = kwargs + message = SimpleNamespace(content="hello from orca", tool_calls=None) + choice = SimpleNamespace(message=message, finish_reason="stop") + usage = SimpleNamespace( + prompt_tokens=7, + completion_tokens=2, + completion_tokens_details=None, + ) + return SimpleNamespace(choices=[choice], usage=usage) + + client = create_client_from_config( + { + "CHAT_COMPLETION_PROVIDER": "orcarouter", + "CHAT_COMPLETION_KWARGS_JSON": {"model": "openai/gpt-4o"}, + } + ) + assert isinstance(client, OrcaRouterClientWrapper) + assert isinstance(client, ChatCompletionClient) + assert client.metadata["provider"] == "orcarouter" + assert str(client.client.base_url).rstrip("/") == "https://api.orcarouter.ai/v1" + + monkeypatch.setattr( + client.client.chat.completions, "create", fake_chat_create, raising=True + ) + result = asyncio.run(client.create(messages=[UserMessage(content="ping")])) + assert result.content == "hello from orca" + assert captured["kwargs"]["model"] == "openai/gpt-4o" + + def test_client_wrapper_from_config_returns_chat_client(monkeypatch): """Backwards-compat alias for callers that still use ``ClientWrapper.from_config``.""" from webeval.oai_clients import ChatCompletionClient, ClientWrapper