diff --git a/.github/agent-server-openapi-weak-schema-allowlist.json b/.github/agent-server-openapi-weak-schema-allowlist.json index e63e342d22..5201424f0f 100644 --- a/.github/agent-server-openapi-weak-schema-allowlist.json +++ b/.github/agent-server-openapi-weak-schema-allowlist.json @@ -77,6 +77,12 @@ "reason": "Agent settings retain a backward-compatible extension surface while known security-sensitive fields remain typed.", "owner": "OpenHands OSS" }, + { + "pointer": "/components/schemas/AskOracleTool/properties/meta/anyOf/0/additionalProperties", + "kind": "unrestricted-additional-properties", + "reason": "Tool metadata is intentionally extensible across tool providers.", + "owner": "OpenHands" + }, { "pointer": "/components/schemas/BrowserClickTool/properties/meta/anyOf/0/additionalProperties", "kind": "unrestricted-additional-properties", diff --git a/examples/01_standalone_sdk/58_ask_oracle_tool/main.py b/examples/01_standalone_sdk/58_ask_oracle_tool/main.py new file mode 100644 index 0000000000..950a53a0af --- /dev/null +++ b/examples/01_standalone_sdk/58_ask_oracle_tool/main.py @@ -0,0 +1,78 @@ +"""Consult the Oracle end-to-end with the ask_oracle tool. + +The Oracle is a saved LLM profile resolved by convention under the name +``oracle``. This example wires two profiles — the agent's primary model and a +separate ``oracle`` model — adds ``Tool(name="ask_oracle")`` to the agent, then +drives a normal conversation: the agent decides to call ``ask_oracle``, the tool +consults the ``oracle`` profile, and the agent uses the Oracle's answer to reply. + +Usage: + LLM_API_KEY=... LLM_BASE_URL=https://llm-proxy.app.all-hands.dev \ + uv run python examples/01_standalone_sdk/58_ask_oracle_tool/main.py + +Note: + The example saves the ``oracle`` profile in a temporary directory so it + does not modify the user's default profile store. +""" + +import os +import tempfile + +from pydantic import SecretStr + +from openhands.sdk import LLM, Agent, LocalConversation, Tool +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.tools.ask_oracle import ORACLE_PROFILE_NAME + + +DEFAULT_BASE_URL = "https://llm-proxy.app.all-hands.dev" +# The agent's primary model (follows the standard LLM_MODEL env like other +# examples). The Oracle defaults to the same model; override ASK_ORACLE_MODEL to +# point the "oracle" profile at a different/stronger model. +PRIMARY_MODEL = os.getenv("ASK_ORACLE_PRIMARY_MODEL") or os.getenv( + "LLM_MODEL", "openai/gpt-5.5" +) +ORACLE_MODEL = os.getenv("ASK_ORACLE_MODEL", PRIMARY_MODEL) + +api_key = os.getenv("LLM_API_KEY") +assert api_key is not None, "LLM_API_KEY environment variable is not set." +base_url = os.getenv("LLM_BASE_URL", DEFAULT_BASE_URL) + +with tempfile.TemporaryDirectory() as profile_store_dir: + store = LLMProfileStore(profile_store_dir) + # The Oracle model is saved under the conventional profile name "oracle". + store.save( + ORACLE_PROFILE_NAME, + LLM( + model=ORACLE_MODEL, + api_key=SecretStr(api_key), + base_url=base_url, + usage_id="oracle", + ), + include_secrets=True, + ) + + primary_llm = LLM( + model=PRIMARY_MODEL, + api_key=SecretStr(api_key), + base_url=base_url, + usage_id="primary", + ) + agent = Agent(llm=primary_llm, tools=[Tool(name="ask_oracle")]) + conversation = LocalConversation( + agent=agent, + workspace=os.getcwd(), + profile_store_dir=profile_store_dir, + ) + + print(f"Primary model: {conversation.agent.llm.model}") + print(f"Oracle model: {ORACLE_MODEL}") + conversation.send_message( + "Call the oracle to ask it for its opinion on the weather today, " + "then just tell me in two words how it's like." + ) + conversation.run() + + combined = conversation.state.stats.get_combined_metrics() + print(f"Total cost: ${combined.accumulated_cost:.6f}") + print(f"EXAMPLE_COST: {combined.accumulated_cost}") diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index 65545b8818..c5146c19d3 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -237,6 +237,7 @@ def __init__( prompt_cache_key: str | None = None, file_store: FileStore | None = None, mcp_tool_provider: MCPToolProvider | None = None, + profile_store_dir: str | Path | None = None, **_: object, ): """Initialize the conversation. @@ -295,6 +296,8 @@ def __init__( file_store: Optional FileStore to use for conversation state and EventLog persistence. If provided, this takes precedence over persistence_dir for state and EventLog storage. + profile_store_dir: Optional directory containing saved LLM profiles. + Defaults to ``~/.openhands/profiles``. """ super().__init__() # Initialize with span tracking # Mark cleanup as initiated as early as possible to avoid races or partially @@ -482,7 +485,7 @@ def _default_callback(e): # Agent initialization is deferred to _ensure_agent_ready() for lazy loading # This ensures plugins are loaded before agent initialization self.llm_registry = LLMRegistry() - self._profile_store = LLMProfileStore() + self._profile_store = LLMProfileStore(profile_store_dir) self._cipher = cipher # Seed agent_context.secrets into the registry for every agent (regular diff --git a/openhands-sdk/openhands/sdk/tool/builtins/__init__.py b/openhands-sdk/openhands/sdk/tool/builtins/__init__.py index 11ec4b42d0..6e88318483 100644 --- a/openhands-sdk/openhands/sdk/tool/builtins/__init__.py +++ b/openhands-sdk/openhands/sdk/tool/builtins/__init__.py @@ -1,8 +1,9 @@ -"""Implementing essential tools that doesn't interact with the environment. +"""SDK-resident tools that do not interact with the environment. -These are built in and are *required* for the agent to work. +`BUILT_IN_TOOLS` contains tools attached to every agent. `BUILT_IN_TOOL_CLASSES` +also includes optional SDK tools that are resolved by name from agent setup. -For tools that require interacting with the environment, add them to `openhands-tools`. +Tools that require interacting with the environment belong in `openhands-tools`. """ from openhands.sdk.tool.builtins.finish import ( diff --git a/openhands-tools/openhands/tools/__init__.py b/openhands-tools/openhands/tools/__init__.py index 62f77a5ee7..ed0b02ac8e 100644 --- a/openhands-tools/openhands/tools/__init__.py +++ b/openhands-tools/openhands/tools/__init__.py @@ -18,6 +18,7 @@ from importlib.metadata import PackageNotFoundError, version +from openhands.tools.ask_oracle import AskOracleTool from openhands.tools.delegate import DelegationVisualizer from openhands.tools.file_editor import FileEditorTool from openhands.tools.preset.default import ( @@ -40,6 +41,7 @@ __all__ = [ "__version__", + "AskOracleTool", "DelegationVisualizer", "FileEditorTool", "TaskToolSet", diff --git a/openhands-tools/openhands/tools/ask_oracle/__init__.py b/openhands-tools/openhands/tools/ask_oracle/__init__.py new file mode 100644 index 0000000000..ad9577a62a --- /dev/null +++ b/openhands-tools/openhands/tools/ask_oracle/__init__.py @@ -0,0 +1,36 @@ +"""Ask-Oracle tool package. + +Provides a read-only ``ask_oracle`` tool: a one-shot, tool-less sub-agent that +consults a stronger/more-capable LLM for a second opinion. The Oracle model is a +saved LLM profile, resolved by convention under the name ``oracle`` (see +``ORACLE_PROFILE_NAME``). + +Usage: + from openhands.tools.ask_oracle import AskOracleTool + + agent = Agent( + llm=llm, + tools=[Tool(name=AskOracleTool.name)], + ) + +The agent's active conversation LLM is never switched. The Oracle call sends only +the Oracle system prompt plus the agent's question and optional context, without +forwarding conversation history or tools. +""" + +from openhands.tools.ask_oracle.definition import ( + ORACLE_PROFILE_NAME, + AskOracleAction, + AskOracleObservation, + AskOracleTool, +) +from openhands.tools.ask_oracle.impl import AskOracleExecutor + + +__all__ = [ + "ORACLE_PROFILE_NAME", + "AskOracleAction", + "AskOracleObservation", + "AskOracleExecutor", + "AskOracleTool", +] diff --git a/openhands-tools/openhands/tools/ask_oracle/definition.py b/openhands-tools/openhands/tools/ask_oracle/definition.py new file mode 100644 index 0000000000..c6a472e641 --- /dev/null +++ b/openhands-tools/openhands/tools/ask_oracle/definition.py @@ -0,0 +1,114 @@ +"""Action, observation, and tool definitions for the ask_oracle tool.""" + +from collections.abc import Sequence +from typing import TYPE_CHECKING, Final, Self + +from pydantic import Field +from rich.text import Text + +from openhands.sdk.tool.registry import register_tool +from openhands.sdk.tool.tool import ( + Action, + Observation, + ToolAnnotations, + ToolDefinition, +) + + +if TYPE_CHECKING: + from openhands.sdk.conversation.state import ConversationState + + +# The Oracle model is a saved LLM profile resolved by convention under this +# name. Save a profile named "oracle" (e.g. via LLMProfileStore.save("oracle", +# llm)) and the tool will consult it. No agent setting or wiring is required. +ORACLE_PROFILE_NAME: Final[str] = "oracle" + + +class AskOracleAction(Action): + """Action for asking the Oracle for advice.""" + + question: str = Field( + description=( + "The specific question or dilemma to ask the Oracle about. Use this " + "when you are stuck, uncertain, or need a second opinion." + ) + ) + context: str | None = Field( + default=None, + description=( + "Optional extra context, such as approaches already tried, constraints, " + "or the recommendation you are considering." + ), + ) + + @property + def visualize(self) -> Text: + content = Text() + content.append("Ask Oracle: ", style="bold cyan") + content.append(self.question) + if self.context: + content.append("\nContext: ", style="bold") + content.append(self.context) + return content + + +class AskOracleObservation(Observation): + """Observation returned by the Oracle consultation.""" + + @property + def visualize(self) -> Text: + content = Text() + if self.is_error: + content.append("Oracle consultation failed", style="bold red") + else: + content.append("Oracle recommendation", style="bold green") + if self.text: + content.append("\n") + content.append(self.text) + return content + + +_DESCRIPTION = ( + "Ask the Oracle for a second opinion. The Oracle is a smart model intended " + "to help with difficult reasoning.\n\n" + "Use this when you are stuck, uncertain, comparing approaches, or need a " + "higher-quality recommendation before proceeding.\n\n" + "Treat the Oracle's response as strong guidance and follow its recommendation " + "unless you have a clear reason not to." +) + + +class AskOracleTool(ToolDefinition[AskOracleAction, AskOracleObservation]): + """Tool for consulting the Oracle (a saved LLM profile named "oracle").""" + + @classmethod + def create( + cls, + conv_state: "ConversationState | None" = None, # noqa: ARG003 + **params, + ) -> Sequence[Self]: + if params: + raise ValueError("AskOracleTool does not accept parameters") + + # Import here to keep module import light and avoid any import cycles. + from openhands.tools.ask_oracle.impl import AskOracleExecutor + + return [ + cls( + description=_DESCRIPTION, + action_type=AskOracleAction, + observation_type=AskOracleObservation, + executor=AskOracleExecutor(), + annotations=ToolAnnotations( + readOnlyHint=True, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + ) + ] + + +# Automatically register when this module is imported. +register_tool(AskOracleTool.name, AskOracleTool) diff --git a/openhands-tools/openhands/tools/ask_oracle/impl.py b/openhands-tools/openhands/tools/ask_oracle/impl.py new file mode 100644 index 0000000000..83817541f5 --- /dev/null +++ b/openhands-tools/openhands/tools/ask_oracle/impl.py @@ -0,0 +1,120 @@ +"""Executor for the ask_oracle tool.""" + +from typing import TYPE_CHECKING + +from openhands.sdk.agent.utils import make_llm_completion +from openhands.sdk.llm import Message, TextContent +from openhands.sdk.tool.tool import ToolExecutor +from openhands.tools.ask_oracle.definition import ( + ORACLE_PROFILE_NAME, + AskOracleAction, + AskOracleObservation, +) + + +if TYPE_CHECKING: + from openhands.sdk.conversation.impl.local_conversation import LocalConversation + + +ORACLE_USAGE_PREFIX = "oracle" + + +_ORACLE_SYSTEM_PROMPT = """\ +You are the Oracle: a highly capable reviewer giving a second opinion to an \ +OpenHands agent. + +Answer the agent's question directly. Do not call tools. Do not perform work \ +directly. Give a concrete recommendation the agent can follow, including important \ +risks or caveats.""" + +_ORACLE_USER_PROMPT_TEMPLATE = """\ +Question: +{question} +{context_section}""" + + +class AskOracleExecutor(ToolExecutor[AskOracleAction, AskOracleObservation]): + """Consult the Oracle: a saved LLM profile named ``oracle``. + + The Oracle is resolved by convention from the conversation's LLM profile + store under the name ``oracle`` (``ORACLE_PROFILE_NAME``). The call is + stateless: it sends only the Oracle system prompt plus the agent's question + and optional context, with no conversation history and no tools, and returns + the Oracle's text. The active conversation LLM is never switched. + """ + + def __call__( + self, + action: AskOracleAction, + conversation: "LocalConversation | None" = None, + ) -> AskOracleObservation: + if conversation is None: + return AskOracleObservation.from_text( + text="Cannot ask the Oracle without an active conversation.", + is_error=True, + ) + + try: + oracle_llm = conversation.get_or_create_profile_llm( + profile_name=ORACLE_PROFILE_NAME, + usage_id=f"{ORACLE_USAGE_PREFIX}:{ORACLE_PROFILE_NAME}", + ) + except FileNotFoundError: + return AskOracleObservation.from_text( + text=( + "The Oracle is not available because no profile named " + f"'{ORACLE_PROFILE_NAME}' was found. Save one to enable it." + ), + is_error=True, + ) + except ValueError as exc: + return AskOracleObservation.from_text( + text=f"The Oracle is not available: {exc}", + is_error=True, + ) + except Exception as exc: + return AskOracleObservation.from_text( + text=f"The Oracle is not available: {type(exc).__name__}: {exc}", + is_error=True, + ) + + context_section = ( + f"\nAdditional context from the agent:\n{action.context}\n" + if action.context + else "" + ) + user_prompt = _ORACLE_USER_PROMPT_TEMPLATE.format( + question=action.question, + context_section=context_section, + ) + messages = [ + Message( + role="system", + content=[TextContent(text=_ORACLE_SYSTEM_PROMPT)], + ), + Message(role="user", content=[TextContent(text=user_prompt)]), + ] + + try: + llm_response = make_llm_completion(oracle_llm, messages) + except Exception as exc: + return AskOracleObservation.from_text( + text=( + "The Oracle encountered an error and did not return a " + f"response: {type(exc).__name__}: {exc}" + ), + is_error=True, + ) + + oracle_text = "".join( + content.text + for content in llm_response.message.content + if isinstance(content, TextContent) + ).strip() + if not oracle_text: + return AskOracleObservation.from_text( + text="The Oracle did not return a response.", + is_error=True, + ) + + return AskOracleObservation.from_text(text=oracle_text) diff --git a/tests/examples/test_examples.py b/tests/examples/test_examples.py index 533a999fb3..1ea26c1ddd 100644 --- a/tests/examples/test_examples.py +++ b/tests/examples/test_examples.py @@ -30,6 +30,7 @@ EXAMPLES_ROOT / "01_standalone_sdk" / "37_llm_profile_store", EXAMPLES_ROOT / "01_standalone_sdk" / "51_agent_hooks", EXAMPLES_ROOT / "01_standalone_sdk" / "57_prompt_hooks", + EXAMPLES_ROOT / "01_standalone_sdk" / "58_ask_oracle_tool", EXAMPLES_ROOT / "02_remote_agent_server" / "06_custom_tool", EXAMPLES_ROOT / "05_skills_and_plugins" / "01_loading_agentskills", EXAMPLES_ROOT / "05_skills_and_plugins" / "02_loading_plugins", @@ -105,6 +106,9 @@ def test_directory_example_is_discovered() -> None: assert ( EXAMPLES_ROOT / "01_standalone_sdk" / "57_prompt_hooks" / "main.py" ) in EXAMPLES + assert ( + EXAMPLES_ROOT / "01_standalone_sdk" / "58_ask_oracle_tool" / "main.py" + ) in EXAMPLES assert ( EXAMPLES_ROOT / "05_skills_and_plugins" diff --git a/tests/sdk/conversation/test_switch_model.py b/tests/sdk/conversation/test_switch_model.py index 17386b5924..a63af511e0 100644 --- a/tests/sdk/conversation/test_switch_model.py +++ b/tests/sdk/conversation/test_switch_model.py @@ -52,13 +52,14 @@ def profile_store(tmp_path, monkeypatch): return store -def _make_conversation() -> LocalConversation: +def _make_conversation(profile_store_dir: Path | None = None) -> LocalConversation: return LocalConversation( agent=Agent( llm=_make_llm("default-model", "test-llm"), tools=[], ), workspace=Path.cwd(), + profile_store_dir=profile_store_dir, ) @@ -267,6 +268,17 @@ def test_switch_profile(profile_store): assert conv.agent.llm.model == "slow-model" +def test_switch_profile_uses_custom_profile_store(tmp_path: Path) -> None: + profile_dir = tmp_path / "profiles" + store = LLMProfileStore(profile_dir) + store.save("fast", _make_llm("fast-model", "fast")) + + conv = _make_conversation(profile_store_dir=profile_dir) + conv.switch_profile("fast") + + assert conv.agent.llm.model == "fast-model" + + def test_switch_profile_updates_state(profile_store): """switch_profile updates conversation state agent.""" conv = _make_conversation() diff --git a/tests/tools/ask_oracle/test_ask_oracle.py b/tests/tools/ask_oracle/test_ask_oracle.py new file mode 100644 index 0000000000..9857282562 --- /dev/null +++ b/tests/tools/ask_oracle/test_ask_oracle.py @@ -0,0 +1,218 @@ +from collections.abc import Sequence +from pathlib import Path +from typing import Any, cast + +import pytest +from pydantic import PrivateAttr + +from openhands.sdk import LLM, LocalConversation, Tool +from openhands.sdk.agent import Agent +from openhands.sdk.llm import ( + LLMResponse, + Message, + TextContent, + TokenCallbackType, + llm_profile_store, +) +from openhands.sdk.llm.llm import LLMCallContext +from openhands.sdk.testing import TestLLM +from openhands.sdk.tool import ToolDefinition +from openhands.tools.ask_oracle import ( + ORACLE_PROFILE_NAME, + AskOracleAction, + AskOracleObservation, + AskOracleTool, +) +from openhands.tools.ask_oracle.impl import AskOracleExecutor + + +class CapturingTestLLM(TestLLM): + _last_messages: list[Message] = PrivateAttr(default_factory=list) + _last_tools: Sequence[ToolDefinition] | None = PrivateAttr(default=None) + + @property + def last_messages(self) -> list[Message]: + return self._last_messages + + @property + def last_tools(self) -> Sequence[ToolDefinition] | None: + return self._last_tools + + def completion( + self, + messages: list[Message], + tools: Sequence[ToolDefinition] | None = None, + add_security_risk_prediction: bool = False, + on_token: TokenCallbackType | None = None, + call_context: LLMCallContext | None = None, + **kwargs: Any, + ) -> LLMResponse: + self._last_messages = list(messages) + self._last_tools = tools + return super().completion( + messages=messages, + tools=tools, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + call_context=call_context, + **kwargs, + ) + + +def _make_llm(model: str, usage_id: str) -> LLM: + return TestLLM.from_messages([], model=model, usage_id=usage_id) + + +def _assistant_message(text: str) -> Message: + return Message(role="assistant", content=[TextContent(text=text)]) + + +def _message_text(message: Message) -> str: + return "".join( + content.text for content in message.content if isinstance(content, TextContent) + ) + + +def _make_conversation() -> LocalConversation: + return LocalConversation( + agent=Agent( + llm=_make_llm("default-model", "default"), + tools=[Tool(name=AskOracleTool.name)], + include_default_tools=[], + ), + workspace=Path.cwd(), + ) + + +def test_ask_oracle_tool_description_guides_second_opinion_usage() -> None: + tool = AskOracleTool.create()[0] + + assert "Ask the Oracle for a second opinion" in tool.description + assert "Treat the Oracle's response as strong guidance" in tool.description + assert tool.annotations is not None + assert tool.annotations.openWorldHint + + +def test_ask_oracle_tool_rejects_parameters() -> None: + with pytest.raises(ValueError, match="does not accept parameters"): + AskOracleTool.create(profile_name="custom") + + +def test_ask_oracle_tool_added_by_name() -> None: + agent = Agent( + llm=_make_llm("default-model", "default"), + tools=[Tool(name=AskOracleTool.name)], + include_default_tools=[], + ) + conversation = LocalConversation(agent=agent, workspace=Path.cwd()) + conversation._ensure_agent_ready() + # Plugin loading replaces conversation.agent with an initialized copy, so + # assert on the live agent rather than the now-stale local reference. + assert "ask_oracle" in conversation.agent.tools_map + + +def test_ask_oracle_tool_requires_active_conversation() -> None: + observation = AskOracleExecutor()( + AskOracleAction(question="What should I do next?") + ) + + assert observation.is_error + assert observation.text == "Cannot ask the Oracle without an active conversation." + + +def test_ask_oracle_tool_returns_oracle_recommendation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + oracle_llm = cast( + CapturingTestLLM, + CapturingTestLLM.from_messages( + [_assistant_message("Prefer the smaller, typed settings field.")], + model="oracle-model", + usage_id="oracle", + ), + ) + + def load_profile( + self: LocalConversation, + profile_name: str, + usage_id: str, + ) -> LLM: + assert profile_name == ORACLE_PROFILE_NAME + assert usage_id == f"oracle:{ORACLE_PROFILE_NAME}" + return oracle_llm + + monkeypatch.setattr(LocalConversation, "get_or_create_profile_llm", load_profile) + conversation = _make_conversation() + + observation = conversation.execute_tool( + "ask_oracle", + AskOracleAction( + question="Should I add one setting or two?", + context="The tool needs an Oracle profile name.", + ), + ) + + assert isinstance(observation, AskOracleObservation) + assert not observation.is_error + assert observation.text == "Prefer the smaller, typed settings field." + assert "Prefer the smaller" in observation.visualize.plain + assert [message.role for message in oracle_llm.last_messages] == ["system", "user"] + assert "You are the Oracle" in _message_text(oracle_llm.last_messages[0]) + assert "Should I add one setting or two?" in _message_text( + oracle_llm.last_messages[1] + ) + assert "The tool needs an Oracle profile name." in _message_text( + oracle_llm.last_messages[1] + ) + assert oracle_llm.last_tools == [] + assert conversation.agent.llm.model == "default-model" + assert conversation.state.agent.llm.model == "default-model" + + +def test_ask_oracle_tool_reports_missing_profile( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + profile_dir = tmp_path / "profiles" + profile_dir.mkdir() + + monkeypatch.setattr(llm_profile_store, "_DEFAULT_PROFILE_DIR", profile_dir) + conversation = _make_conversation() + + observation = conversation.execute_tool( + "ask_oracle", + AskOracleAction(question="What should I do next?"), + ) + + assert isinstance(observation, AskOracleObservation) + assert observation.is_error + assert "not available" in observation.text + assert ORACLE_PROFILE_NAME in observation.text + + +def test_ask_oracle_tool_reports_empty_oracle_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + oracle_llm = TestLLM.from_messages( + [Message(role="assistant", content=[])], + model="oracle-model", + usage_id="oracle", + ) + + def load_profile( + self: LocalConversation, + profile_name: str, + usage_id: str, + ) -> LLM: + return oracle_llm + + monkeypatch.setattr(LocalConversation, "get_or_create_profile_llm", load_profile) + conversation = _make_conversation() + + observation = conversation.execute_tool( + "ask_oracle", + AskOracleAction(question="What should I do next?"), + ) + + assert isinstance(observation, AskOracleObservation) + assert observation.is_error + assert "did not return a response" in observation.text