diff --git a/.env.example b/.env.example index d0ce43b..078d0bb 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,7 @@ JINA_KEY="your_jina_key" # https://jina.ai/ MINERU_TOKEN="your_mineru_token" # https://mineru.net/ # Optional +# JINA_READER_BASE_URL="http://127.0.0.1:3001" # Optional Reader-compatible endpoint; defaults to hosted https://r.jina.ai when unset. WORKSPACE_ROOT="./workspace" # Default local workspace root when --workspace-root is not provided. MAX_ROUNDS=500 # Maximum ReAct loop rounds before forced termination. MAX_RUNTIME_SECONDS=10800 # Maximum wall-clock runtime per agent run. diff --git a/README.md b/README.md index ddf6eee..2768ef2 100644 --- a/README.md +++ b/README.md @@ -324,12 +324,13 @@ used when they are exposed through a compatible endpoint. > **🚨 Required setup before running ResearchHarness** > > Fill in all required environment variables before starting the agent: -> `API_KEY`, `API_BASE`, `MODEL_NAME`, `SERPER_KEY`, `JINA_KEY`, and `MINERU_TOKEN`. +> `API_KEY`, `API_BASE`, `MODEL_NAME`, `SERPER_KEY`, and `MINERU_TOKEN`, plus `JINA_KEY` when using the default hosted Reader backend. > > Service key sources: > - LLM provider key and endpoint: your OpenAI-compatible provider. > - Serper API key for `WebSearch` and `ScholarSearch`: https://serper.dev/ -> - Jina API key for `WebFetch`: https://jina.ai/ +> - Jina API key for the default hosted `WebFetch` backend: https://jina.ai/ +> - Or set `JINA_READER_BASE_URL` to an absolute HTTP(S) Reader-compatible endpoint. Custom endpoints do not require or receive `JINA_KEY`. > - MinerU token plus [`structai`](https://github.com/black-yt/structai) for `ReadPDF`: https://mineru.net/ > > Before using ResearchHarness for real tasks, run the full tool availability check and require every tool to pass: @@ -347,12 +348,13 @@ Required variables: - `API_BASE` - `MODEL_NAME` - `SERPER_KEY` -- `JINA_KEY` +- `JINA_KEY` when `JINA_READER_BASE_URL` is unset and the default hosted Reader backend is used - `MINERU_TOKEN` Optional variables: - `WORKSPACE_ROOT` +- `JINA_READER_BASE_URL` (defaults to `https://r.jina.ai`; ResearchHarness does not start or manage a custom service) - `MAX_ROUNDS` - `MAX_RUNTIME_SECONDS` - `TIMEOUT_SECONDS` @@ -387,8 +389,12 @@ MODEL_NAME="gpt-5.5" SERPER_KEY="your_serper_key" JINA_KEY="your_jina_key" MINERU_TOKEN="your_mineru_token" +# JINA_READER_BASE_URL="http://127.0.0.1:3001" ``` +When `JINA_READER_BASE_URL` is configured, `WebFetch` never forwards `JINA_KEY` +to that endpoint and never falls back to hosted Jina if the custom service fails. + Sampling defaults and retry policy have code defaults, can be set with environment variables, and can be overridden per agent through the Python API. diff --git a/agent_base/tools/README.md b/agent_base/tools/README.md index ed898a0..abd0082 100644 --- a/agent_base/tools/README.md +++ b/agent_base/tools/README.md @@ -468,6 +468,10 @@ Arguments: Behavior: - Fetches page text through Jina Reader. +- Uses `https://r.jina.ai` by default and requires `JINA_KEY` for that hosted backend. +- Accepts an absolute HTTP(S) `JINA_READER_BASE_URL` for a custom Reader-compatible endpoint. +- Never sends `JINA_KEY` to a custom endpoint and never falls back to hosted Jina when that endpoint fails. +- Does not start or manage the configured Reader service. - Use multiple WebFetch calls when you need to inspect multiple URLs. - Applies simple deterministic cleanup to whitespace and blank lines. - Applies the requested line range and per-call character limit. @@ -475,7 +479,8 @@ Behavior: Dependencies: -- `JINA_KEY` +- `JINA_KEY` only for the default hosted backend +- optional `JINA_READER_BASE_URL` Returns: diff --git a/agent_base/tools/tool_web.py b/agent_base/tools/tool_web.py index 8496e7d..1612d99 100644 --- a/agent_base/tools/tool_web.py +++ b/agent_base/tools/tool_web.py @@ -4,6 +4,7 @@ import sys import time from typing import Optional, Union +from urllib.parse import urlparse import requests @@ -12,6 +13,8 @@ DEFAULT_WEBFETCH_TIMEOUT_SECONDS = 300.0 DEFAULT_WEBFETCH_MAX_CHARS = 16384 +DEFAULT_JINA_READER_BASE_URL = "https://r.jina.ai" +JINA_READER_BASE_URL_ENV = "JINA_READER_BASE_URL" def webfetch_timeout_seconds() -> float: @@ -28,6 +31,15 @@ def webfetch_default_max_chars() -> int: return max_chars +def jina_reader_base_url() -> str: + raw_value = os.getenv(JINA_READER_BASE_URL_ENV) + base_url = DEFAULT_JINA_READER_BASE_URL if raw_value is None else raw_value.strip().rstrip("/") + parsed = urlparse(base_url) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + raise ValueError("JINA_READER_BASE_URL must be an absolute HTTP(S) URL.") + return base_url + + def search_debug_enabled() -> bool: return env_flag("DEBUG_SEARCH") @@ -370,21 +382,25 @@ def call(self, params: Union[str, dict], **kwargs) -> str: def jina_readpage(self, url: str, runtime_deadline: Optional[float] = None) -> str: max_retries = 3 timeout = 50 + try: + reader_base_url = jina_reader_base_url() + except ValueError as exc: + return f"[WebFetch] {exc}" + + use_hosted_reader = reader_base_url == DEFAULT_JINA_READER_BASE_URL jina_api_key = os.getenv("JINA_KEY", "").strip() - if not jina_api_key: + if use_hosted_reader and not jina_api_key: return "[WebFetch] JINA_KEY is not set." + headers = {"Authorization": f"Bearer {jina_api_key}"} if use_hosted_reader else {} last_error = "unknown page-fetch error" for attempt in range(max_retries): - headers = { - "Authorization": f"Bearer {jina_api_key}", - } try: remaining = self._remaining_budget_seconds(runtime_deadline) if remaining is not None and remaining <= 0: return "[WebFetch] Failed to read page: agent runtime limit reached." response = requests.get( - f"https://r.jina.ai/{url}", + f"{reader_base_url}/{url}", headers=headers, timeout=min(timeout, max(remaining, 0.001)) if remaining is not None else timeout, ) diff --git a/agent_base/utils.py b/agent_base/utils.py index 7a0967c..6a4a55e 100644 --- a/agent_base/utils.py +++ b/agent_base/utils.py @@ -82,8 +82,12 @@ def env_flag(name: str) -> bool: return os.getenv(name, "").lower() in {"1", "true", "yes", "on"} -def missing_required_env(required: tuple[str, ...] = REQUIRED_ENV_VARS) -> list[str]: - return [key for key in required if not os.getenv(key, "").strip()] +def missing_required_env(required: Optional[tuple[str, ...]] = None) -> list[str]: + required_env = REQUIRED_ENV_VARS if required is None else required + missing = [key for key in required_env if not os.getenv(key, "").strip()] + if required is None and os.getenv("JINA_READER_BASE_URL", "").strip(): + missing = [key for key in missing if key != "JINA_KEY"] + return missing def require_required_env(context: str = "ResearchHarness") -> None: diff --git a/docs/tutorial_en.md b/docs/tutorial_en.md index 47fe6d7..ef18a69 100644 --- a/docs/tutorial_en.md +++ b/docs/tutorial_en.md @@ -99,13 +99,14 @@ Required variables: | `API_BASE` | Base URL for the OpenAI-compatible chat-completions endpoint. | | `MODEL_NAME` | Main model used by ResearchHarness. | | `SERPER_KEY` | Serper key for `WebSearch` and `ScholarSearch`: https://serper.dev/ | -| `JINA_KEY` | Jina key for `WebFetch`: https://jina.ai/ | +| `JINA_KEY` | Jina key for the default hosted `WebFetch` backend: https://jina.ai/. Not required when `JINA_READER_BASE_URL` is configured. | | `MINERU_TOKEN` | MinerU token for `ReadPDF`: https://mineru.net/ | Optional variables: | Variable | Default | Meaning | | --- | --- | --- | +| `JINA_READER_BASE_URL` | `https://r.jina.ai` | Absolute HTTP(S) Reader-compatible endpoint. Custom endpoints do not receive `JINA_KEY`; ResearchHarness does not manage the service or fall back to hosted Jina on failure. | | `WORKSPACE_ROOT` | `./workspace` | Default workspace root when no explicit workspace is passed. | | `MAX_ROUNDS` | `500` | Maximum ReAct loop rounds. | | `MAX_RUNTIME_SECONDS` | `10800` | Maximum wall-clock runtime for one agent run. | diff --git a/docs/tutorial_zh.md b/docs/tutorial_zh.md index f938966..4301dcf 100644 --- a/docs/tutorial_zh.md +++ b/docs/tutorial_zh.md @@ -96,13 +96,14 @@ pip install -e . --no-deps | `API_BASE` | OpenAI-compatible chat-completions endpoint 的 base URL。 | | `MODEL_NAME` | ResearchHarness 使用的主模型。 | | `SERPER_KEY` | `WebSearch` 和 `ScholarSearch` 使用的 Serper key：https://serper.dev/ | -| `JINA_KEY` | `WebFetch` 使用的 Jina key：https://jina.ai/ | +| `JINA_KEY` | 默认 hosted `WebFetch` 后端使用的 Jina key：https://jina.ai/。配置 `JINA_READER_BASE_URL` 时不需要。 | | `MINERU_TOKEN` | `ReadPDF` 使用的 MinerU token：https://mineru.net/ | 可选变量： | 变量 | 默认值 | 含义 | | --- | --- | --- | +| `JINA_READER_BASE_URL` | `https://r.jina.ai` | Reader-compatible 的绝对 HTTP(S) endpoint。custom endpoint 不会收到 `JINA_KEY`；ResearchHarness 不负责启动该服务，失败时也不会回退到 hosted Jina。 | | `WORKSPACE_ROOT` | `./workspace` | 未显式传入 workspace 时使用的默认 workspace root。 | | `MAX_ROUNDS` | `500` | ReAct loop 最大轮次。 | | `MAX_RUNTIME_SECONDS` | `10800` | 单次 agent run 的最大运行秒数。 | diff --git a/tests/test_edge_case_checks.py b/tests/test_edge_case_checks.py index a9c6628..ad1e72b 100644 --- a/tests/test_edge_case_checks.py +++ b/tests/test_edge_case_checks.py @@ -69,6 +69,34 @@ def check_required_env_is_enforced() -> tuple[bool, str]: os.environ[key] = value +def check_custom_reader_endpoint_makes_jina_key_optional() -> tuple[bool, str]: + from agent_base.utils import REQUIRED_ENV_VARS, missing_required_env + + env_names = (*REQUIRED_ENV_VARS, "JINA_READER_BASE_URL") + previous = {key: os.environ.get(key) for key in env_names} + try: + for key in REQUIRED_ENV_VARS: + os.environ[key] = f"test-{key.lower()}" + os.environ.pop("JINA_KEY", None) + os.environ.pop("JINA_READER_BASE_URL", None) + hosted_missing = missing_required_env() + + os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3001" + custom_missing = missing_required_env() + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + detail = json.dumps( + {"hosted_missing": hosted_missing, "custom_missing": custom_missing}, + indent=2, + ) + return hosted_missing == ["JINA_KEY"] and custom_missing == [], detail + + def check_llm_hard_timeout_interrupts_blocking_call() -> tuple[bool, str]: from agent_base.react_agent import LLMHardTimeoutError, llm_hard_timeout @@ -2358,6 +2386,101 @@ def fake_readpage(url, runtime_deadline=None): return ok, detail +def check_webfetch_reader_endpoint_selection_and_auth_isolation() -> tuple[bool, str]: + from agent_base.tools import tool_web + from agent_base.tools.tool_web import WebFetch + + saved = {name: os.environ.get(name) for name in ("JINA_KEY", "JINA_READER_BASE_URL")} + original_get = tool_web.requests.get + calls: list[dict[str, object]] = [] + + class FakeResponse: + def __init__(self, status_code: int, text: str): + self.status_code = status_code + self.text = text + + def fake_get(url, *, headers, timeout): + calls.append({"url": url, "headers": dict(headers), "timeout": timeout}) + if url.startswith("http://127.0.0.1:3002/"): + return FakeResponse(503, "local reader unavailable") + return FakeResponse(200, "reader response") + + try: + tool_web.requests.get = fake_get + os.environ.pop("JINA_KEY", None) + os.environ.pop("JINA_READER_BASE_URL", None) + missing_key = WebFetch().jina_readpage("https://example.com") + missing_key_calls = list(calls) + + os.environ["JINA_KEY"] = "hosted-secret" + hosted = WebFetch().jina_readpage("https://example.com/hosted") + hosted_call = calls[-1] + + os.environ.pop("JINA_KEY", None) + os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3001/" + custom_without_key = WebFetch().jina_readpage("https://example.com/custom") + custom_without_key_call = calls[-1] + + os.environ["JINA_KEY"] = "must-not-leak" + custom_with_key = WebFetch().jina_readpage("https://example.com/custom-key") + custom_with_key_call = calls[-1] + + os.environ["JINA_READER_BASE_URL"] = "file:///tmp/reader" + calls_before_invalid = len(calls) + invalid = WebFetch().jina_readpage("https://example.com/invalid") + invalid_made_request = len(calls) != calls_before_invalid + + os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3002" + calls_before_failure = len(calls) + custom_failure = WebFetch().jina_readpage("https://example.com/failure") + failure_calls = calls[calls_before_failure:] + finally: + tool_web.requests.get = original_get + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + detail = json.dumps( + { + "missing_key": missing_key, + "missing_key_calls": missing_key_calls, + "hosted": hosted, + "hosted_call": hosted_call, + "custom_without_key": custom_without_key, + "custom_without_key_call": custom_without_key_call, + "custom_with_key": custom_with_key, + "custom_with_key_call": custom_with_key_call, + "invalid": invalid, + "invalid_made_request": invalid_made_request, + "custom_failure": custom_failure, + "failure_calls": failure_calls, + }, + ensure_ascii=False, + indent=2, + ) + ok = ( + missing_key == "[WebFetch] JINA_KEY is not set." + and missing_key_calls == [] + and hosted == "reader response" + and hosted_call["url"] == "https://r.jina.ai/https://example.com/hosted" + and hosted_call["headers"] == {"Authorization": "Bearer hosted-secret"} + and custom_without_key == "reader response" + and custom_without_key_call["url"] == "http://127.0.0.1:3001/https://example.com/custom" + and custom_without_key_call["headers"] == {} + and custom_with_key == "reader response" + and custom_with_key_call["headers"] == {} + and invalid == "[WebFetch] JINA_READER_BASE_URL must be an absolute HTTP(S) URL." + and not invalid_made_request + and custom_failure.startswith("[WebFetch] Failed to read page: HTTP 503") + and len(failure_calls) == 3 + and all(call["url"].startswith("http://127.0.0.1:3002/") for call in failure_calls) + and all("r.jina.ai" not in str(call["url"]) for call in failure_calls) + ) + return ok, detail + + def check_session_state_is_only_written_with_trace_dir() -> tuple[bool, str]: from agent_base.react_agent import MultiTurnReactAgent @@ -2469,6 +2592,7 @@ def main() -> int: checks = [ ("Workspace root auto-create", check_workspace_root_is_created_when_missing), ("Required env enforced", check_required_env_is_enforced), + ("Custom Reader makes Jina key optional", check_custom_reader_endpoint_makes_jina_key_optional), ("ReadPDF relative image path", check_readpdf_relative_image_path), ("ReadPDF timeout result", check_readpdf_timeout_returns_tool_result), ("TerminalInterrupt remainder", check_terminal_interrupt_preserves_remainder), @@ -2495,6 +2619,7 @@ def main() -> int: ("Terminal error accepts artifact", check_terminal_error_can_be_accepted_after_completion_artifact), ("Claude runtime sampling params", check_claude_models_skip_sampling_params_in_agent_runtime), ("WebFetch range-bounded page text", check_webfetch_returns_range_bounded_page_text_without_summary_llm), + ("WebFetch Reader endpoint isolation", check_webfetch_reader_endpoint_selection_and_auth_isolation), ("Tool schema provider compatibility", check_tool_schemas_avoid_provider_incompatible_mixed_types), ("Plaintext result max rounds", check_plaintext_result_rejection_hits_max_rounds), ("Bash output bounding", check_bash_output_bounding_and_repeat_collapse),