feat: support configurable Jina Reader endpoints

.env.example

@@ -7,6 +7,7 @@ JINA_KEY="your_jina_key"                                 # https://jina.ai/
 MINERU_TOKEN="your_mineru_token"                         # https://mineru.net/
 
 # Optional
+# JINA_READER_BASE_URL="http://127.0.0.1:3001"          # Optional Reader-compatible endpoint; defaults to hosted https://r.jina.ai when unset.
 WORKSPACE_ROOT="./workspace"                             # Default local workspace root when --workspace-root is not provided.
 MAX_ROUNDS=500                                           # Maximum ReAct loop rounds before forced termination.
 MAX_RUNTIME_SECONDS=10800                                # Maximum wall-clock runtime per agent run.

README.md

@@ -324,12 +324,13 @@ used when they are exposed through a compatible endpoint.
 > **🚨 Required setup before running ResearchHarness**
 >
 > Fill in all required environment variables before starting the agent:
-> `API_KEY`, `API_BASE`, `MODEL_NAME`, `SERPER_KEY`, `JINA_KEY`, and `MINERU_TOKEN`.
+> `API_KEY`, `API_BASE`, `MODEL_NAME`, `SERPER_KEY`, and `MINERU_TOKEN`, plus `JINA_KEY` when using the default hosted Reader backend.
 >
 > Service key sources:
 > - LLM provider key and endpoint: your OpenAI-compatible provider.
 > - Serper API key for `WebSearch` and `ScholarSearch`: https://serper.dev/
-> - Jina API key for `WebFetch`: https://jina.ai/
+> - Jina API key for the default hosted `WebFetch` backend: https://jina.ai/
+> - Or set `JINA_READER_BASE_URL` to an absolute HTTP(S) Reader-compatible endpoint. Custom endpoints do not require or receive `JINA_KEY`.
 > - MinerU token plus [`structai`](https://github.com/black-yt/structai) for `ReadPDF`: https://mineru.net/
 >
 > Before using ResearchHarness for real tasks, run the full tool availability check and require every tool to pass:
@@ -347,12 +348,13 @@ Required variables:
 - `API_BASE`
 - `MODEL_NAME`
 - `SERPER_KEY`
-- `JINA_KEY`
+- `JINA_KEY` when `JINA_READER_BASE_URL` is unset and the default hosted Reader backend is used
 - `MINERU_TOKEN`
 
 Optional variables:
 
 - `WORKSPACE_ROOT`
+- `JINA_READER_BASE_URL` (defaults to `https://r.jina.ai`; ResearchHarness does not start or manage a custom service)
 - `MAX_ROUNDS`
 - `MAX_RUNTIME_SECONDS`
 - `TIMEOUT_SECONDS`
@@ -387,8 +389,12 @@ MODEL_NAME="gpt-5.5"
 SERPER_KEY="your_serper_key"
 JINA_KEY="your_jina_key"
 MINERU_TOKEN="your_mineru_token"
+# JINA_READER_BASE_URL="http://127.0.0.1:3001"
 ```
 
+When `JINA_READER_BASE_URL` is configured, `WebFetch` never forwards `JINA_KEY`
+to that endpoint and never falls back to hosted Jina if the custom service fails.
+
 Sampling defaults and retry policy have code defaults, can be set with
 environment variables, and can be overridden per agent through the Python API.
 

agent_base/tools/README.md

@@ -468,14 +468,19 @@ Arguments:
 Behavior:
 
 - Fetches page text through Jina Reader.
+- Uses `https://r.jina.ai` by default and requires `JINA_KEY` for that hosted backend.
+- Accepts an absolute HTTP(S) `JINA_READER_BASE_URL` for a custom Reader-compatible endpoint.
+- Never sends `JINA_KEY` to a custom endpoint and never falls back to hosted Jina when that endpoint fails.
+- Does not start or manage the configured Reader service.
 - Use multiple WebFetch calls when you need to inspect multiple URLs.
 - Applies simple deterministic cleanup to whitespace and blank lines.
 - Applies the requested line range and per-call character limit.
 - Does not call an LLM inside the tool; the main agent is responsible for reading, reasoning over, and summarizing the returned content.
 
 Dependencies:
 
-- `JINA_KEY`
+- `JINA_KEY` only for the default hosted backend
+- optional `JINA_READER_BASE_URL`
 
 Returns:
 

agent_base/tools/tool_web.py

@@ -4,6 +4,7 @@
 import sys
 import time
 from typing import Optional, Union
+from urllib.parse import urlparse
 
 import requests
 
@@ -12,6 +13,8 @@
 
 DEFAULT_WEBFETCH_TIMEOUT_SECONDS = 300.0
 DEFAULT_WEBFETCH_MAX_CHARS = 16384
+DEFAULT_JINA_READER_BASE_URL = "https://r.jina.ai"
+JINA_READER_BASE_URL_ENV = "JINA_READER_BASE_URL"
 
 
 def webfetch_timeout_seconds() -> float:
@@ -28,6 +31,15 @@ def webfetch_default_max_chars() -> int:
     return max_chars
 
 
+def jina_reader_base_url() -> str:
+    raw_value = os.getenv(JINA_READER_BASE_URL_ENV)
+    base_url = DEFAULT_JINA_READER_BASE_URL if raw_value is None else raw_value.strip().rstrip("/")
+    parsed = urlparse(base_url)
+    if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("JINA_READER_BASE_URL must be an absolute HTTP(S) URL.")
+    return base_url
+
+
 def search_debug_enabled() -> bool:
     return env_flag("DEBUG_SEARCH")
 
@@ -370,21 +382,25 @@ def call(self, params: Union[str, dict], **kwargs) -> str:
     def jina_readpage(self, url: str, runtime_deadline: Optional[float] = None) -> str:
         max_retries = 3
         timeout = 50
+        try:
+            reader_base_url = jina_reader_base_url()
+        except ValueError as exc:
+            return f"[WebFetch] {exc}"
+
+        use_hosted_reader = reader_base_url == DEFAULT_JINA_READER_BASE_URL
         jina_api_key = os.getenv("JINA_KEY", "").strip()
-        if not jina_api_key:
+        if use_hosted_reader and not jina_api_key:
             return "[WebFetch] JINA_KEY is not set."
+        headers = {"Authorization": f"Bearer {jina_api_key}"} if use_hosted_reader else {}
 
         last_error = "unknown page-fetch error"
         for attempt in range(max_retries):
-            headers = {
-                "Authorization": f"Bearer {jina_api_key}",
-            }
             try:
                 remaining = self._remaining_budget_seconds(runtime_deadline)
                 if remaining is not None and remaining <= 0:
                     return "[WebFetch] Failed to read page: agent runtime limit reached."
                 response = requests.get(
-                    f"https://r.jina.ai/{url}",
+                    f"{reader_base_url}/{url}",
                     headers=headers,
                     timeout=min(timeout, max(remaining, 0.001)) if remaining is not None else timeout,
                 )

agent_base/utils.py

@@ -82,8 +82,12 @@ def env_flag(name: str) -> bool:
     return os.getenv(name, "").lower() in {"1", "true", "yes", "on"}
 
 
-def missing_required_env(required: tuple[str, ...] = REQUIRED_ENV_VARS) -> list[str]:
-    return [key for key in required if not os.getenv(key, "").strip()]
+def missing_required_env(required: Optional[tuple[str, ...]] = None) -> list[str]:
+    required_env = REQUIRED_ENV_VARS if required is None else required
+    missing = [key for key in required_env if not os.getenv(key, "").strip()]
+    if required is None and os.getenv("JINA_READER_BASE_URL", "").strip():
+        missing = [key for key in missing if key != "JINA_KEY"]
+    return missing
 
 
 def require_required_env(context: str = "ResearchHarness") -> None:

docs/tutorial_en.md

@@ -99,13 +99,14 @@ Required variables:
 | `API_BASE` | Base URL for the OpenAI-compatible chat-completions endpoint. |
 | `MODEL_NAME` | Main model used by ResearchHarness. |
 | `SERPER_KEY` | Serper key for `WebSearch` and `ScholarSearch`: https://serper.dev/ |
-| `JINA_KEY` | Jina key for `WebFetch`: https://jina.ai/ |
+| `JINA_KEY` | Jina key for the default hosted `WebFetch` backend: https://jina.ai/. Not required when `JINA_READER_BASE_URL` is configured. |
 | `MINERU_TOKEN` | MinerU token for `ReadPDF`: https://mineru.net/ |
 
 Optional variables:
 
 | Variable | Default | Meaning |
 | --- | --- | --- |
+| `JINA_READER_BASE_URL` | `https://r.jina.ai` | Absolute HTTP(S) Reader-compatible endpoint. Custom endpoints do not receive `JINA_KEY`; ResearchHarness does not manage the service or fall back to hosted Jina on failure. |
 | `WORKSPACE_ROOT` | `./workspace` | Default workspace root when no explicit workspace is passed. |
 | `MAX_ROUNDS` | `500` | Maximum ReAct loop rounds. |
 | `MAX_RUNTIME_SECONDS` | `10800` | Maximum wall-clock runtime for one agent run. |

docs/tutorial_zh.md

@@ -96,13 +96,14 @@ pip install -e . --no-deps
 | `API_BASE` | OpenAI-compatible chat-completions endpoint 的 base URL。 |
 | `MODEL_NAME` | ResearchHarness 使用的主模型。 |
 | `SERPER_KEY` | `WebSearch` 和 `ScholarSearch` 使用的 Serper key：https://serper.dev/ |
-| `JINA_KEY` | `WebFetch` 使用的 Jina key：https://jina.ai/ |
+| `JINA_KEY` | 默认 hosted `WebFetch` 后端使用的 Jina key：https://jina.ai/。配置 `JINA_READER_BASE_URL` 时不需要。 |
 | `MINERU_TOKEN` | `ReadPDF` 使用的 MinerU token：https://mineru.net/ |
 
 可选变量：
 
 | 变量 | 默认值 | 含义 |
 | --- | --- | --- |
+| `JINA_READER_BASE_URL` | `https://r.jina.ai` | Reader-compatible 的绝对 HTTP(S) endpoint。custom endpoint 不会收到 `JINA_KEY`；ResearchHarness 不负责启动该服务，失败时也不会回退到 hosted Jina。 |
 | `WORKSPACE_ROOT` | `./workspace` | 未显式传入 workspace 时使用的默认 workspace root。 |
 | `MAX_ROUNDS` | `500` | ReAct loop 最大轮次。 |
 | `MAX_RUNTIME_SECONDS` | `10800` | 单次 agent run 的最大运行秒数。 |

tests/test_edge_case_checks.py

@@ -69,6 +69,34 @@ def check_required_env_is_enforced() -> tuple[bool, str]:
                 os.environ[key] = value
 
 
+def check_custom_reader_endpoint_makes_jina_key_optional() -> tuple[bool, str]:
+    from agent_base.utils import REQUIRED_ENV_VARS, missing_required_env
+
+    env_names = (*REQUIRED_ENV_VARS, "JINA_READER_BASE_URL")
+    previous = {key: os.environ.get(key) for key in env_names}
+    try:
+        for key in REQUIRED_ENV_VARS:
+            os.environ[key] = f"test-{key.lower()}"
+        os.environ.pop("JINA_KEY", None)
+        os.environ.pop("JINA_READER_BASE_URL", None)
+        hosted_missing = missing_required_env()
+
+        os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3001"
+        custom_missing = missing_required_env()
+    finally:
+        for key, value in previous.items():
+            if value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+
+    detail = json.dumps(
+        {"hosted_missing": hosted_missing, "custom_missing": custom_missing},
+        indent=2,
+    )
+    return hosted_missing == ["JINA_KEY"] and custom_missing == [], detail
+
+
 def check_llm_hard_timeout_interrupts_blocking_call() -> tuple[bool, str]:
     from agent_base.react_agent import LLMHardTimeoutError, llm_hard_timeout
 
@@ -2358,6 +2386,101 @@ def fake_readpage(url, runtime_deadline=None):
     return ok, detail
 
 
+def check_webfetch_reader_endpoint_selection_and_auth_isolation() -> tuple[bool, str]:
+    from agent_base.tools import tool_web
+    from agent_base.tools.tool_web import WebFetch
+
+    saved = {name: os.environ.get(name) for name in ("JINA_KEY", "JINA_READER_BASE_URL")}
+    original_get = tool_web.requests.get
+    calls: list[dict[str, object]] = []
+
+    class FakeResponse:
+        def __init__(self, status_code: int, text: str):
+            self.status_code = status_code
+            self.text = text
+
+    def fake_get(url, *, headers, timeout):
+        calls.append({"url": url, "headers": dict(headers), "timeout": timeout})
+        if url.startswith("http://127.0.0.1:3002/"):
+            return FakeResponse(503, "local reader unavailable")
+        return FakeResponse(200, "reader response")
+
+    try:
+        tool_web.requests.get = fake_get
+        os.environ.pop("JINA_KEY", None)
+        os.environ.pop("JINA_READER_BASE_URL", None)
+        missing_key = WebFetch().jina_readpage("https://example.com")
+        missing_key_calls = list(calls)
+
+        os.environ["JINA_KEY"] = "hosted-secret"
+        hosted = WebFetch().jina_readpage("https://example.com/hosted")
+        hosted_call = calls[-1]
+
+        os.environ.pop("JINA_KEY", None)
+        os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3001/"
+        custom_without_key = WebFetch().jina_readpage("https://example.com/custom")
+        custom_without_key_call = calls[-1]
+
+        os.environ["JINA_KEY"] = "must-not-leak"
+        custom_with_key = WebFetch().jina_readpage("https://example.com/custom-key")
+        custom_with_key_call = calls[-1]
+
+        os.environ["JINA_READER_BASE_URL"] = "file:///tmp/reader"
+        calls_before_invalid = len(calls)
+        invalid = WebFetch().jina_readpage("https://example.com/invalid")
+        invalid_made_request = len(calls) != calls_before_invalid
+
+        os.environ["JINA_READER_BASE_URL"] = "http://127.0.0.1:3002"
+        calls_before_failure = len(calls)
+        custom_failure = WebFetch().jina_readpage("https://example.com/failure")
+        failure_calls = calls[calls_before_failure:]
+    finally:
+        tool_web.requests.get = original_get
+        for name, value in saved.items():
+            if value is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = value
+
+    detail = json.dumps(
+        {
+            "missing_key": missing_key,
+            "missing_key_calls": missing_key_calls,
+            "hosted": hosted,
+            "hosted_call": hosted_call,
+            "custom_without_key": custom_without_key,
+            "custom_without_key_call": custom_without_key_call,
+            "custom_with_key": custom_with_key,
+            "custom_with_key_call": custom_with_key_call,
+            "invalid": invalid,
+            "invalid_made_request": invalid_made_request,
+            "custom_failure": custom_failure,
+            "failure_calls": failure_calls,
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+    ok = (
+        missing_key == "[WebFetch] JINA_KEY is not set."
+        and missing_key_calls == []
+        and hosted == "reader response"
+        and hosted_call["url"] == "https://r.jina.ai/https://example.com/hosted"
+        and hosted_call["headers"] == {"Authorization": "Bearer hosted-secret"}
+        and custom_without_key == "reader response"
+        and custom_without_key_call["url"] == "http://127.0.0.1:3001/https://example.com/custom"
+        and custom_without_key_call["headers"] == {}
+        and custom_with_key == "reader response"
+        and custom_with_key_call["headers"] == {}
+        and invalid == "[WebFetch] JINA_READER_BASE_URL must be an absolute HTTP(S) URL."
+        and not invalid_made_request
+        and custom_failure.startswith("[WebFetch] Failed to read page: HTTP 503")
+        and len(failure_calls) == 3
+        and all(call["url"].startswith("http://127.0.0.1:3002/") for call in failure_calls)
+        and all("r.jina.ai" not in str(call["url"]) for call in failure_calls)
+    )
+    return ok, detail
+
+
 def check_session_state_is_only_written_with_trace_dir() -> tuple[bool, str]:
     from agent_base.react_agent import MultiTurnReactAgent
 
@@ -2469,6 +2592,7 @@ def main() -> int:
     checks = [
         ("Workspace root auto-create", check_workspace_root_is_created_when_missing),
         ("Required env enforced", check_required_env_is_enforced),
+        ("Custom Reader makes Jina key optional", check_custom_reader_endpoint_makes_jina_key_optional),
         ("ReadPDF relative image path", check_readpdf_relative_image_path),
         ("ReadPDF timeout result", check_readpdf_timeout_returns_tool_result),
         ("TerminalInterrupt remainder", check_terminal_interrupt_preserves_remainder),
@@ -2495,6 +2619,7 @@ def main() -> int:
         ("Terminal error accepts artifact", check_terminal_error_can_be_accepted_after_completion_artifact),
         ("Claude runtime sampling params", check_claude_models_skip_sampling_params_in_agent_runtime),
         ("WebFetch range-bounded page text", check_webfetch_returns_range_bounded_page_text_without_summary_llm),
+        ("WebFetch Reader endpoint isolation", check_webfetch_reader_endpoint_selection_and_auth_isolation),
         ("Tool schema provider compatibility", check_tool_schemas_avoid_provider_incompatible_mixed_types),
         ("Plaintext result max rounds", check_plaintext_result_rejection_hits_max_rounds),
         ("Bash output bounding", check_bash_output_bounding_and_repeat_collapse),