fix(evaluator-sdk): align Fabric runtime calls to current nemo-fabric API (enable_relay, adapter ids)

.github/actions/changes/action.yaml

@@ -13,6 +13,9 @@ outputs:
   deps:
     description: "'true' if any dependency-related files changed"
     value: ${{ steps.filter.outputs.deps }}
+  fabric:
+    description: "'true' if the Fabric agent-eval runtime, its tests, or its dependency extra changed"
+    value: ${{ steps.filter.outputs.fabric }}
   e2e:
     description: "'true' if any e2e test files changed"
     value: ${{ steps.filter.outputs.e2e }}
@@ -74,6 +77,12 @@ runs:
             - 'pyproject.toml'
             - 'uv.lock'
             - '.pre-commit-config.yaml'
+          fabric:
+            - 'packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/**'
+            - 'packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_*.py'
+            - 'packages/nemo_evaluator_sdk/pyproject.toml'
+            - '.github/workflows/ci.yaml'
+            - '.github/actions/changes/action.yaml'
           e2e:
             - 'e2e/**'
           docs:

.github/workflows/ci.yaml

@@ -37,6 +37,7 @@ jobs:
       openapi: ${{ steps.changes.outputs.openapi }}
       test: ${{ steps.changes.outputs.test }}
       deps: ${{ steps.changes.outputs.deps }}
+      fabric: ${{ steps.changes.outputs.fabric }}
       e2e: ${{ steps.changes.outputs.e2e }}
       docs: ${{ steps.changes.outputs.docs }}
       web-studio: ${{ steps.changes.outputs.web-studio }}
@@ -899,9 +900,11 @@ jobs:
   fabric-wheel-smoke:
     name: Fabric wheel install smoke (Linux)
     needs: [changes]
+    # A Fabric wheel bump lands in uv.lock (deps), and edits to the runtime's own Fabric call sites or
+    # its tests land under the fabric filter — either can surface API drift, so run on both.
     if: >
       !cancelled() &&
-      needs.changes.outputs.deps == 'true'
+      (needs.changes.outputs.deps == 'true' || needs.changes.outputs.fabric == 'true')
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -933,11 +936,25 @@ jobs:
               Fabric,
               FabricConfig,
               FabricProfileConfig,
+              RelayAtifConfig,
+              RelayAtofConfig,
+              RelayObservabilityConfig,
               RunRequest,
               RunResult,
           )
           print("nemo_fabric import OK:", nemo_fabric.__file__)
           PY
+      # A bare import can't catch API drift in the runtime's *call sites* — the enable_relay keyword and
+      # the harness adapter ids both moved under us while the hermetic fakes kept passing. These contract
+      # tests exercise those call sites against the real wheels (importorskip elsewhere), so a future
+      # drift turns this job red instead of slipping through.
+      - name: Run Fabric runtime contract tests against the real wheels
+        # pytest lives in the root `dev` group, which the minimal `--package ... --extra fabric` sync
+        # above does not pull; layer it (and pytest-asyncio, for the SDK package's asyncio_mode=auto)
+        # in ephemerally rather than bloating the smoke env with the whole dev group.
+        run: |
+          uv run --frozen --no-sync --with pytest --with pytest-asyncio pytest \
+            packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py -v
 
   python-integration-test:
     name: Python integration tests

packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb

@@ -551,7 +551,7 @@
    "id": "cell-24",
    "metadata": {},
    "outputs": [],
-   "source": "from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime\nfrom nemo_fabric import (  # ty: ignore[unresolved-import]\n    EnvironmentConfig,\n    FabricConfig,\n    HarnessConfig,\n    MetadataConfig,\n    RuntimeConfig,\n)\n\n# A typed Fabric agent config: Codex CLI harness, one-shot text in / message out, local execution.\ncodex_via_fabric = FabricConfig(\n    metadata=MetadataConfig(name=\"coding-agent-eval\"),\n    harness=HarnessConfig(\n        adapter_id=\"nvidia.fabric.codex.cli\",  # Fabric drives the Codex CLI under the hood\n        resolution=\"preinstalled\",\n        settings={\"sandbox\": \"workspace-write\", \"skip_git_repo_check\": True, \"timeout_seconds\": 180},\n    ),\n    runtime=RuntimeConfig(mode=\"oneshot\", transport=\"cli\", input_schema=\"text\", output_schema=\"message\"),\n    environment=EnvironmentConfig(provider=\"local\"),  # the per-task workspace is set by the runtime\n)\n\ntarget = FabricAgentRuntime(\n    config=codex_via_fabric,\n    model=os.environ.get(\"CODEX_MODEL\"),  # None → the adapter's default model\n    base_dir=Path(os.environ[\"NEMO_FABRIC_REPO\"]),  # resolves adapters/codex-cli\n    work_root=OUTPUT_DIR / \"fabric\",\n    capture_trajectory=True,  # capture the agent's ATIF trajectory as trace evidence\n)\nprint(\"Target:\", type(target).__name__)"
+   "source": "from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime\nfrom nemo_fabric import (  # ty: ignore[unresolved-import]\n    EnvironmentConfig,\n    FabricConfig,\n    HarnessConfig,\n    MetadataConfig,\n    RuntimeConfig,\n)\n\n# A typed Fabric agent config: Codex CLI harness, one-shot text in / message out, local execution.\ncodex_via_fabric = FabricConfig(\n    metadata=MetadataConfig(name=\"coding-agent-eval\"),\n    harness=HarnessConfig(\n        adapter_id=\"nvidia.fabric.codex\",  # Fabric drives the Codex CLI under the hood\n        resolution=\"preinstalled\",\n        settings={\"sandbox\": \"workspace-write\", \"skip_git_repo_check\": True, \"timeout_seconds\": 180},\n    ),\n    runtime=RuntimeConfig(mode=\"oneshot\", transport=\"cli\", input_schema=\"text\", output_schema=\"message\"),\n    environment=EnvironmentConfig(provider=\"local\"),  # the per-task workspace is set by the runtime\n)\n\ntarget = FabricAgentRuntime(\n    config=codex_via_fabric,\n    model=os.environ.get(\"CODEX_MODEL\"),  # None → the adapter's default model\n    base_dir=Path(os.environ[\"NEMO_FABRIC_REPO\"]),  # resolves adapters/codex-cli\n    work_root=OUTPUT_DIR / \"fabric\",\n    capture_trajectory=True,  # capture the agent's ATIF trajectory as trace evidence\n)\nprint(\"Target:\", type(target).__name__)"
   },
   {
    "cell_type": "markdown",
@@ -668,7 +668,7 @@
    "cell_type": "markdown",
    "id": "cell-35",
    "metadata": {},
-   "source": "<a id=\"next\"></a>\n## 8. Where to go next\n\n- **Swap the harness.** Fabric selects the agent by `harness.adapter_id`. Point it at another adapter\n  (e.g. `nvidia.fabric.hermes.cli`) to evaluate a different coding agent with the same tasks and\n  metrics.\n- **Evaluate your own agent.** Implement the `AgentTaskRunner` protocol — a class with one async\n  `run_tasks(tasks, config)` that runs each task and returns a trial (output + workspace evidence).\n  Everything else in this notebook stays the same.\n- **Re-score without re-running the agent.** Pass precomputed `trials=` instead of `target=` to\n  `run_sync` to apply new metrics to trials you already have.\n- **Grade against the trajectory.** A metric can open the `trace` evidence (ATIF) to score *how* the\n  agent worked — tool calls, retries, steps — not just its final files.\n- **Add signals to a view, or weight them.** `SemanticReducer` also offers `ANY`, `MEAN`, and\n  `WEIGHTED_MEAN` (with `ViewSignal(weight=...)`) — e.g. a partial-credit `correctness` from `pass_rate`.\n- **Keep ground truth held out.** Anything a metric grades on — tests, reference solutions, rubrics —\n  belongs in `reference`, overlaid or checksummed at scoring time, never in the agent's workspace.\n- **Decide pass/fail in your app.** The evaluator reports scores; thresholds and gating belong to your\n  CI/release process, not the evaluation itself."
+   "source": "<a id=\"next\"></a>\n## 8. Where to go next\n\n- **Swap the harness.** Fabric selects the agent by `harness.adapter_id`. Point it at another adapter\n  (e.g. `nvidia.fabric.hermes`) to evaluate a different coding agent with the same tasks and\n  metrics.\n- **Evaluate your own agent.** Implement the `AgentTaskRunner` protocol — a class with one async\n  `run_tasks(tasks, config)` that runs each task and returns a trial (output + workspace evidence).\n  Everything else in this notebook stays the same.\n- **Re-score without re-running the agent.** Pass precomputed `trials=` instead of `target=` to\n  `run_sync` to apply new metrics to trials you already have.\n- **Grade against the trajectory.** A metric can open the `trace` evidence (ATIF) to score *how* the\n  agent worked — tool calls, retries, steps — not just its final files.\n- **Add signals to a view, or weight them.** `SemanticReducer` also offers `ANY`, `MEAN`, and\n  `WEIGHTED_MEAN` (with `ViewSignal(weight=...)`) — e.g. a partial-credit `correctness` from `pass_rate`.\n- **Keep ground truth held out.** Anything a metric grades on — tests, reference solutions, rubrics —\n  belongs in `reference`, overlaid or checksummed at scoring time, never in the agent's workspace.\n- **Decide pass/fail in your app.** The evaluator reports scores; thresholds and gating belong to your\n  CI/release process, not the evaluation itself."
   }
  ],
  "metadata": {

packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py

@@ -46,7 +46,7 @@ async def main() -> int:
         # chosen by harness.adapter_id, never inferred from the model.
         FabricConfig(
             metadata=MetadataConfig(name="hermes-eval"),
-            harness=HarnessConfig(adapter_id="nvidia.fabric.hermes.sdk", resolution="preinstalled"),
+            harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
             models={"default": {"provider": "nvidia", "model": model}},
             runtime=RuntimeConfig(mode="oneshot", transport="library", input_schema="chat", output_schema="message"),
         ),

packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py

@@ -8,9 +8,9 @@
 ``harness.adapter_id``, never inferred from a model. Across harnesses the shape differs mainly in that
 ``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``:
 
-* **Codex CLI** (``nvidia.fabric.codex.cli``) runs the agent as a subprocess — ``transport="cli"`` —
+* **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` —
   and takes codex-specific ``harness.settings`` (sandbox mode, git-repo check, ...).
-* **Hermes SDK** (``nvidia.fabric.hermes.sdk``) runs in-library — ``transport="library"`` — and
+* **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and
   declares its ``input``/``output`` schemas instead.
 
 An optional ``model=`` slug (e.g. ``"openai/gpt-5.4"``) can be passed to ``FabricAgentRuntime`` to
@@ -39,7 +39,7 @@
 CODEX_CLI_CONFIG = FabricConfig(
     metadata=MetadataConfig(name="codex-eval"),
     harness=HarnessConfig(
-        adapter_id="nvidia.fabric.codex.cli",
+        adapter_id="nvidia.fabric.codex",
         settings={"sandbox": "read-only", "skip_git_repo_check": True},
     ),
     models={"default": {"provider": "openai", "model": "gpt-5.4"}},
@@ -49,7 +49,7 @@
 #: Hermes SDK harness — in-library transport, explicit chat/message schemas.
 HERMES_SDK_CONFIG = FabricConfig(
     metadata=MetadataConfig(name="hermes-eval"),
-    harness=HarnessConfig(adapter_id="nvidia.fabric.hermes.sdk", resolution="preinstalled"),
+    harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
     models={"default": {"provider": "nvidia", "model": "qwen2.5-coder-32b"}},
     runtime=RuntimeConfig(mode="oneshot", transport="library", input_schema="chat", output_schema="message"),
 )

packages/nemo_evaluator_sdk/examples/skill_eval/README.md

@@ -68,7 +68,7 @@ block and exits non-zero rather than showing an empty-but-tidy table.
 Example output (with `nvidia/nemotron-3-super-120b-a12b`):
 
 ```text
-Harness: nvidia.fabric.hermes.sdk   model: nvidia/nemotron-3-super-120b-a12b   tasks: 2
+Harness: nvidia.fabric.hermes   model: nvidia/nemotron-3-super-120b-a12b   tasks: 2
 runs: baseline (baseline) vs treated (treated)
 
   metric.output                             baseline   with-skill

packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py

@@ -242,7 +242,7 @@ async def _main() -> int:
     fabric_config = {
         "metadata": {"name": "skill-eval-hermes"},
         "harness": {
-            "adapter_id": "nvidia.fabric.hermes.sdk",
+            "adapter_id": "nvidia.fabric.hermes",
             "resolution": "preinstalled",
             "settings": {"max_iterations": 50},
         },

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/image.py

@@ -25,7 +25,7 @@
 That only works on NeMo-Fabric's ``installed-adapter-discovery`` branch (which bundles the adapters
 under ``python/src/nemo_fabric/adapters`` and adds ``AdapterDescriptorSource::Installed``). On today's
 ``main`` the wheel ships no adapter descriptors, so a wheel-only image cannot resolve e.g.
-``nvidia.fabric.hermes.sdk``. Once that lands on ``main``, switch to installing the top-level
+``nvidia.fabric.hermes``. Once that lands on ``main``, switch to installing the top-level
 ``adapters/*`` packages explicitly here instead of relying on the branch's packaging.
 """
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py

@@ -64,6 +64,7 @@
         Fabric,
         FabricConfig,
         FabricProfileConfig,
+        RelayObservabilityConfig,
         RunOutput,
         RunResult,
     )
@@ -479,7 +480,7 @@ def _compose_config(
             artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR
             relay_dir.mkdir(parents=True, exist_ok=True)
             artifacts_dir.mkdir(parents=True, exist_ok=True)
-            cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir))
+            cfg.enable_relay(output_dir=str(relay_dir), observability=self._relay_config(relay_dir))
             cfg.runtime.artifacts = str(artifacts_dir)
             cfg.environment.artifacts = str(artifacts_dir)
 
@@ -523,39 +524,35 @@ def _eval_lock_profiles(
             )
         return overlays
 
-    def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
-        # The observability component is built from nemo_relay's own typed config objects so Relay owns
-        # its schema (no hand-maintained dict that silently drifts when Relay changes it); imported
-        # lazily since nemo-relay, like nemo-fabric, is an optional native dependency.
-        try:
-            from nemo_relay.observability import (  # ty: ignore[unresolved-import]
-                AtifConfig,
-                AtofConfig,
-                ComponentSpec,
-                ObservabilityConfig,
-            )
-        except ImportError as exc:
-            raise RuntimeError(_MISSING_RELAY_MSG) from exc
+    def _relay_config(self, relay_dir: Path) -> RelayObservabilityConfig:
+        # The ATIF/ATOF observability config is built from Fabric's own typed relay-config objects so
+        # Fabric owns the schema (no hand-maintained dict that silently drifts when Fabric changes it),
+        # mirroring nemo_fabric's own Harbor integration. It is handed straight to ``enable_relay`` via
+        # its ``observability=`` parameter — the SDK only configures ATIF/ATOF observability, so it needs
+        # neither a generic ``components`` list nor the legacy component-wrapped shape. nemo_fabric is
+        # already imported+validated in ``run_tasks``, so this is a cached sys.modules lookup.
+        from nemo_fabric import (  # ty: ignore[unresolved-import]
+            RelayAtifConfig,
+            RelayAtofConfig,
+            RelayObservabilityConfig,
+        )
 
         relay_dir_str = str(relay_dir)
-        observability = ComponentSpec(
-            config=ObservabilityConfig(
-                atif=AtifConfig(
-                    enabled=True,
-                    output_directory=relay_dir_str,
-                    filename_template=_ATIF_FILENAME_TEMPLATE,
-                    agent_name=self._runtime_name,
-                    agent_version="fabric",
-                ),
-                atof=AtofConfig(
-                    enabled=True,
-                    output_directory=relay_dir_str,
-                    filename=_ATOF_FILENAME,
-                    mode="overwrite",
-                ),
-            )
+        return RelayObservabilityConfig(
+            atif=RelayAtifConfig(
+                enabled=True,
+                output_directory=relay_dir_str,
+                filename_template=_ATIF_FILENAME_TEMPLATE,
+                agent_name=self._runtime_name,
+                agent_version="fabric",
+            ),
+            atof=RelayAtofConfig(
+                enabled=True,
+                output_directory=relay_dir_str,
+                filename=_ATOF_FILENAME,
+                mode="overwrite",
+            ),
         )
-        return {"version": 1, "components": [observability.to_dict()]}
 
     def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path:
         root = self._work_root

packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_container_runtime.py

@@ -26,7 +26,7 @@
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus
 from nemo_evaluator_sdk.values.common import SecretRef
 
-_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.hermes.sdk"}}
+_CONFIG = {"metadata": {"name": "eval"}, "harness": {"adapter_id": "nvidia.fabric.hermes"}}
 
 
 @pytest.fixture(autouse=True)