ci: dispatchable model-onboarding check (one model, one product)

.github/workflows/model-onboarding.yml

@@ -0,0 +1,117 @@
+# Model onboarding check: run the langchain integration suite for ONE model that is not (yet) in
+# the committed test matrix — e.g. a model the LLM Gateway just onboarded — as a given requesting
+# product, against the LLM Gateway environment held in a GitHub environment of this repo.
+#
+# Dispatched by hand, or by UiPath/llm-gateway-config when a model is routed for agents/agenthub
+# in prod. The model is injected into the per-provider conftest matrix for this run only, and only
+# into the vendor clients whose API flavor the gateway declares for it (SupportedApiFlavors).
+#
+# target_environment: LLMGW_SETTINGS (alpha, exists) or a prod-tier environment such as
+# LLMGW_SETTINGS_PROD — same variable/secret names as ci.yml's test job.
+name: Model onboarding check
+
+on:
+  workflow_dispatch:
+    inputs:
+      model:
+        description: model id as passed to the gateway (e.g. gemini-3.7-flash)
+        required: true
+        type: string
+      family:
+        description: LLM Gateway ModelFamily
+        required: true
+        type: choice
+        options: [GoogleGemini, AnthropicClaude, OpenAi]
+      vendor:
+        description: LLM Gateway Vendor
+        required: true
+        type: choice
+        options: [VertexAi, AwsBedrock, OpenAi, NativeOpenAi]
+      product:
+        description: X-UiPath-LlmGateway-RequestingProduct
+        required: true
+        type: choice
+        options: [agents, agenthub, agent-gym, testproduct]
+      api_flavors:
+        description: comma-separated LLM Gateway SupportedApiFlavors (selects which vendor clients run)
+        required: true
+        type: string
+      operations:
+        description: comma-separated SupportedNormalizedOperations
+        required: false
+        default: NormalizedChat
+        type: string
+      target_environment:
+        description: GitHub environment holding the LLMGW_* settings
+        required: false
+        default: LLMGW_SETTINGS
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }}
+    runs-on: uipath-ubuntu-latest
+    environment: ${{ inputs.target_environment }}
+    env:
+      UIPATH_LLM_SERVICE: ${{ vars.UIPATH_LLM_SERVICE }}
+      LLMGW_URL: ${{ vars.LLMGW_URL }}
+      LLMGW_SEMANTIC_ORG_ID: ${{ secrets.LLMGW_SEMANTIC_ORG_ID }}
+      LLMGW_SEMANTIC_USER_ID: ${{ secrets.LLMGW_SEMANTIC_USER_ID }}
+      LLMGW_SEMANTIC_TENANT_ID: ${{ secrets.LLMGW_SEMANTIC_TENANT_ID }}
+      LLMGW_CLIENT_ID: ${{ secrets.LLMGW_CLIENT_ID }}
+      LLMGW_CLIENT_SECRET: ${{ secrets.LLMGW_CLIENT_SECRET }}
+      LLMGW_REQUESTING_PRODUCT: ${{ inputs.product }}
+      LLMGW_REQUESTING_FEATURE: ${{ inputs.product == 'agenthub' && 'llm-call' || inputs.product == 'agents' && 'agents-prompt' || vars.LLMGW_REQUESTING_FEATURE }}
+      LLMGW_OPERATION_CODE: ${{ inputs.product == 'agenthub' && 'AgentHub.LLM' || inputs.product == 'agents' && 'Agents.Execution' || '' }}
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803  # v6.1.0
+        with:
+          lfs: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78  # v7.6.0
+        with:
+          version: "0.9.27"
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install dependencies
+        run: uv sync --dev --all-extras
+
+      - name: Inject the model into the test matrix (this run only)
+        id: inject
+        run: |
+          out=$(python scripts/inject_model_into_matrix.py --repo . \
+                  --model "${{ inputs.model }}" --family "${{ inputs.family }}" --vendor "${{ inputs.vendor }}" \
+                  --flavors "${{ inputs.api_flavors }}" --operations "${{ inputs.operations }}")
+          echo "$out"
+          echo "dirs=$(echo "$out" | tail -1)" >> "$GITHUB_OUTPUT"
+
+      # record_mode=new_episodes: an unseen model goes to the wire. The cassette DB is not
+      # uploaded or committed, so this never pollutes the committed recordings.
+      - name: Run the langchain integration suite for the model
+        run: |
+          paths=""; for d in ${{ steps.inject.outputs.dirs }}; do paths="$paths tests/langchain/clients/$d/test_integration.py"; done
+          uv run pytest $paths -k "${{ inputs.model }}" -q --no-header \
+            --junitxml "report-${{ inputs.model }}-${{ inputs.product }}.xml" | tee "run-${{ inputs.model }}-${{ inputs.product }}.log"
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "### ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }}" >> "$GITHUB_STEP_SUMMARY"
+          tail -3 "run-${{ inputs.model }}-${{ inputs.product }}.log" >> "$GITHUB_STEP_SUMMARY" || true
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: model-onboarding-${{ inputs.model }}-${{ inputs.product }}-${{ inputs.target_environment }}
+          path: |
+            report-*.xml
+            run-*.log

scripts/inject_model_into_matrix.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Add a model to the langchain integration matrix for ONE run, selecting clients by the
+LLM Gateway's declared SupportedApiFlavors (see .github/workflows/model-onboarding.yml).
+
+Each vendor client speaks one gateway API flavor; a model is only exercised through clients
+whose flavor it declares:
+
+  GeminiGenerateContent  -> google (UiPathChatGoogleGenerativeAI), litellm GEMINI_CONFIGS
+  AnthropicMessages      -> anthropic (UiPathChatAnthropic, vendor_type by vendor)
+                            + vertexai (UiPathChatAnthropicVertex)      [vendor VertexAi]
+                            + bedrock  (UiPathChatAnthropicBedrock)     [vendor AwsBedrock]
+                            + litellm VERTEX_CLAUDE_CONFIGS               [vendor VertexAi]
+  AwsBedrockInvoke       -> bedrock (UiPathChatBedrock), litellm BEDROCK_INVOKE_CONFIGS
+  AwsBedrockConverse     -> bedrock (UiPathChatBedrockConverse), litellm BEDROCK_CONVERSE_CONFIGS
+  OpenAiChatCompletions  -> openai (UiPathAzureChatOpenAI), litellm OPENAI_CONFIGS
+  OpenAiResponses        -> openai (UiPathAzureChatOpenAI, use_responses_api), litellm OPENAI_RESPONSES_CONFIGS
+  NormalizedChat (op)    -> normalized (UiPathChat) always, thinking/reasoning config by family
+
+  inject_model_into_matrix.py --repo . --model <id> --family <ModelFamily> --vendor <Vendor> \
+      --flavors GeminiGenerateContent[,...] [--operations NormalizedChat]
+Prints the conftest dirs touched on the last stdout line (consumed by the workflow).
+"""
+import argparse
+import re
+import sys
+from pathlib import Path
+
+CLIENTS = Path("tests/langchain/clients")
+THINKING = '{"max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}}'
+
+
+def entries(cls: str, extra: str | None = None) -> str:
+    """Config entries (plain + optional kwargs variant) as Python source, existing-list style."""
+    base = f'{{"model_class": {cls}}}'
+    if extra is None:
+        return base
+    return f'{base}, {{"model_class": {cls}, "model_kwargs": {extra}}}'
+
+
+def gemini_list(model: str) -> str:
+    return "GEMINI_3_CONFIGS" if re.search(r"gemini-3", model.lower()) else "GEMINI_2_5_CONFIGS"
+
+
+def gpt_reasoning(model: str) -> bool:
+    m = model.lower()
+    return bool(re.match(r"(gpt-5|o\d)", m)) and "chat" not in m
+
+
+def plan(model: str, family: str, vendor: str, flavors: set[str], operations: set[str]) -> dict[str, str]:
+    """conftest dir -> Python expression for the model's config list."""
+    out: dict[str, str] = {}
+    lite: list[str] = []
+    if "NormalizedChat" in operations:
+        if family == "GoogleGemini":
+            out["normalized"] = gemini_list(model)
+        elif family == "AnthropicClaude":
+            out["normalized"] = "CLAUDE_BEDROCK_CONFIGS" if vendor == "AwsBedrock" else "CLAUDE_VERTEXAI_CONFIGS"
+        else:
+            out["normalized"] = "GPT_REASONING_CONFIGS" if gpt_reasoning(model) else "GPT_NON_REASONING_CONFIGS"
+    # Gateway convention: Vertex-hosted Claude entries declare GeminiGenerateContent (the enum has no
+    # Vertex-Anthropic flavor); it means "Vertex passthrough", which for Claude is Anthropic Messages.
+    if family == "AnthropicClaude" and vendor == "VertexAi" and "GeminiGenerateContent" in flavors:
+        flavors = (flavors - {"GeminiGenerateContent"}) | {"AnthropicMessages"}
+    if "GeminiGenerateContent" in flavors:
+        out["google"] = gemini_list(model)
+        lite.append("GEMINI_CONFIGS")
+    if "AnthropicMessages" in flavors:
+        if vendor == "AwsBedrock":
+            out["anthropic"] = "CLAUDE_BEDROCK_CONFIGS"
+        else:
+            out["anthropic"] = "CLAUDE_VERTEXAI_CONFIGS"
+            out["vertexai"] = "CLAUDE_VERTEXAI_CONFIGS"
+            lite.append("VERTEX_CLAUDE_CONFIGS")
+    bedrock: list[str] = []
+    if "AnthropicMessages" in flavors and vendor == "AwsBedrock":
+        bedrock.append(entries("UiPathChatAnthropicBedrock", THINKING))
+    if "AwsBedrockInvoke" in flavors:
+        bedrock.append(entries("UiPathChatBedrock", THINKING))
+        lite.append("BEDROCK_INVOKE_CONFIGS")
+    if "AwsBedrockConverse" in flavors:
+        bedrock.append(entries("UiPathChatBedrockConverse", THINKING))
+        lite.append("BEDROCK_CONVERSE_CONFIGS")
+    if bedrock:
+        out["bedrock"] = "[" + ", ".join(bedrock) + "]"
+    openai: list[str] = []
+    if "OpenAiChatCompletions" in flavors:
+        openai.append(entries("UiPathAzureChatOpenAI", '{"reasoning_effort": "low"}' if gpt_reasoning(model) else None))
+        lite.append("OPENAI_CONFIGS")
+    if "OpenAiResponses" in flavors:
+        if gpt_reasoning(model):
+            kw = '{"use_responses_api": True, "reasoning": {"effort": "low", "summary": "auto"}, "verbosity": "low"}'
+        else:
+            kw = '{"use_responses_api": True}'
+        openai.append(f'{{"model_class": UiPathAzureChatOpenAI, "model_kwargs": {kw}}}')
+        lite.append("OPENAI_RESPONSES_CONFIGS")
+    if openai:
+        out["openai"] = "[" + ", ".join(openai) + "]"
+    if lite:
+        out["litellm"] = " + ".join(lite)
+    return out
+
+
+def inject(conftest: Path, model: str, expr: str) -> bool:
+    text = conftest.read_text(encoding="utf-8")
+    if f'"{model}"' in text:
+        return False
+    for name in re.findall(r"\b([A-Z][A-Z0-9_]+_CONFIGS)\b", expr):
+        if not re.search(rf"^{name} = \[", text, re.M):
+            raise SystemExit(f"{conftest}: config list {name} not found")
+    new, n = re.subn(r"(COMPLETIONS_MODELS_WITH_CONFIGS = \{\n)", rf'\1    "{model}": {expr},\n', text, count=1)
+    if n != 1:
+        raise SystemExit(f"{conftest}: COMPLETIONS_MODELS_WITH_CONFIGS dict not found")
+    conftest.write_text(new, encoding="utf-8")
+    return True
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--repo", required=True)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--family", required=True)
+    ap.add_argument("--vendor", required=True)
+    ap.add_argument("--flavors", required=True, help="comma-separated SupportedApiFlavors")
+    ap.add_argument("--operations", default="NormalizedChat", help="comma-separated SupportedNormalizedOperations")
+    a = ap.parse_args()
+    flavors = {f.strip() for f in a.flavors.split(",") if f.strip()}
+    ops = {o.strip() for o in a.operations.split(",") if o.strip()}
+    p = plan(a.model, a.family, a.vendor, flavors, ops)
+    if not p:
+        raise SystemExit(f"no client covers flavors {sorted(flavors)} / operations {sorted(ops)}")
+    touched = [d for d, expr in p.items() if inject(Path(a.repo) / CLIENTS / d / "conftest.py", a.model, expr)]
+    for d, expr in p.items():
+        print(f"  {d:11s} <- {expr}")
+    print(f"injected {a.model} (flavors {','.join(sorted(flavors))}) into: {' '.join(touched) or '(already present)'}")
+    print(" ".join(p))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())