diff --git a/.github/workflows/model-onboarding.yml b/.github/workflows/model-onboarding.yml new file mode 100644 index 0000000..f4826b5 --- /dev/null +++ b/.github/workflows/model-onboarding.yml @@ -0,0 +1,117 @@ +# Model onboarding check: run the langchain integration suite for ONE model that is not (yet) in +# the committed test matrix — e.g. a model the LLM Gateway just onboarded — as a given requesting +# product, against the LLM Gateway environment held in a GitHub environment of this repo. +# +# Dispatched by hand, or by UiPath/llm-gateway-config when a model is routed for agents/agenthub +# in prod. The model is injected into the per-provider conftest matrix for this run only, and only +# into the vendor clients whose API flavor the gateway declares for it (SupportedApiFlavors). +# +# target_environment: LLMGW_SETTINGS (alpha, exists) or a prod-tier environment such as +# LLMGW_SETTINGS_PROD — same variable/secret names as ci.yml's test job. +name: Model onboarding check + +on: + workflow_dispatch: + inputs: + model: + description: model id as passed to the gateway (e.g. gemini-3.7-flash) + required: true + type: string + family: + description: LLM Gateway ModelFamily + required: true + type: choice + options: [GoogleGemini, AnthropicClaude, OpenAi] + vendor: + description: LLM Gateway Vendor + required: true + type: choice + options: [VertexAi, AwsBedrock, OpenAi, NativeOpenAi] + product: + description: X-UiPath-LlmGateway-RequestingProduct + required: true + type: choice + options: [agents, agenthub, agent-gym, testproduct] + api_flavors: + description: comma-separated LLM Gateway SupportedApiFlavors (selects which vendor clients run) + required: true + type: string + operations: + description: comma-separated SupportedNormalizedOperations + required: false + default: NormalizedChat + type: string + target_environment: + description: GitHub environment holding the LLMGW_* settings + required: false + default: LLMGW_SETTINGS + type: string + +permissions: + contents: read + +jobs: + test: + name: ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }} + runs-on: uipath-ubuntu-latest + environment: ${{ inputs.target_environment }} + env: + UIPATH_LLM_SERVICE: ${{ vars.UIPATH_LLM_SERVICE }} + LLMGW_URL: ${{ vars.LLMGW_URL }} + LLMGW_SEMANTIC_ORG_ID: ${{ secrets.LLMGW_SEMANTIC_ORG_ID }} + LLMGW_SEMANTIC_USER_ID: ${{ secrets.LLMGW_SEMANTIC_USER_ID }} + LLMGW_SEMANTIC_TENANT_ID: ${{ secrets.LLMGW_SEMANTIC_TENANT_ID }} + LLMGW_CLIENT_ID: ${{ secrets.LLMGW_CLIENT_ID }} + LLMGW_CLIENT_SECRET: ${{ secrets.LLMGW_CLIENT_SECRET }} + LLMGW_REQUESTING_PRODUCT: ${{ inputs.product }} + LLMGW_REQUESTING_FEATURE: ${{ inputs.product == 'agenthub' && 'llm-call' || inputs.product == 'agents' && 'agents-prompt' || vars.LLMGW_REQUESTING_FEATURE }} + LLMGW_OPERATION_CODE: ${{ inputs.product == 'agenthub' && 'AgentHub.LLM' || inputs.product == 'agents' && 'Agents.Execution' || '' }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + lfs: true + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: "0.9.27" + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version-file: ".python-version" + + - name: Install dependencies + run: uv sync --dev --all-extras + + - name: Inject the model into the test matrix (this run only) + id: inject + run: | + out=$(python scripts/inject_model_into_matrix.py --repo . \ + --model "${{ inputs.model }}" --family "${{ inputs.family }}" --vendor "${{ inputs.vendor }}" \ + --flavors "${{ inputs.api_flavors }}" --operations "${{ inputs.operations }}") + echo "$out" + echo "dirs=$(echo "$out" | tail -1)" >> "$GITHUB_OUTPUT" + + # record_mode=new_episodes: an unseen model goes to the wire. The cassette DB is not + # uploaded or committed, so this never pollutes the committed recordings. + - name: Run the langchain integration suite for the model + run: | + paths=""; for d in ${{ steps.inject.outputs.dirs }}; do paths="$paths tests/langchain/clients/$d/test_integration.py"; done + uv run pytest $paths -k "${{ inputs.model }}" -q --no-header \ + --junitxml "report-${{ inputs.model }}-${{ inputs.product }}.xml" | tee "run-${{ inputs.model }}-${{ inputs.product }}.log" + + - name: Summary + if: always() + run: | + echo "### ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }}" >> "$GITHUB_STEP_SUMMARY" + tail -3 "run-${{ inputs.model }}-${{ inputs.product }}.log" >> "$GITHUB_STEP_SUMMARY" || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: model-onboarding-${{ inputs.model }}-${{ inputs.product }}-${{ inputs.target_environment }} + path: | + report-*.xml + run-*.log diff --git a/scripts/inject_model_into_matrix.py b/scripts/inject_model_into_matrix.py new file mode 100644 index 0000000..fa8b5a1 --- /dev/null +++ b/scripts/inject_model_into_matrix.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Add a model to the langchain integration matrix for ONE run, selecting clients by the +LLM Gateway's declared SupportedApiFlavors (see .github/workflows/model-onboarding.yml). + +Each vendor client speaks one gateway API flavor; a model is only exercised through clients +whose flavor it declares: + + GeminiGenerateContent -> google (UiPathChatGoogleGenerativeAI), litellm GEMINI_CONFIGS + AnthropicMessages -> anthropic (UiPathChatAnthropic, vendor_type by vendor) + + vertexai (UiPathChatAnthropicVertex) [vendor VertexAi] + + bedrock (UiPathChatAnthropicBedrock) [vendor AwsBedrock] + + litellm VERTEX_CLAUDE_CONFIGS [vendor VertexAi] + AwsBedrockInvoke -> bedrock (UiPathChatBedrock), litellm BEDROCK_INVOKE_CONFIGS + AwsBedrockConverse -> bedrock (UiPathChatBedrockConverse), litellm BEDROCK_CONVERSE_CONFIGS + OpenAiChatCompletions -> openai (UiPathAzureChatOpenAI), litellm OPENAI_CONFIGS + OpenAiResponses -> openai (UiPathAzureChatOpenAI, use_responses_api), litellm OPENAI_RESPONSES_CONFIGS + NormalizedChat (op) -> normalized (UiPathChat) always, thinking/reasoning config by family + + inject_model_into_matrix.py --repo . --model --family --vendor \ + --flavors GeminiGenerateContent[,...] [--operations NormalizedChat] +Prints the conftest dirs touched on the last stdout line (consumed by the workflow). +""" +import argparse +import re +import sys +from pathlib import Path + +CLIENTS = Path("tests/langchain/clients") +THINKING = '{"max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}}' + + +def entries(cls: str, extra: str | None = None) -> str: + """Config entries (plain + optional kwargs variant) as Python source, existing-list style.""" + base = f'{{"model_class": {cls}}}' + if extra is None: + return base + return f'{base}, {{"model_class": {cls}, "model_kwargs": {extra}}}' + + +def gemini_list(model: str) -> str: + return "GEMINI_3_CONFIGS" if re.search(r"gemini-3", model.lower()) else "GEMINI_2_5_CONFIGS" + + +def gpt_reasoning(model: str) -> bool: + m = model.lower() + return bool(re.match(r"(gpt-5|o\d)", m)) and "chat" not in m + + +def plan(model: str, family: str, vendor: str, flavors: set[str], operations: set[str]) -> dict[str, str]: + """conftest dir -> Python expression for the model's config list.""" + out: dict[str, str] = {} + lite: list[str] = [] + if "NormalizedChat" in operations: + if family == "GoogleGemini": + out["normalized"] = gemini_list(model) + elif family == "AnthropicClaude": + out["normalized"] = "CLAUDE_BEDROCK_CONFIGS" if vendor == "AwsBedrock" else "CLAUDE_VERTEXAI_CONFIGS" + else: + out["normalized"] = "GPT_REASONING_CONFIGS" if gpt_reasoning(model) else "GPT_NON_REASONING_CONFIGS" + # Gateway convention: Vertex-hosted Claude entries declare GeminiGenerateContent (the enum has no + # Vertex-Anthropic flavor); it means "Vertex passthrough", which for Claude is Anthropic Messages. + if family == "AnthropicClaude" and vendor == "VertexAi" and "GeminiGenerateContent" in flavors: + flavors = (flavors - {"GeminiGenerateContent"}) | {"AnthropicMessages"} + if "GeminiGenerateContent" in flavors: + out["google"] = gemini_list(model) + lite.append("GEMINI_CONFIGS") + if "AnthropicMessages" in flavors: + if vendor == "AwsBedrock": + out["anthropic"] = "CLAUDE_BEDROCK_CONFIGS" + else: + out["anthropic"] = "CLAUDE_VERTEXAI_CONFIGS" + out["vertexai"] = "CLAUDE_VERTEXAI_CONFIGS" + lite.append("VERTEX_CLAUDE_CONFIGS") + bedrock: list[str] = [] + if "AnthropicMessages" in flavors and vendor == "AwsBedrock": + bedrock.append(entries("UiPathChatAnthropicBedrock", THINKING)) + if "AwsBedrockInvoke" in flavors: + bedrock.append(entries("UiPathChatBedrock", THINKING)) + lite.append("BEDROCK_INVOKE_CONFIGS") + if "AwsBedrockConverse" in flavors: + bedrock.append(entries("UiPathChatBedrockConverse", THINKING)) + lite.append("BEDROCK_CONVERSE_CONFIGS") + if bedrock: + out["bedrock"] = "[" + ", ".join(bedrock) + "]" + openai: list[str] = [] + if "OpenAiChatCompletions" in flavors: + openai.append(entries("UiPathAzureChatOpenAI", '{"reasoning_effort": "low"}' if gpt_reasoning(model) else None)) + lite.append("OPENAI_CONFIGS") + if "OpenAiResponses" in flavors: + if gpt_reasoning(model): + kw = '{"use_responses_api": True, "reasoning": {"effort": "low", "summary": "auto"}, "verbosity": "low"}' + else: + kw = '{"use_responses_api": True}' + openai.append(f'{{"model_class": UiPathAzureChatOpenAI, "model_kwargs": {kw}}}') + lite.append("OPENAI_RESPONSES_CONFIGS") + if openai: + out["openai"] = "[" + ", ".join(openai) + "]" + if lite: + out["litellm"] = " + ".join(lite) + return out + + +def inject(conftest: Path, model: str, expr: str) -> bool: + text = conftest.read_text(encoding="utf-8") + if f'"{model}"' in text: + return False + for name in re.findall(r"\b([A-Z][A-Z0-9_]+_CONFIGS)\b", expr): + if not re.search(rf"^{name} = \[", text, re.M): + raise SystemExit(f"{conftest}: config list {name} not found") + new, n = re.subn(r"(COMPLETIONS_MODELS_WITH_CONFIGS = \{\n)", rf'\1 "{model}": {expr},\n', text, count=1) + if n != 1: + raise SystemExit(f"{conftest}: COMPLETIONS_MODELS_WITH_CONFIGS dict not found") + conftest.write_text(new, encoding="utf-8") + return True + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--family", required=True) + ap.add_argument("--vendor", required=True) + ap.add_argument("--flavors", required=True, help="comma-separated SupportedApiFlavors") + ap.add_argument("--operations", default="NormalizedChat", help="comma-separated SupportedNormalizedOperations") + a = ap.parse_args() + flavors = {f.strip() for f in a.flavors.split(",") if f.strip()} + ops = {o.strip() for o in a.operations.split(",") if o.strip()} + p = plan(a.model, a.family, a.vendor, flavors, ops) + if not p: + raise SystemExit(f"no client covers flavors {sorted(flavors)} / operations {sorted(ops)}") + touched = [d for d, expr in p.items() if inject(Path(a.repo) / CLIENTS / d / "conftest.py", a.model, expr)] + for d, expr in p.items(): + print(f" {d:11s} <- {expr}") + print(f"injected {a.model} (flavors {','.join(sorted(flavors))}) into: {' '.join(touched) or '(already present)'}") + print(" ".join(p)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())