feat: Add TrustyAI distribution

trustyai-distribution/Containerfile

@@ -0,0 +1,48 @@
+# WARNING: This file is auto-generated. Do not modify it manually.
+# Generated by: trustyai-distribution/build.py
+
+FROM registry.access.redhat.com/ubi9/python-312:latest
+WORKDIR /opt/app-root
+
+RUN pip install sqlalchemy # somehow sqlalchemy[asyncio] is not sufficient
+RUN pip install \
+    aiosqlite \
+    autoevals \
+    chardet \
+    datasets \
+    fastapi \
+    fire \
+    httpx \
+    kubernetes \
+    llama_stack_provider_lmeval==0.1.7 \
+    llama_stack_provider_trustyai_fms==0.1.4 \
+    matplotlib \
+    mcp>=1.8.1 \
+    nltk \
+    numpy \
+    openai \
+    opentelemetry-exporter-otlp-proto-http \
+    opentelemetry-sdk \
+    pandas \
+    pillow \
+    psycopg2-binary \
+    pymilvus>=2.4.10 \
+    pymongo \
+    pypdf \
+    redis \
+    requests \
+    scikit-learn \
+    scipy \
+    sentencepiece \
+    sqlalchemy[asyncio] \
+    tqdm \
+    transformers \
+    uvicorn
+RUN pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision
+RUN pip install --no-deps sentence-transformers
+RUN pip install --no-cache llama-stack==0.2.16
+RUN mkdir -p ${HOME}/.llama/providers.d ${HOME}/.cache
+COPY trustyai-distribution/run.yaml ${APP_ROOT}/run.yaml
+COPY trustyai-distribution/providers.d/ ${HOME}/.llama/providers.d/
+
+ENTRYPOINT ["python", "-m", "llama_stack.distribution.server.server", "--config", "/opt/app-root/run.yaml"]

trustyai-distribution/Containerfile.in

@@ -0,0 +1,11 @@
+FROM registry.access.redhat.com/ubi9/python-312:latest
+WORKDIR /opt/app-root
+
+RUN pip install sqlalchemy # somehow sqlalchemy[asyncio] is not sufficient
+{dependencies}
+RUN pip install --no-cache llama-stack==0.2.16
+RUN mkdir -p ${{HOME}}/.llama/providers.d ${{HOME}}/.cache
+COPY trustyai-distribution/run.yaml ${{APP_ROOT}}/run.yaml
+COPY trustyai-distribution/providers.d/ ${{HOME}}/.llama/providers.d/
+
+ENTRYPOINT ["python", "-m", "llama_stack.distribution.server.server", "--config", "/opt/app-root/run.yaml"]

trustyai-distribution/README.md

@@ -0,0 +1,48 @@
+# TrustyAI Distribution Build Instructions
+
+This directory contains the necessary files to build a TrustyAI compatible container image for the llama-stack.
+
+## Prerequisites
+
+- Python >=3.12
+- `llama` CLI tool installed: `pip install llama-stack`
+- Podman or Docker installed
+
+## Generating the Containerfile
+
+The Containerfile is auto-generated from a template. To generate it:
+
+1. Make sure you have the `llama` CLI tool installed
+2. Run the build script from root of this git repo:
+   ```bash
+   ./trustyai-distribution/build.py
+   ```
+
+This will:
+- Check for the llama CLI installation
+- Generate dependencies using `llama stack build`
+- Create a new `Containerfile` with the required dependencies
+
+## Editing the Containerfile
+
+The Containerfile is auto-generated from a template. To edit it, you can modify the template in `trustyai-distribution/Containerfile.in` and run the build script again.
+NEVER edit the generated `Containerfile` manually.
+
+## Building the Container Image
+
+Once the Containerfile is generated, you can build the image using either Podman or Docker:
+
+### Using Podman build image for x86_64
+```bash
+podman build --platform linux/amd64 -f trustyai-distribution/Containerfile -t trustyai .
+```
+
+## Notes
+
+- The generated Containerfile should not be modified manually as it will be overwritten the next time you run the build script
+
+## Push the image to a registry
+
+```bash
+podman push <build-ID> quay.io/trustyai/llama-stack:trustyai-distribution
+```

trustyai-distribution/build.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# Usage: ./trustyai-distribution/build.py
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+BASE_REQUIREMENTS = [
+    "llama-stack==0.2.16",
+]
+
+
+def check_llama_installed():
+    """Check if llama binary is installed and accessible."""
+    if not shutil.which("llama"):
+        print("Error: llama binary not found. Please install it first.")
+        sys.exit(1)
+
+
+def check_llama_stack_version():
+    """Check if the llama-stack version in BASE_REQUIREMENTS matches the installed version."""
+    try:
+        result = subprocess.run(
+            ["llama stack --version"],
+            shell=True,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        installed_version = result.stdout.strip()
+
+        # Extract version from BASE_REQUIREMENTS
+        expected_version = None
+        for req in BASE_REQUIREMENTS:
+            if req.startswith("llama-stack=="):
+                expected_version = req.split("==")[1]
+                break
+
+        if expected_version and installed_version != expected_version:
+            print("Error: llama-stack version mismatch!")
+            print(f"  Expected: {expected_version}")
+            print(f"  Installed: {installed_version}")
+            print(
+                "  If you just bumped the llama-stack version in BASE_REQUIREMENTS, you must update the version from .pre-commit-config.yaml"
+            )
+            sys.exit(1)
+
+    except subprocess.CalledProcessError as e:
+        print(f"Warning: Could not check llama-stack version: {e}")
+        print("Continuing without version validation...")
+
+
+def get_dependencies():
+    """Execute the llama stack build command and capture dependencies."""
+    cmd = "llama stack build --config trustyai-distribution/build.yaml --print-deps-only"
+    try:
+        result = subprocess.run(cmd, shell=True, capture_output=True, text=True, check=True)
+        # Categorize and sort different types of pip install commands
+        standard_deps = []
+        torch_deps = []
+        no_deps = []
+        no_cache = []
+
+        for line in result.stdout.splitlines():
+            if line.strip().startswith("uv pip"):
+                # Split the line into command and packages
+                parts = line.replace("uv ", "RUN ", 1).split(" ", 3)
+                if len(parts) >= 4:  # We have packages to sort
+                    cmd_parts = parts[:3]  # "RUN pip install"
+                    packages = sorted(set(parts[3].split()))  # Sort the package names and remove duplicates
+
+                    # Determine command type and format accordingly
+                    if "--index-url" in line:
+                        full_cmd = " ".join(cmd_parts + [" ".join(packages)])
+                        torch_deps.append(full_cmd)
+                    elif "--no-deps" in line:
+                        full_cmd = " ".join(cmd_parts + [" ".join(packages)])
+                        no_deps.append(full_cmd)
+                    elif "--no-cache" in line:
+                        full_cmd = " ".join(cmd_parts + [" ".join(packages)])
+                        no_cache.append(full_cmd)
+                    else:
+                        formatted_packages = " \\\n    ".join(packages)
+                        full_cmd = f"{' '.join(cmd_parts)} \\\n    {formatted_packages}"
+                        standard_deps.append(full_cmd)
+                else:
+                    standard_deps.append(" ".join(parts))
+
+        # Combine all dependencies in specific order
+        all_deps = []
+        all_deps.extend(sorted(standard_deps))  # Regular pip installs first
+        all_deps.extend(sorted(torch_deps))  # PyTorch specific installs
+        all_deps.extend(sorted(no_deps))  # No-deps installs
+        all_deps.extend(sorted(no_cache))  # No-cache installs
+
+        return "\n".join(all_deps)
+    except subprocess.CalledProcessError as e:
+        print(f"Error executing command: {e}")
+        print(f"Command output: {e.output}")
+        sys.exit(1)
+
+
+def generate_containerfile(dependencies):
+    """Generate Containerfile from template with dependencies."""
+    template_path = Path("trustyai-distribution/Containerfile.in")
+    output_path = Path("trustyai-distribution/Containerfile")
+
+    if not template_path.exists():
+        print(f"Error: Template file {template_path} not found")
+        sys.exit(1)
+
+    # Read template
+    with open(template_path) as f:
+        template_content = f.read()
+
+    # Add warning message at the top
+    warning = "# WARNING: This file is auto-generated. Do not modify it manually.\n# Generated by: trustyai-distribution/build.py\n\n"
+
+    # Process template using string formatting
+    containerfile_content = warning + template_content.format(dependencies=dependencies.rstrip())
+
+    # Write output
+    with open(output_path, "w") as f:
+        f.write(containerfile_content)
+
+    print(f"Successfully generated {output_path}")
+
+
+def main():
+    print("Checking llama installation...")
+    check_llama_installed()
+
+    print("Checking llama-stack version...")
+    check_llama_stack_version()
+
+    print("Getting dependencies...")
+    dependencies = get_dependencies()
+
+    print("Generating Containerfile...")
+    generate_containerfile(dependencies)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

trustyai-distribution/build.yaml

@@ -0,0 +1,36 @@
+version: 2
+distribution_spec:
+  description: TrustyAI distribution of Llama Stack
+  providers:
+    inference:
+    - provider_type: remote::vllm
+    - provider_type: inline::sentence-transformers
+    vector_io:
+    - provider_type: inline::milvus
+    safety:
+    - provider_type: remote::trustyai_fms
+    agents:
+    - provider_type: inline::meta-reference
+    eval:
+    - provider_type: remote::trustyai_lmeval
+    datasetio:
+    - provider_type: remote::huggingface
+    - provider_type: inline::localfs
+    scoring:
+    - provider_type: inline::basic
+    - provider_type: inline::llm-as-judge
+    - provider_type: inline::braintrust
+    telemetry:
+    - provider_type: inline::meta-reference
+    tool_runtime:
+    - provider_type: remote::brave-search
+    - provider_type: remote::tavily-search
+    - provider_type: inline::rag-runtime
+    - provider_type: remote::model-context-protocol
+  container_image: registry.redhat.io/ubi9/python-312:latest
+additional_pip_packages:
+- aiosqlite
+- sqlalchemy[asyncio]
+image_type: container
+image_name: llama-stack-trustyai
+external_providers_dir: trustyai-distribution/providers.d

trustyai-distribution/providers.d/remote/eval/trustyai_lmeval.yaml

@@ -0,0 +1,7 @@
+adapter:
+  adapter_type: trustyai_lmeval
+  pip_packages: ["kubernetes", "llama_stack_provider_lmeval==0.1.7"]
+  config_class: llama_stack_provider_lmeval.config.LMEvalEvalProviderConfig
+  module: llama_stack_provider_lmeval
+api_dependencies: ["inference"]
+optional_api_dependencies: []

trustyai-distribution/providers.d/remote/safety/trustyai_fms.yaml

@@ -0,0 +1,7 @@
+adapter:
+  adapter_type: trustyai_fms
+  pip_packages: ["llama_stack_provider_trustyai_fms==0.1.2"]
+  config_class: llama_stack_provider_trustyai_fms.config.FMSSafetyProviderConfig
+  module: llama_stack_provider_trustyai_fms
+api_dependencies: ["safety"]
+optional_api_dependencies: ["shields"]