diff --git a/.gitignore b/.gitignore index 3b56fe43..f71eaa6e 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,6 @@ cython_debug/ evaluation/dataset/* evaluation/reports/* evaluation/known_non_issues_data/* + +# Tekton EventListener generated config +deploy/tekton/eventlistener/benchmark-config.yaml diff --git a/deploy/Makefile b/deploy/Makefile index 2421d5be..0a0fbf23 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -10,7 +10,6 @@ NAMESPACE ?= $(shell oc config view --minify --output 'jsonpath={..namespace}') CO := oc --context $(CONTEXT) # Pipeline parameters (overrideable on the CLI): -REPO_REMOTE_URL ?= source/code/url HUMAN_VERIFIED_FILE_PATH ?= "" LLM_URL ?= http://<> @@ -22,7 +21,7 @@ PROJECT_NAME ?= project-name PROJECT_VERSION ?= project-version DOWNLOAD_REPO ?= false -REPO_REMOTE_URL ?= "" +REPO_REMOTE_URL ?= source/code/url REPO_LOCAL_PATH ?= /path/to/repo INPUT_REPORT_FILE_PATH ?= http://<> @@ -56,6 +55,10 @@ S3_INPUT_BUCKET_NAME ?= test GITHUB_REPO_URL ?= https://github.com/RHEcosystemAppEng/sast-ai-workflow.git ARGOCD_NAMESPACE ?= sast-ai +# EventListener Configuration +# Default uses K8s service DNS with namespace parameter +ORCHESTRATOR_API_URL ?= http://sast-ai-orchestrator.$(NAMESPACE).svc.cluster.local:80 + # Secret configuration (loaded from .env file) GITLAB_TOKEN ?= "" LLM_API_KEY ?= "" @@ -64,21 +67,34 @@ GOOGLE_SERVICE_ACCOUNT_JSON_PATH ?= ./service_account.json GCS_SERVICE_ACCOUNT_JSON_PATH ?= ./gcs_service_account.json DOCKER_CONFIG_PATH ?= $(HOME)/.config/containers/auth.json + # S3/Minio Configuration -S3_OUTPUT_BUCKET_NAME ?= "" +S3_OUTPUT_BUCKET_NAME ?= bucket-name AWS_ACCESS_KEY_ID ?= "" AWS_SECRET_ACCESS_KEY ?= "" S3_ENDPOINT_URL ?= "" - -.PHONY: deploy setup tasks secrets pipeline scripts configmaps run clean generate-prompts prompts argocd-deploy-mlops argocd-deploy-prod argocd-clean +.PHONY: deploy deploy-dev deploy-prod deploy-mlops setup tasks-dev tasks-prod tasks-mlops secrets pipeline scripts configmaps run clean generate-prompts prompts argocd-deploy-dev argocd-deploy-prod argocd-clean eventlistener eventlistener-clean # Unified deploy command # Usage: # make deploy # Deploy base (Google Drive, :latest) # make deploy ENV=mlops # Deploy MLOps (S3/Minio, :latest) # make deploy ENV=prod IMAGE_VERSION=1.2.3 # Deploy prod (Google Drive, versioned) -deploy: - @if [ "$(ENV)" = "prod" ] && [ -z "$(IMAGE_VERSION)" ]; then \ +deploy: deploy-$(ENV) + +deploy-dev: CONTAINER_IMAGE=$(IMAGE_REGISTRY)/$(IMAGE_NAME):latest +deploy-dev: setup tasks-dev argocd-deploy-dev + @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + @echo "🚀 SAST AI Workflow - Development Deployment" + @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + @echo " Environment: Development" + @echo " Container Image: $(CONTAINER_IMAGE)" + @echo "" + @echo "✅ Development deployment completed successfully!" + +deploy-prod: CONTAINER_IMAGE=$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) +deploy-prod: setup tasks-prod argocd-deploy-prod + @if [ -z "$(IMAGE_VERSION)" ]; then \ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \ echo "❌ ERROR: IMAGE_VERSION is required for production deployment"; \ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \ @@ -87,55 +103,49 @@ deploy: echo ""; \ echo "Available versions can be found at:"; \ echo "https://quay.io/repository/ecosystem-appeng/sast-ai-workflow?tab=tags"; \ - echo ""; \ exit 1; \ fi + +deploy-mlops: CONTAINER_IMAGE=$(IMAGE_REGISTRY)/$(IMAGE_NAME):latest +deploy-mlops: setup tasks-mlops argocd-deploy-mlops + @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "🚀 SAST AI Workflow - Deployment" + @echo "🤖 SAST AI Workflow - MLOps Benchmarking Deployment" @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @if [ "$(ENV)" = "mlops" ]; then \ - echo " Environment: MLOps"; \ - echo " Storage: S3/Minio output upload"; \ - echo " Container Image: $(IMAGE_REGISTRY)/$(IMAGE_NAME):latest"; \ - elif [ "$(ENV)" = "prod" ]; then \ - echo " Environment: Production"; \ - echo " Storage: Google Drive upload"; \ - echo " Container Image: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)"; \ - else \ - echo " Environment: Base"; \ - echo " Storage: Google Drive upload"; \ - echo " Container Image: $(IMAGE_REGISTRY)/$(IMAGE_NAME):latest"; \ - fi + @echo " Environment: MLOps (Benchmarking)" + @echo " Namespace: $(NAMESPACE)" + @echo " Container Image: $(CONTAINER_IMAGE)" + @echo " Orchestrator URL: $(ORCHESTRATOR_API_URL)" @echo "" - @if [ "$(ENV)" = "mlops" ]; then \ - $(MAKE) --no-print-directory ENV=mlops setup scripts tasks prompts configmaps argocd-deploy-mlops; \ - elif [ "$(ENV)" = "prod" ]; then \ - $(MAKE) --no-print-directory ENV=prod CONTAINER_IMAGE=$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) setup scripts tasks prompts configmaps argocd-deploy-prod; \ - else \ - $(MAKE) --no-print-directory setup scripts tasks prompts configmaps; \ - fi + @echo "🎯 Deploying EventListener..." + @sed -e 's|ORCHESTRATOR_API_URL_PLACEHOLDER|$(ORCHESTRATOR_API_URL)|g' \ + tekton/eventlistener/benchmark-config.yaml.example > tekton/eventlistener/benchmark-config.yaml + @$(CO) apply -k tekton/eventlistener/ -n $(NAMESPACE) || \ + { echo " ❌ Failed to deploy EventListener resources"; exit 1; } + @echo " ✓ EventListener deployed" + @echo "" + @echo "✅ MLOps deployment completed successfully!" -setup: - @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - @echo "🚀 SAST AI Workflow - Infrastructure Setup" +setup: secrets scripts prompts configmaps @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + @echo "🚀 Common Infrastructure Ready" @echo " Context: $(CONTEXT)" @echo " Namespace: $(NAMESPACE)" - @echo "" - @$(MAKE) --no-print-directory secrets -tasks: - @echo "📋 Setting up Tekton Resources..." - @if [ "$(ENV)" = "prod" ]; then \ - $(CO) apply -k tekton/overlays/prod -n $(NAMESPACE) && \ - echo " ✓ Tekton resources deployed (production overlay)"; \ - elif [ "$(ENV)" = "mlops" ]; then \ - $(CO) apply -k tekton/overlays/mlops -n $(NAMESPACE) && \ - echo " ✓ Tekton resources deployed (mlops overlay - S3 output storage)"; \ - else \ - $(CO) apply -k tekton/base -n $(NAMESPACE) && \ - echo " ✓ Tekton resources deployed (base - Google Drive storage)"; \ - fi +tasks-dev: + @echo "📋 Deploying Tekton resources (dev)..." + @$(CO) apply -k tekton/base -n $(NAMESPACE) + @echo " ✓ Base Tekton resources (base - Google Drive storage)" + +tasks-prod: + @echo "📋 Deploying Tekton resources (prod)..." + @$(CO) apply -k tekton/overlays/prod -n $(NAMESPACE) + @echo " ✓ Production Tekton resources (versioned)" + +tasks-mlops: + @echo "📋 Deploying Tekton resources (mlops)..." + @$(CO) apply -k tekton/overlays/mlops -n $(NAMESPACE) + @echo " ✓ MLOps Tekton resources (MinIO/S3)" secrets: @echo "🔐 Configuring Secrets..." @@ -248,10 +258,6 @@ secrets: { echo " ❌ Failed to patch pipeline service account"; exit 1; } @echo " ✓ Service account configured" -pipeline: - @echo "🔧 Pipeline..." - @echo " ✓ Pipeline deployed with Tekton resources (via kustomize)" - scripts: @echo "📜 Setting up Scripts..." @$(CO) apply -n $(NAMESPACE) -f tekton/scripts/upload_to_drive_cm.yaml || \ @@ -294,7 +300,7 @@ run: @echo " Container Image: $(CONTAINER_IMAGE)" @echo " 🔄 Removing old pipeline runs..." @$(CO) delete pipelinerun sast-ai-workflow-pipelinerun \ - -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 + -n $(NAMESPACE) --ignore-not-found # Create PipelineRun with current parameters @sed \ -e 's|PROJECT_NAME_PLACEHOLDER|$(PROJECT_NAME)|g' \ @@ -362,6 +368,39 @@ argocd-clean: $(CO) patch application sast-ai-tekton-pipeline-syncer-prod -n $(NAMESPACE) -p '{"metadata":{"finalizers":null}}' --type=merge > /dev/null 2>&1 || true @echo " ✓ ArgoCD Applications removed" +eventlistener: + @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + @echo "🎯 EventListener Standalone Update" + @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + @echo " ⚠️ Use 'make deploy-mlops' for full deployment" + @echo "" + @echo "Using namespace: $(NAMESPACE)" + @echo "Using orchestrator URL: $(ORCHESTRATOR_API_URL)" + @echo "" + @echo "🎯 Deploying EventListener..." + @sed -e 's||$(NAMESPACE)|g' \ + tekton/eventlistener/benchmark-config.yaml.template > tekton/eventlistener/benchmark-config.yaml + @$(CO) apply -k tekton/eventlistener/ -n $(NAMESPACE) || \ + { echo " ❌ Failed to deploy EventListener resources"; exit 1; } + @echo "" + @echo "✅ EventListener updated" + @echo "" + @echo "📊 Verify: oc get eventlistener,task,pipeline -l app.kubernetes.io/component=benchmark-mlop -n $(NAMESPACE)" + @echo "🧪 Test: cd tekton/eventlistener && ./test-eventlistener.sh" + @echo "" + +eventlistener-clean: + @echo "🧹 Removing EventListener resources..." + @echo " 🏃 Cleaning benchmark PipelineRuns..." + @$(CO) delete pipelinerun -l app.kubernetes.io/component=benchmark-mlop -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true + @echo " ✓ Benchmark PipelineRuns removed" + @echo " 📋 Cleaning benchmark TaskRuns..." + @$(CO) delete taskrun -l app.kubernetes.io/component=benchmark-mlop -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true + @echo " ✓ Benchmark TaskRuns removed" + @echo " 🗑️ Removing EventListener infrastructure..." + @$(CO) delete -k tekton/eventlistener/ -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true + @echo " ✓ EventListener resources removed" + clean: @echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @echo "🧹 SAST AI Workflow - Cleanup" @@ -384,6 +423,9 @@ clean: @if [ "$(ENV)" = "prod" ]; then \ $(CO) delete -k tekton/overlays/prod -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true; \ echo " ✓ Production Tekton resources removed (kustomize overlay)"; \ + elif [ "$(ENV)" = "mlop" ]; then \ + $(CO) delete -k tekton/overlays/mlop -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true; \ + echo " ✓ MLOp Tekton resources removed (kustomize overlay)"; \ else \ $(CO) delete -k tekton/base -n $(NAMESPACE) --ignore-not-found > /dev/null 2>&1 || true; \ echo " ✓ Base Tekton resources removed (kustomize base)"; \ @@ -443,7 +485,7 @@ clean: @echo "🔐 Removing Secrets..." @$(CO) delete secret sast-ai-gitlab-token \ sast-ai-default-llm-creds \ - sast-ai-google-drive-service-account \ + sast-ai-google-service-account \ sast-ai-gcs-service-account \ sast-ai-s3-output-credentials \ sast-ai-quay-registry-config \ diff --git a/deploy/tekton/eventlistener/README.md b/deploy/tekton/eventlistener/README.md new file mode 100644 index 00000000..a11d8470 --- /dev/null +++ b/deploy/tekton/eventlistener/README.md @@ -0,0 +1,468 @@ +# Tekton EventListener for MLOps Benchmarking + +This directory contains a Tekton EventListener implementation that triggers the sast-ai-orchestrator MLOps batch API via webhook. This enables automated MLOps performance testing and benchmarking with DVC data versioning. + +## 🎯 Purpose + +Enable MLOps benchmark testing for batch SAST analysis jobs: +- ✅ Webhook-based triggering (curl/HTTP POST) +- ✅ Integration with sast-ai-orchestrator MLOps API (`/api/v1/mlops-batches`) +- ✅ DVC data versioning support +- ✅ Container image version testing +- ✅ Separation from production workflows +- ✅ Fork-friendly configuration + +## 📁 Directory Contents + +``` +eventlistener/ +├── README.md # This file +├── kustomization.yaml # Kustomize configuration +├── benchmark-config.yaml.template # ConfigMap template +├── benchmark-config.yaml # Generated ConfigMap (git-ignored) +├── call-orchestrator-api.yaml # Task that calls orchestrator MLOps API +├── poll-batch-status.yaml # Task that monitors batch completion +├── benchmark-pipeline.yaml # MLOps benchmark pipeline +├── eventlistener.yaml # EventListener + Service +├── triggerbinding.yaml # Extracts webhook parameters (including MLOps params) +├── triggertemplate.yaml # Generates PipelineRuns +└── test-eventlistener.sh # Helper script for testing +``` + +**Note:** `benchmark-config.yaml` is automatically generated from `benchmark-config.yaml.template` when you run `make eventlistener` and is git-ignored. + +## 📋 Prerequisites + +- OpenShift/Kubernetes cluster with Tekton Pipelines installed +- `oc` or `kubectl` CLI tool +- `curl` for sending test requests +- (Optional) `tkn` CLI for easier pipeline management +- (Optional) `jq` for JSON parsing + +Check Tekton installation: +```bash +oc get pods -n openshift-pipelines +# or +kubectl get pods -n tekton-pipelines +``` + +## 🚀 Quick Start + +### Step 1: Deploy MLOps Pipeline + +First, ensure you have the MLOps pipeline deployed: + +```bash +cd deploy +make tasks ENV=mlop NAMESPACE=your-namespace +``` + +### Step 2: Deploy EventListener + +Deploy the EventListener (uses defaults for both namespace and URL): + +```bash +cd deploy +make eventlistener +``` + +**Default Configuration:** +- Namespace: Auto-detected from current `oc` context +- Orchestrator URL: `http://sast-ai-orchestrator..svc.cluster.local:80` +- Uses existing orchestrator service (matches Helm deployment) +- Uses automatic K8s service discovery +- No manual configuration needed + +**Override Options:** +```bash +# Override namespace only +make eventlistener NAMESPACE=custom-namespace + +# Override orchestrator URL only +make eventlistener ORCHESTRATOR_API_URL=http://custom-service.sast-ai.svc.cluster.local:8080 + +# Override both +make eventlistener \ + ORCHESTRATOR_API_URL=http://custom-service.custom-ns.svc.cluster.local:8080 \ + NAMESPACE=custom-ns +``` + +**Parameters:** +- `NAMESPACE` - Target namespace (optional, auto-detected from current context) +- `ORCHESTRATOR_API_URL` - Orchestrator service URL (optional, uses K8s service DNS default) + +### Step 3: Verify Orchestrator Service + +The workflow uses the orchestrator's existing Helm service. + +**Quick Verification:** +```bash +# Verify orchestrator service exists +oc get svc sast-ai-orchestrator -n your-namespace + +**Expected Service Configuration:** +- **Name**: `sast-ai-orchestrator` (from orchestrator's Helm chart) +- **Port**: 80 (maps to targetPort 8080) +- **Type**: ClusterIP +- **Endpoints**: Should show pod IP:8080 + +**What happens:** +- ✅ Validates required parameters +- ✅ Generates `benchmark-config.yaml` with orchestrator URL and API endpoint +- ✅ Deploys all EventListener resources via Kustomize +- ✅ Shows verification and testing commands + +**Note:** The EventListener always calls `/api/v1/mlops-batches` endpoint (hardcoded for MLOps benchmarking). + +Verify deployment: +```bash +oc get eventlistener,task,pipeline,cm -l app.kubernetes.io/component=benchmark-mlop -n your-namespace +``` + +### Step 4: Test the EventListener + +**Option A: Manual testing** + +1. Port-forward to the EventListener service (for testing from outside the cluster): +```bash +oc port-forward svc/el-benchmark-mlop-listener 8080:8080 -n your-namespace +``` + +**Note:** Port-forwarding is **only needed for external testing** (e.g., from your local machine). The EventListener service is already accessible within the cluster at: +``` +http://el-benchmark-mlop-listener..svc.cluster.local:8080 +``` + +2. In another terminal, send a test request from your local machine: +```bash +curl -X POST http://localhost:8080 \ + -H 'Content-Type: application/json' \ + -d '{ + "submitted_by": "manual-test", + "image_version": "v2.1.0", + "dvc_nvr_version": "v1.0.0", + "dvc_known_false_positives_version": "v1.0.0", + "dvc_prompts_version": "v1.0.0" + }' +``` + +**Optional:** Test with custom container image: +```bash +curl -X POST http://localhost:8080 \ + -H 'Content-Type: application/json' \ + -d '{ + "submitted_by": "version-test", + "image_version": "v2.1.0", + "dvc_nvr_version": "v1.0.0", + "dvc_known_false_positives_version": "v1.0.0", + "dvc_prompts_version": "v1.0.0" + }' +``` + +3. Watch the PipelineRun: +```bash +# With tkn CLI +tkn pipelinerun logs -L -f + +# With kubectl/oc +oc get pipelinerun -l app.kubernetes.io/component=benchmark-mlop +oc logs -l tekton.dev/pipelineTask=call-orchestrator-api -f +``` + +## 📊 Expected Results + +### Successful Test + +When everything works correctly, you should see: + +1. **EventListener Response** (HTTP 201): +```json +{ + "eventListener": "benchmark-mlop-listener", + "namespace": "your-namespace", + "eventID": "abc123..." +} +``` + +2. **PipelineRun Created**: +```bash +$ oc get pipelinerun -l app.kubernetes.io/component=benchmark-mlop +NAME SUCCEEDED REASON STARTTIME COMPLETIONTIME +benchmark-mlop-pipeline-abc123 True Succeeded 5m 2m +``` + +3. **Task Logs Show API Call**: +``` +========================================= +Calling Orchestrator MLOps Batch API +========================================= +Configuration: + Orchestrator URL: http://sast-ai-orchestrator... + API Endpoint: /api/v1/mlops-batches (MLOps benchmarking) + Image Version: v2.1.0 + DVC NVR Version: v1.0.0 + DVC Prompts Version: v1.0.0 + DVC Known False Positives Version: v1.0.0 + ... +✓ API call successful! +Batch ID: batch-12345 + +Polling batch status... +✓ Batch completed successfully! +``` + +### Troubleshooting + +#### EventListener Pod Not Running + +```bash +# Check pod status +oc get pods -l eventlistener=benchmark-mlop-listener + +# Check pod logs +oc logs -l eventlistener=benchmark-mlop-listener +``` + +**Common issues:** +- Service account `pipeline` doesn't exist (create with Tekton operator) +- RBAC permissions missing + +#### API Call Fails + +Check task logs for detailed error: +```bash +oc logs -l tekton.dev/pipelineTask=call-orchestrator-api --tail=100 +``` + +**Common issues:** +- Orchestrator URL incorrect in ConfigMap +- Orchestrator service not running: `oc get pods -l app=sast-ai-orchestrator` +- Network policy blocking connections +- DVC version parameters not provided in webhook payload + +#### Verify ConfigMap + +```bash +# View current configuration +oc get configmap benchmark-config -o yaml -n your-namespace + +# Update if needed - regenerate (uses current namespace by default) +cd deploy +make eventlistener + +# Or override namespace +make eventlistener NAMESPACE=your-namespace + +# Or with custom orchestrator URL +make eventlistener \ + ORCHESTRATOR_API_URL=http://custom-orchestrator-service.your-namespace.svc.cluster.local:8080 +``` + +## 🔧 Configuration Reference + +### Webhook Payload Format + +Send JSON payload with these fields: + +```json +{ + "submitted_by": "trigger-source", + "dvc_nvr_version": "v1.2.3", + "dvc_known_false_positives_version": "v1.2.3", + "dvc_prompts_version": "v1.2.3", + "image_version": "v2.0.0" +} +``` + +**Required Fields:** +- `dvc_nvr_version` - DVC NVR resource version +- `dvc_prompts_version` - DVC prompts resource version +- `dvc_known_false_positives_version` - DVC known false positives resource version + +**Optional Fields:** +- `submitted_by` - Defaults to "eventlistener-webhook" +- `image_version` - Defaults to "latest" (e.g., "v2.1.0", "sha-abc123") + +### ConfigMap Keys + +The `benchmark-config` ConfigMap is automatically generated by `make eventlistener`: + +| Key | Description | Example | +|-----|-------------|---------| +| `orchestrator-api-url` | Base URL of orchestrator service | `http://sast-ai-orchestrator..svc.cluster.local:80` | +| `api-batch-endpoint` | API endpoint path for MLOps batches | `/api/v1/mlops-batches` | + +**Note:** The `api-batch-endpoint` is automatically set to `/api/v1/mlops-batches` for MLOps benchmarking. + +**To regenerate:** Simply run `make eventlistener` again with updated parameters. + +### Pipeline Parameters + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `dvc-nvr-version` | string | **Yes** | - | DVC NVR resource version | +| `dvc-prompts-version` | string | **Yes** | - | DVC prompts resource version | +| `dvc-known-false-positives-version` | string | **Yes** | - | DVC known false positives resource version | +| `submitted-by` | string | No | `eventlistener-webhook` | Trigger source identifier | +| `image-version` | string | No | `latest` | Workflow image version for testing (tag only, e.g., "v2.1.0") | + +## 🎓 Understanding the Architecture + +### Flow Diagram + +``` +┌─────────┐ +│ curl │ POST JSON payload +│ webhook │────────────────────┐ +└─────────┘ │ + ▼ + ┌──────────────────────┐ + │ EventListener │ + │ (benchmark-mlop) │ + └──────────┬───────────┘ + │ Creates + ▼ + ┌──────────────────────┐ + │ PipelineRun │ + │ (auto-generated name)│ + └──────────┬───────────┘ + │ Executes + ▼ + ┌──────────────────────┐ + │ Pipeline │ + │ (benchmark-mlop) │ + └──────────┬───────────┘ + │ Runs Tasks + ▼ + ┌──────────────────────┐ + │ Task 1 │ + │ (call-orchestrator) │ + └──────────┬───────────┘ + │ Then + ▼ + ┌──────────────────────┐ + │ Task 2 │ + │ (poll-batch-status) │ + └──────────┬───────────┘ + │ Reads Config + ▼ + ┌──────────────────────┐ + │ ConfigMap │ + │ (benchmark-config) │ + └──────────┬───────────┘ + │ Uses URL + ▼ + ┌──────────────────────┐ + │ Orchestrator API │ + │ POST /api/v1/ │ + │ mlops-batches │ + │ (with DVC versions) │ + └──────────────────────┘ +``` + +### Component Responsibilities + +1. **EventListener**: Accepts webhook (exposed as Kubernetes Service), validates request, triggers pipeline + - Service name: `el-benchmark-mlop-listener` + - Internal cluster access: `http://el-benchmark-mlop-listener..svc.cluster.local:8080` + - External testing: Use `oc port-forward` (see testing section) +2. **TriggerBinding**: Extracts parameters from webhook JSON payload (including MLOps params) +3. **TriggerTemplate**: Generates PipelineRun with extracted parameters +4. **Pipeline**: Orchestrates task execution, monitors completion, handles results +5. **Task 1 (call-orchestrator-api)**: Calls orchestrator MLOps API with DVC version params +6. **Task 2 (poll-batch-status)**: Monitors batch completion until done or timeout +7. **ConfigMap**: Stores environment-specific configuration (orchestrator URL, API endpoint) + +## 🔄 Production Enhancements + +For production use, consider: + +### Automation + +1. **Create CronJob** for scheduled benchmarking +2. **Set up monitoring** (Prometheus metrics) +3. **Configure notifications** (Slack/email on completion/failure) +4. **Add retry logic** for transient failures + +### Production Deployment + +Deploy to dedicated namespace: + +```bash +# Create and switch to namespace +oc new-project sast-ai-benchmark + +# Deploy MLOps pipeline overlay (uses current namespace) +cd deploy +make tasks ENV=mlop + +# Deploy EventListener (auto-detects namespace from context) +make eventlistener + +# Verify orchestrator service exists (from orchestrator's Helm deployment) +oc get svc sast-ai-orchestrator -n sast-ai-benchmark +``` + +**Note:** The default configuration auto-detects the current namespace and uses `http://sast-ai-orchestrator..svc.cluster.local:80` (matches the orchestrator's existing Helm service). + +This creates both: +- The `mlop-sast-ai-workflow-pipeline` that the orchestrator will trigger +- The EventListener webhook endpoint for triggering benchmarks + +## 🧹 Cleanup + +To remove all MLOps benchmark resources: + +```bash +# From deploy directory - Recommended +cd deploy +make eventlistener-clean NAMESPACE=your-namespace + +# Or manual cleanup +oc delete -k deploy/tekton/eventlistener/ -n your-namespace + +# Or individually +oc delete eventlistener benchmark-mlop-listener -n your-namespace +oc delete pipeline benchmark-mlop-pipeline -n your-namespace +oc delete task call-orchestrator-api-mlop poll-batch-status-mlop -n your-namespace +oc delete configmap benchmark-config -n your-namespace +oc delete service el-benchmark-mlop-listener -n your-namespace +``` + +## 📚 Additional Resources + +- [Tekton Triggers Documentation](https://tekton.dev/docs/triggers/) +- [EventListener Guide](https://tekton.dev/docs/triggers/eventlisteners/) + +## 🤝 For Project Forks + +If you're using this project as a base for your own: + +1. **Switch to your namespace and deploy** (auto-detects namespace): + ```bash + oc project + cd deploy + make eventlistener + ``` + +2. **Ensure orchestrator service** is deployed: + ```bash + oc get svc sast-ai-orchestrator -n + + # Should show port 80 -> targetPort 8080 + # The workflow will use: http://sast-ai-orchestrator..svc.cluster.local:80 + ``` + +3. **Customize** labels and naming if needed (edit YAML files in `tekton/eventlistener/`) +4. **Test** with your orchestrator instance using `test-eventlistener.sh` +5. **Extend** pipeline with your specific requirements + +All configuration is passed as parameters - no manual file editing needed! + +## ❓ Questions or Issues? + +- Check troubleshooting section above +- Review EventListener logs: `oc logs -l eventlistener=benchmark-mlop-listener` +- Review task logs: `oc logs -l tekton.dev/pipelineTask=call-orchestrator-api` +- Validate ConfigMap: `oc get cm benchmark-config -o yaml` +- Test orchestrator connectivity from a pod diff --git a/deploy/tekton/eventlistener/benchmark-config.yaml.template b/deploy/tekton/eventlistener/benchmark-config.yaml.template new file mode 100644 index 00000000..1e65f3b0 --- /dev/null +++ b/deploy/tekton/eventlistener/benchmark-config.yaml.template @@ -0,0 +1,30 @@ +# MLOps Benchmark Configuration Template +# +# This is a TEMPLATE file. The Makefile generates benchmark-config.yaml from this template. +# The placeholder will be replaced with the actual namespace during deployment. +# +# Default K8s Service Pattern: +# Uses existing orchestrator service: http://sast-ai-orchestrator..svc.cluster.local:80 +# +# Matches the orchestrator's existing Helm service: +# - Service name: sast-ai-orchestrator +# - Port: 80 (service port) -> 8080 (target port) +# - Type: ClusterIP +# +# Note: The generated benchmark-config.yaml is git-ignored and should not be committed. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: benchmark-config + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +data: + # Orchestrator API base URL (cluster-internal service) + # Default: Uses existing orchestrator service (sast-ai-orchestrator:80) + orchestrator-api-url: "http://sast-ai-orchestrator..svc.cluster.local:80" + + # API endpoint path for MLOps batches + api-batch-endpoint: "/api/v1/mlops-batches" + diff --git a/deploy/tekton/eventlistener/benchmark-pipeline.yaml b/deploy/tekton/eventlistener/benchmark-pipeline.yaml new file mode 100644 index 00000000..d0787663 --- /dev/null +++ b/deploy/tekton/eventlistener/benchmark-pipeline.yaml @@ -0,0 +1,173 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: benchmark-mlop-pipeline + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + description: >- + MLOps benchmark pipeline for triggering batch SAST analysis via EventListener. + This pipeline calls the sast-ai-orchestrator MLOps API endpoint and monitors + batch completion. Designed for performance testing and MLOps workflows. + + params: + - name: submitted-by + type: string + description: "Trigger source identifier" + default: "eventlistener-webhook" + + - name: image-version + type: string + description: "Workflow image version for testing (e.g., v2.1.0, sha-abc123)" + default: "latest" + + # DVC version parameters (required) + - name: dvc-nvr-version + type: string + description: "DVC NVR resource version" + + - name: dvc-prompts-version + type: string + description: "DVC prompts resource version" + + - name: dvc-known-false-positives-version + type: string + description: "DVC known false positives resource version" + + - name: use-known-false-positive-file + type: string + description: "Whether to use known false positive file" + default: "true" + + tasks: + - name: call-orchestrator-api + taskRef: + name: call-orchestrator-api-mlop + params: + - name: submitted-by + value: $(params.submitted-by) + - name: image-version + value: $(params.image-version) + - name: dvc-nvr-version + value: $(params.dvc-nvr-version) + - name: dvc-prompts-version + value: $(params.dvc-prompts-version) + - name: dvc-known-false-positives-version + value: $(params.dvc-known-false-positives-version) + - name: use-known-false-positive-file + value: $(params.use-known-false-positive-file) + + - name: poll-batch-status + taskRef: + name: poll-batch-status-mlop + runAfter: + - call-orchestrator-api + params: + - name: batch-id + value: $(tasks.call-orchestrator-api.results.batch-id) + - name: poll-interval + value: "30" + - name: timeout + value: "480" + + results: + - name: batch-id + description: "Orchestrator batch job ID" + value: $(tasks.call-orchestrator-api.results.batch-id) + + - name: trigger-status + description: "Batch trigger status" + value: $(tasks.call-orchestrator-api.results.status) + + - name: final-status + description: "Final batch completion status" + value: $(tasks.poll-batch-status.results.final-status) + + - name: total-jobs + description: "Total jobs in batch" + value: $(tasks.poll-batch-status.results.total-jobs) + + - name: completed-jobs + description: "Completed jobs count" + value: $(tasks.poll-batch-status.results.completed-jobs) + + - name: failed-jobs + description: "Failed jobs count" + value: $(tasks.poll-batch-status.results.failed-jobs) + + finally: + - name: log-completion + params: + - name: batch-id + value: $(tasks.call-orchestrator-api.results.batch-id) + - name: trigger-status + value: $(tasks.call-orchestrator-api.results.status) + - name: final-status + value: $(tasks.poll-batch-status.results.final-status) + - name: total-jobs + value: $(tasks.poll-batch-status.results.total-jobs) + - name: completed-jobs + value: $(tasks.poll-batch-status.results.completed-jobs) + - name: failed-jobs + value: $(tasks.poll-batch-status.results.failed-jobs) + taskSpec: + params: + - name: batch-id + - name: trigger-status + - name: final-status + - name: total-jobs + - name: completed-jobs + - name: failed-jobs + steps: + - name: log + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + script: | + #!/bin/sh + echo "=========================================" + echo "Pipeline Execution Summary" + echo "=========================================" + echo "Batch ID: $(params.batch-id)" + echo "Trigger Status: $(params.trigger-status)" + echo "Final Status: $(params.final-status)" + echo "" + echo "Jobs Summary:" + echo " Total: $(params.total-jobs)" + echo " Completed: $(params.completed-jobs)" + echo " Failed: $(params.failed-jobs)" + echo "" + + if [ "$(params.final-status)" = "COMPLETED" ]; then + if [ "$(params.failed-jobs)" = "0" ]; then + echo "✓ All batch jobs completed successfully!" + echo "" + echo "Next steps:" + echo " 1. Review results in Google Drive" + echo " 2. Check individual job outputs" + echo " 3. Analyze metrics in MLflow (if configured)" + else + echo "⚠ Batch completed but some jobs failed" + echo "" + echo "Troubleshooting failed jobs:" + echo " 1. Check orchestrator logs: oc logs -l app=sast-ai-orchestrator" + echo " 2. Review failed PipelineRuns: oc get pr -l status=failed" + echo " 3. Check individual job logs" + fi + elif [ "$(params.final-status)" = "TIMEOUT" ]; then + echo "⚠ Batch monitoring timed out" + echo "" + echo "The batch may still be processing." + echo "Check status manually:" + echo " curl http://sast-ai-orchestrator/api/v1/job-batches/$(params.batch-id)" + else + echo "✗ Batch failed or encountered an error" + echo "" + echo "Troubleshooting:" + echo " 1. Check orchestrator logs: oc logs -l app=sast-ai-orchestrator" + echo " 2. Verify ConfigMap: oc get cm benchmark-config -o yaml" + echo " 3. Check batch status: curl http://sast-ai-orchestrator/api/v1/job-batches/$(params.batch-id)" + fi + + echo "=========================================" + diff --git a/deploy/tekton/eventlistener/call-orchestrator-api.yaml b/deploy/tekton/eventlistener/call-orchestrator-api.yaml new file mode 100644 index 00000000..b3dad80c --- /dev/null +++ b/deploy/tekton/eventlistener/call-orchestrator-api.yaml @@ -0,0 +1,224 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: call-orchestrator-api-mlop + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + description: >- + Calls the sast-ai-orchestrator MLOps batch API to trigger batch SAST + analysis jobs with DVC and S3 integration. Used for MLOps performance + testing and benchmark automation. + + params: + - name: submitted-by + type: string + description: "Identifier of who/what triggered this batch" + default: "tekton-eventlistener-mlop" + + - name: image-version + type: string + description: "Workflow image version for testing (e.g., v2.1.0, sha-abc123)" + default: "latest" + + # DVC version parameters (required) + - name: dvc-nvr-version + type: string + description: "DVC NVR resource version" + + - name: dvc-prompts-version + type: string + description: "DVC prompts resource version" + + - name: dvc-known-false-positives-version + type: string + description: "DVC known false positives resource version" + + - name: use-known-false-positive-file + type: string + description: "Whether to use known false positive file" + default: "true" + + results: + - name: batch-id + description: "Batch job ID returned by the orchestrator" + + - name: status + description: "Status of the API call (success/failed)" + + steps: + - name: call-orchestrator-api + image: quay.io/curl/curl:latest + env: + # Read orchestrator URL from ConfigMap + - name: ORCHESTRATOR_URL + valueFrom: + configMapKeyRef: + name: benchmark-config + key: orchestrator-api-url + + # Read API endpoint from ConfigMap + - name: API_ENDPOINT + valueFrom: + configMapKeyRef: + name: benchmark-config + key: api-batch-endpoint + + script: | + #!/bin/sh + set -e + + echo "=========================================" + echo "Calling Orchestrator MLOps Batch API" + echo "=========================================" + echo "" + + # Display configuration + echo "Configuration:" + echo " Orchestrator URL: $ORCHESTRATOR_URL" + echo " API Endpoint: $API_ENDPOINT" + echo " Submitted By: $(params.submitted-by)" + echo " Image Version: $(params.image-version)" + echo " DVC NVR Version: $(params.dvc-nvr-version)" + echo " DVC Prompts Version: $(params.dvc-prompts-version)" + echo " DVC Known False Positives Version: $(params.dvc-known-false-positives-version)" + echo " Use Known False Positive File: $(params.use-known-false-positive-file)" + echo "" + + # Validate required parameters + echo "Validating parameters..." + VALIDATION_FAILED=0 + + if [ -z "$ORCHESTRATOR_URL" ]; then + echo "ERROR: ORCHESTRATOR_URL is empty or not set" + echo " Check that ConfigMap 'benchmark-config' exists and has key 'orchestrator-api-url'" + VALIDATION_FAILED=1 + fi + + if [ -z "$API_ENDPOINT" ]; then + echo "ERROR: API_ENDPOINT is empty or not set" + echo " Check that ConfigMap 'benchmark-config' exists and has key 'api-batch-endpoint'" + VALIDATION_FAILED=1 + fi + + if [ -z "$(params.dvc-nvr-version)" ]; then + echo "ERROR: dvc-nvr-version parameter is required but empty" + VALIDATION_FAILED=1 + fi + + if [ -z "$(params.dvc-prompts-version)" ]; then + echo "ERROR: dvc-prompts-version parameter is required but empty" + VALIDATION_FAILED=1 + fi + + if [ -z "$(params.dvc-known-false-positives-version)" ]; then + echo "ERROR: dvc-known-false-positives-version parameter is required but empty" + VALIDATION_FAILED=1 + fi + + # Validate use-known-false-positive-file is a valid boolean + USE_KFP="$(params.use-known-false-positive-file)" + if [ "$USE_KFP" != "true" ] && [ "$USE_KFP" != "false" ]; then + echo "ERROR: use-known-false-positive-file must be 'true' or 'false', got: $USE_KFP" + VALIDATION_FAILED=1 + fi + + if [ $VALIDATION_FAILED -eq 1 ]; then + echo "" + echo "Parameter validation failed. Cannot proceed." + echo -n "failed" > $(results.status.path) + echo -n "error" > $(results.batch-id.path) + exit 1 + fi + + echo "✓ All parameters validated successfully" + echo "" + + # Construct full API URL by concatenating base URL with endpoint + FULL_API_URL="${ORCHESTRATOR_URL}${API_ENDPOINT}" + echo "Full API URL: $FULL_API_URL" + echo "" + + # Prepare JSON payload for MLOps endpoint + # Note: API expects camelCase field names + PAYLOAD=$(cat <&1) || { + echo "ERROR: curl command failed" + echo "This could mean:" + echo " - Orchestrator service is not reachable" + echo " - Network policy blocking connection" + echo " - Orchestrator URL is incorrect in ConfigMap" + echo "" + echo "Response: $RESPONSE" + echo -n "failed" > $(results.status.path) + echo -n "error" > $(results.batch-id.path) + exit 1 + } + + # Extract HTTP status and body + HTTP_STATUS=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2) + BODY=$(echo "$RESPONSE" | sed '/HTTP_STATUS:/d') + + echo "HTTP Status: $HTTP_STATUS" + echo "Response Body:" + echo "$BODY" | sed 's/^/ /' + echo "" + + # Check if request was successful + if [ "$HTTP_STATUS" -ge 200 ] && [ "$HTTP_STATUS" -lt 300 ]; then + echo "✓ API call successful!" + + # Extract batchId from response + # API returns: {"batchId":1, ...} + if echo "$BODY" | grep -q "batchId"; then + # Extract batchId using grep and cut (works without jq) + BATCH_ID=$(echo "$BODY" | grep -o '"batchId":[0-9]*' | cut -d':' -f2 || echo "unknown") + echo "Batch ID: $BATCH_ID" + echo -n "$BATCH_ID" > $(results.batch-id.path) + else + echo "Warning: Could not extract batchId from response" + echo -n "unknown" > $(results.batch-id.path) + fi + + echo -n "success" > $(results.status.path) + + else + echo "✗ API call failed with HTTP status: $HTTP_STATUS" + echo "" + echo "Possible issues:" + echo " - Orchestrator service returned an error" + echo " - Invalid payload format" + echo " - Batch sheet URL not accessible" + echo " - Missing required parameters" + echo "" + echo -n "failed" > $(results.status.path) + echo -n "error" > $(results.batch-id.path) + exit 1 + fi + + echo "" + echo "=========================================" + echo "Orchestrator API call completed" + echo "=========================================" + diff --git a/deploy/tekton/eventlistener/eventlistener.yaml b/deploy/tekton/eventlistener/eventlistener.yaml new file mode 100644 index 00000000..88c3dd69 --- /dev/null +++ b/deploy/tekton/eventlistener/eventlistener.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: EventListener +metadata: + name: benchmark-mlop-listener + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + serviceAccountName: pipeline + triggers: + - name: benchmark-mlop-trigger + interceptors: + # CEL Interceptor for validating required parameters + # Fails fast before creating PipelineRun if validation fails + - name: validate-required-params + ref: + name: cel + params: + - name: filter + value: >- + has(body.dvc_nvr_version) && body.dvc_nvr_version != '' && + has(body.dvc_prompts_version) && body.dvc_prompts_version != '' && + has(body.dvc_known_false_positives_version) && body.dvc_known_false_positives_version != '' + - name: overlays + value: + # Add validation status for debugging + - key: validation.passed + expression: "true" + bindings: + - ref: benchmark-mlop-binding + template: + ref: benchmark-mlop-template + +--- +apiVersion: v1 +kind: Service +metadata: + name: el-benchmark-mlop-listener + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop + eventlistener: benchmark-mlop-listener +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + # Only use the eventlistener label that Tekton guarantees + # Kustomization commonLabels would override other labels + eventlistener: benchmark-mlop-listener + diff --git a/deploy/tekton/eventlistener/kustomization.yaml b/deploy/tekton/eventlistener/kustomization.yaml new file mode 100644 index 00000000..2334d4b0 --- /dev/null +++ b/deploy/tekton/eventlistener/kustomization.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Namespace (can be overridden via kubectl -n or kustomize) +# namespace: sast-ai + +# Common labels applied to all resources +# Note: Don't use app.kubernetes.io/part-of as commonLabel +# because it conflicts with Tekton-managed labels on EventListener pods +commonLabels: + app.kubernetes.io/component: benchmark-mlop + +# Resources to deploy +resources: + - benchmark-config.yaml + - call-orchestrator-api.yaml + - poll-batch-status.yaml + - benchmark-pipeline.yaml + - triggerbinding.yaml + - triggertemplate.yaml + - eventlistener.yaml + +# ConfigMap generator (alternative to static file) +# Uncomment to generate ConfigMap from properties +# Note: Use K8s service DNS pattern for orchestrator URL +# configMapGenerator: +# - name: benchmark-config +# literals: +# - orchestrator-api-url=http://sast-ai-orchestrator..svc.cluster.local:80 +# - api-batch-endpoint=/api/v1/mlops-batches + diff --git a/deploy/tekton/eventlistener/poll-batch-status.yaml b/deploy/tekton/eventlistener/poll-batch-status.yaml new file mode 100644 index 00000000..cc00b74f --- /dev/null +++ b/deploy/tekton/eventlistener/poll-batch-status.yaml @@ -0,0 +1,267 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: poll-batch-status-mlop + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + description: >- + Polls the orchestrator batch status API until the batch job completes + (status COMPLETED or FAILED) or timeout is reached. + + params: + - name: batch-id + type: string + description: "Batch ID to monitor (from previous task)" + + - name: poll-interval + type: string + description: "Seconds to wait between status checks" + default: "30" + + - name: timeout + type: string + description: "Maximum time to wait in minutes" + default: "480" + + results: + - name: final-status + description: "Final batch status (COMPLETED, FAILED, or TIMEOUT)" + + - name: total-jobs + description: "Total number of jobs in the batch" + + - name: completed-jobs + description: "Number of successfully completed jobs" + + - name: failed-jobs + description: "Number of failed jobs" + + steps: + - name: poll-status + image: quay.io/curl/curl:latest + env: + - name: ORCHESTRATOR_URL + valueFrom: + configMapKeyRef: + name: benchmark-config + key: orchestrator-api-url + + # Read API endpoint from ConfigMap + - name: API_ENDPOINT + valueFrom: + configMapKeyRef: + name: benchmark-config + key: api-batch-endpoint + + - name: BATCH_ID + value: $(params.batch-id) + + - name: POLL_INTERVAL + value: $(params.poll-interval) + + - name: TIMEOUT_MINUTES + value: $(params.timeout) + + script: | + #!/bin/sh + set -e + + echo "=========================================" + echo "Polling Batch Status" + echo "=========================================" + echo "" + echo "Configuration:" + echo " Orchestrator URL: $ORCHESTRATOR_URL" + echo " API Endpoint: $API_ENDPOINT" + echo " Batch ID: $BATCH_ID" + echo " Poll Interval: ${POLL_INTERVAL}s" + echo " Timeout: ${TIMEOUT_MINUTES} minutes" + echo "" + + # Validate required parameters + echo "Validating parameters..." + VALIDATION_FAILED=0 + + if [ -z "$ORCHESTRATOR_URL" ]; then + echo "ERROR: ORCHESTRATOR_URL is empty or not set" + echo " Check that ConfigMap 'benchmark-config' exists and has key 'orchestrator-api-url'" + VALIDATION_FAILED=1 + fi + + if [ -z "$API_ENDPOINT" ]; then + echo "ERROR: API_ENDPOINT is empty or not set" + echo " Check that ConfigMap 'benchmark-config' exists and has key 'api-batch-endpoint'" + VALIDATION_FAILED=1 + fi + + if [ -z "$BATCH_ID" ]; then + echo "ERROR: batch-id parameter is empty or not set" + echo " This parameter should be passed from the previous task" + VALIDATION_FAILED=1 + elif [ "$BATCH_ID" = "error" ] || [ "$BATCH_ID" = "unknown" ]; then + echo "ERROR: Invalid batch-id: $BATCH_ID" + echo " The previous task may have failed to create the batch" + VALIDATION_FAILED=1 + fi + + # Validate poll-interval is a positive integer + if ! echo "$POLL_INTERVAL" | grep -qE '^[0-9]+$'; then + echo "ERROR: poll-interval must be a positive integer, got: $POLL_INTERVAL" + VALIDATION_FAILED=1 + elif [ "$POLL_INTERVAL" -le 0 ]; then + echo "ERROR: poll-interval must be greater than 0, got: $POLL_INTERVAL" + VALIDATION_FAILED=1 + fi + + # Validate timeout is a positive integer + if ! echo "$TIMEOUT_MINUTES" | grep -qE '^[0-9]+$'; then + echo "ERROR: timeout must be a positive integer, got: $TIMEOUT_MINUTES" + VALIDATION_FAILED=1 + elif [ "$TIMEOUT_MINUTES" -le 0 ]; then + echo "ERROR: timeout must be greater than 0, got: $TIMEOUT_MINUTES" + VALIDATION_FAILED=1 + fi + + if [ $VALIDATION_FAILED -eq 1 ]; then + echo "" + echo "Parameter validation failed. Cannot proceed." + echo -n "VALIDATION_FAILED" > $(results.final-status.path) + echo -n "0" > $(results.total-jobs.path) + echo -n "0" > $(results.completed-jobs.path) + echo -n "0" > $(results.failed-jobs.path) + exit 1 + fi + + echo "✓ All parameters validated successfully" + echo "" + + # Calculate timeout in seconds + TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + START_TIME=$(date +%s) + + # Construct status API URL using endpoint from ConfigMap + STATUS_URL="${ORCHESTRATOR_URL}${API_ENDPOINT}/${BATCH_ID}" + echo "Status API URL: $STATUS_URL" + echo "" + + # Initialize counters + POLL_COUNT=0 + + # Poll loop + while true; do + POLL_COUNT=$((POLL_COUNT + 1)) + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + + echo "----------------------------------------" + echo "Poll #${POLL_COUNT} (elapsed: ${ELAPSED}s)" + echo "----------------------------------------" + + # Check timeout + if [ $ELAPSED -ge $TIMEOUT_SECONDS ]; then + echo "✗ TIMEOUT reached after ${TIMEOUT_MINUTES} minutes" + echo "" + echo "Batch did not complete within the timeout period." + echo "You can check the status manually:" + echo " curl $STATUS_URL" + echo "" + echo -n "TIMEOUT" > $(results.final-status.path) + echo -n "0" > $(results.total-jobs.path) + echo -n "0" > $(results.completed-jobs.path) + echo -n "0" > $(results.failed-jobs.path) + exit 1 + fi + + # Call status API + RESPONSE=$(curl -s -w "\nHTTP_STATUS:%{http_code}" "$STATUS_URL" 2>&1) || { + echo "ERROR: Failed to call status API" + sleep $POLL_INTERVAL + continue + } + + # Parse response + HTTP_STATUS=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2) + BODY=$(echo "$RESPONSE" | sed '/HTTP_STATUS:/d') + + if [ "$HTTP_STATUS" != "200" ]; then + echo "ERROR: API returned status $HTTP_STATUS" + echo "Response: $BODY" + sleep $POLL_INTERVAL + continue + fi + + # Extract status fields (without jq) + STATUS=$(echo "$BODY" | grep -o '"status":"[^"]*"' | cut -d'"' -f4) + TOTAL=$(echo "$BODY" | grep -o '"totalJobs":[0-9]*' | cut -d':' -f2) + COMPLETED=$(echo "$BODY" | grep -o '"completedJobs":[0-9]*' | cut -d':' -f2) + FAILED=$(echo "$BODY" | grep -o '"failedJobs":[0-9]*' | cut -d':' -f2) + + echo "Status: $STATUS" + echo "Progress: $COMPLETED/$TOTAL jobs completed ($FAILED failed)" + + # Check if batch is done + case "$STATUS" in + COMPLETED) + echo "" + echo "=========================================" + echo "✓ Batch COMPLETED Successfully!" + echo "=========================================" + echo "Total Jobs: $TOTAL" + echo "Completed: $COMPLETED" + echo "Failed: $FAILED" + echo "" + + if [ "$FAILED" -gt 0 ]; then + echo "⚠ Warning: Some jobs failed ($FAILED out of $TOTAL)" + else + echo "✓ All jobs completed successfully!" + fi + + echo "" + echo "Total execution time: ${ELAPSED}s ($((ELAPSED / 60)) minutes)" + echo "=========================================" + + echo -n "COMPLETED" > $(results.final-status.path) + echo -n "$TOTAL" > $(results.total-jobs.path) + echo -n "$COMPLETED" > $(results.completed-jobs.path) + echo -n "$FAILED" > $(results.failed-jobs.path) + exit 0 + ;; + + FAILED) + echo "" + echo "=========================================" + echo "✗ Batch FAILED" + echo "=========================================" + echo "Total Jobs: $TOTAL" + echo "Completed: $COMPLETED" + echo "Failed: $FAILED" + echo "" + echo "Check orchestrator logs for details:" + echo " oc logs -l app=sast-ai-orchestrator" + echo "=========================================" + + echo -n "FAILED" > $(results.final-status.path) + echo -n "$TOTAL" > $(results.total-jobs.path) + echo -n "$COMPLETED" > $(results.completed-jobs.path) + echo -n "$FAILED" > $(results.failed-jobs.path) + exit 1 + ;; + + PROCESSING|PENDING) + echo "Batch still processing... waiting ${POLL_INTERVAL}s" + echo "" + sleep $POLL_INTERVAL + ;; + + *) + echo "Unknown status: $STATUS" + echo "Full response: $BODY" + sleep $POLL_INTERVAL + ;; + esac + done + diff --git a/deploy/tekton/eventlistener/test-eventlistener.sh b/deploy/tekton/eventlistener/test-eventlistener.sh new file mode 100755 index 00000000..8d5902c0 --- /dev/null +++ b/deploy/tekton/eventlistener/test-eventlistener.sh @@ -0,0 +1,377 @@ +#!/bin/bash + +# Test script for Tekton EventListener MLOps Benchmarking +# This script helps validate the EventListener setup and trigger test PipelineRuns + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +NAMESPACE="${NAMESPACE:-$(oc project -q 2>/dev/null || echo "default")}" +SERVICE_NAME="el-benchmark-mlop-listener" +LOCAL_PORT="${LOCAL_PORT:-8080}" + +echo -e "${BLUE}=========================================${NC}" +echo -e "${BLUE}Tekton EventListener MLOps Benchmark Test${NC}" +echo -e "${BLUE}=========================================${NC}" +echo "" +echo "Namespace: $NAMESPACE" +echo "Service: $SERVICE_NAME" +echo "Local Port: $LOCAL_PORT" +echo "" +echo "Environment variables (optional):" +echo " TRIGGER_SOURCE: ${TRIGGER_SOURCE:-manual-test} (argocd, webhook, jenkins, etc.)" +echo " IMAGE_VERSION: ${IMAGE_VERSION:-latest}" +echo " DVC_NVR_VERSION: ${DVC_NVR_VERSION:-(empty)}" +echo " DVC_PROMPTS_VERSION: ${DVC_PROMPTS_VERSION:-(empty)}" +echo " DVC_KNOWN_FALSE_POSITIVES_VERSION: ${DVC_KNOWN_FALSE_POSITIVES_VERSION:-(empty)}" +echo " USE_KNOWN_FP: ${USE_KNOWN_FP:-true}" +echo "" + +# Function to check if command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Function to check prerequisites +check_prerequisites() { + echo -e "${YELLOW}Checking prerequisites...${NC}" + + # Check for oc or kubectl + if command_exists oc; then + KUBECTL="oc" + echo -e "${GREEN}✓${NC} Found oc CLI" + elif command_exists kubectl; then + KUBECTL="kubectl" + echo -e "${GREEN}✓${NC} Found kubectl CLI" + else + echo -e "${RED}✗${NC} Neither oc nor kubectl found. Please install OpenShift or Kubernetes CLI." + exit 1 + fi + + # Check for curl + if ! command_exists curl; then + echo -e "${RED}✗${NC} curl not found. Please install curl." + exit 1 + fi + echo -e "${GREEN}✓${NC} Found curl" + + # Check for jq (optional) + if command_exists jq; then + echo -e "${GREEN}✓${NC} Found jq (for JSON parsing)" + HAS_JQ=true + else + echo -e "${YELLOW}⚠${NC} jq not found (optional, for pretty JSON output)" + HAS_JQ=false + fi + + # Check for tkn (optional) + if command_exists tkn; then + echo -e "${GREEN}✓${NC} Found tkn CLI (for watching PipelineRuns)" + HAS_TKN=true + else + echo -e "${YELLOW}⚠${NC} tkn CLI not found (optional, for easier pipeline monitoring)" + HAS_TKN=false + fi + + echo "" +} + +# Function to check if resources are deployed +check_deployment() { + echo -e "${YELLOW}Checking if EventListener resources are deployed...${NC}" + + # Check ConfigMap + if $KUBECTL get configmap benchmark-config -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} ConfigMap 'benchmark-config' exists" + else + echo -e "${RED}✗${NC} ConfigMap 'benchmark-config' not found" + echo "" + echo "Please deploy the EventListener resources first:" + echo " cd deploy" + echo " make eventlistener # Auto-detects namespace from current context" + exit 1 + fi + + # Check Tasks + if $KUBECTL get task call-orchestrator-api-mlop -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} Task 'call-orchestrator-api-mlop' exists" + else + echo -e "${RED}✗${NC} Task 'call-orchestrator-api-mlop' not found" + exit 1 + fi + + if $KUBECTL get task poll-batch-status-mlop -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} Task 'poll-batch-status-mlop' exists" + else + echo -e "${RED}✗${NC} Task 'poll-batch-status-mlop' not found" + exit 1 + fi + + # Check Pipeline + if $KUBECTL get pipeline benchmark-mlop-pipeline -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} Pipeline 'benchmark-mlop-pipeline' exists" + else + echo -e "${RED}✗${NC} Pipeline 'benchmark-mlop-pipeline' not found" + exit 1 + fi + + # Check EventListener + if $KUBECTL get eventlistener benchmark-mlop-listener -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} EventListener 'benchmark-mlop-listener' exists" + else + echo -e "${RED}✗${NC} EventListener 'benchmark-mlop-listener' not found" + exit 1 + fi + + # Check Service + if $KUBECTL get service "$SERVICE_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then + echo -e "${GREEN}✓${NC} Service '$SERVICE_NAME' exists" + else + echo -e "${RED}✗${NC} Service '$SERVICE_NAME' not found" + exit 1 + fi + + # Check if EventListener pod is running + POD_NAME=$($KUBECTL get pods -n "$NAMESPACE" -l eventlistener=benchmark-mlop-listener -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [ -n "$POD_NAME" ]; then + POD_STATUS=$($KUBECTL get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.phase}') + if [ "$POD_STATUS" = "Running" ]; then + echo -e "${GREEN}✓${NC} EventListener pod is running: $POD_NAME" + else + echo -e "${YELLOW}⚠${NC} EventListener pod status: $POD_STATUS" + fi + else + echo -e "${YELLOW}⚠${NC} EventListener pod not found (may still be starting)" + fi + + echo "" +} + +# Function to display ConfigMap configuration +show_config() { + echo -e "${YELLOW}Current Configuration:${NC}" + $KUBECTL get configmap benchmark-config -n "$NAMESPACE" -o jsonpath='{.data}' | grep -o '"[^"]*"[[:space:]]*:[[:space:]]*"[^"]*"' | sed 's/^/ /' || echo " (unable to read)" + echo "" +} + +# Function to start port-forward +start_port_forward() { + echo -e "${YELLOW}Starting port-forward to EventListener...${NC}" + echo "This will forward localhost:$LOCAL_PORT -> $SERVICE_NAME:8080" + echo "" + echo -e "${BLUE}Port-forward command:${NC}" + echo " $KUBECTL port-forward svc/$SERVICE_NAME $LOCAL_PORT:8080 -n $NAMESPACE" + echo "" + echo -e "${YELLOW}Note: Keep this terminal open. Use another terminal to send test requests.${NC}" + echo "" + + # Start port-forward + $KUBECTL port-forward "svc/$SERVICE_NAME" "$LOCAL_PORT:8080" -n "$NAMESPACE" +} + +# Function to send test request +send_test_request() { + echo -e "${YELLOW}Sending test request to EventListener...${NC}" + + # Generate timestamp for unique identification + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + + # Prepare payload with all MLOps parameters (using underscores to match TriggerBinding) + PAYLOAD=$(cat <&1) + + # Parse response + HTTP_STATUS=$(echo "$RESPONSE" | grep "HTTP_STATUS:" | cut -d':' -f2) + BODY=$(echo "$RESPONSE" | sed '/HTTP_STATUS:/d') + + echo "Response Status: $HTTP_STATUS" + echo "Response Body:" + if [ "$HAS_JQ" = true ] && echo "$BODY" | jq '.' >/dev/null 2>&1; then + echo "$BODY" | jq '.' + else + echo "$BODY" + fi + echo "" + + if [ "$HTTP_STATUS" -ge 200 ] && [ "$HTTP_STATUS" -lt 300 ]; then + echo -e "${GREEN}✓ EventListener accepted the request!${NC}" + echo "" + echo "A PipelineRun should have been created. Check with:" + if [ "$HAS_TKN" = true ]; then + echo " tkn pipelinerun list -n $NAMESPACE" + echo " tkn pipelinerun logs -L -f -n $NAMESPACE" + else + echo " $KUBECTL get pipelinerun -n $NAMESPACE" + echo " $KUBECTL get pipelinerun -n $NAMESPACE -l app.kubernetes.io/component=benchmark-mlop" + fi + else + echo -e "${RED}✗ EventListener returned an error${NC}" + echo "Check EventListener logs:" + echo " $KUBECTL logs -l eventlistener=benchmark-mlop-listener -n $NAMESPACE" + fi + echo "" +} + +# Function to watch pipeline runs +watch_pipelineruns() { + echo -e "${YELLOW}Watching recent benchmark PipelineRuns...${NC}" + echo "" + + if [ "$HAS_TKN" = true ]; then + echo "Using tkn to watch PipelineRuns:" + tkn pipelinerun list -n "$NAMESPACE" -l app.kubernetes.io/component=benchmark-mlop + echo "" + echo "To follow logs of the latest run:" + echo " tkn pipelinerun logs -L -f -n $NAMESPACE" + else + echo "Recent benchmark PipelineRuns:" + $KUBECTL get pipelinerun -n "$NAMESPACE" -l app.kubernetes.io/component=benchmark-mlop --sort-by=.metadata.creationTimestamp + echo "" + echo "To view logs:" + echo " $KUBECTL logs -l tekton.dev/pipelineTask=call-orchestrator-api -n $NAMESPACE --tail=100" + fi + echo "" + + echo -e "${YELLOW}Query PipelineRuns by trigger source:${NC}" + echo " # Manual tests:" + echo " $KUBECTL get pr -n $NAMESPACE -l sast-ai.redhat.com/trigger-source=manual-test" + echo "" + echo " # ArgoCD triggers:" + echo " $KUBECTL get pr -n $NAMESPACE -l sast-ai.redhat.com/trigger-source=argocd" + echo "" + echo " # All benchmark runs:" + echo " $KUBECTL get pr -n $NAMESPACE -l app.kubernetes.io/component=benchmark-mlop" + echo "" + + echo -e "${YELLOW}To clean up test PipelineRuns:${NC}" + echo " cd ../.. && make eventlistener-clean NAMESPACE=$NAMESPACE" + echo "" +} + +# Main menu +show_menu() { + echo -e "${BLUE}=========================================${NC}" + echo -e "${BLUE}What would you like to do?${NC}" + echo -e "${BLUE}=========================================${NC}" + echo "1. Check deployment and configuration" + echo "2. Start port-forward (keep terminal open)" + echo "3. Send test request (requires port-forward in another terminal)" + echo "4. Watch PipelineRuns" + echo "5. Show current configuration" + echo "6. Full test (port-forward in background, send request, watch)" + echo "0. Exit" + echo "" + read -p "Enter choice [0-6]: " choice + echo "" + + case $choice in + 1) + check_deployment + show_config + read -p "Press Enter to continue..." + show_menu + ;; + 2) + start_port_forward + ;; + 3) + send_test_request + read -p "Press Enter to continue..." + show_menu + ;; + 4) + watch_pipelineruns + read -p "Press Enter to continue..." + show_menu + ;; + 5) + show_config + read -p "Press Enter to continue..." + show_menu + ;; + 6) + check_deployment + show_config + echo -e "${YELLOW}Starting port-forward in background...${NC}" + $KUBECTL port-forward "svc/$SERVICE_NAME" "$LOCAL_PORT:8080" -n "$NAMESPACE" & + PF_PID=$! + sleep 3 + send_test_request + sleep 2 + watch_pipelineruns + kill $PF_PID 2>/dev/null || true + echo -e "${GREEN}Port-forward stopped${NC}" + ;; + 0) + echo "Exiting..." + exit 0 + ;; + *) + echo -e "${RED}Invalid choice${NC}" + show_menu + ;; + esac +} + +# Main execution +check_prerequisites + +# If script is run with argument, execute that action directly +if [ $# -gt 0 ]; then + case "$1" in + check|status) + check_deployment + show_config + ;; + port-forward|pf) + check_deployment + start_port_forward + ;; + test|trigger) + send_test_request + ;; + watch|logs) + watch_pipelineruns + ;; + *) + echo "Usage: $0 [check|port-forward|test|watch]" + exit 1 + ;; + esac +else + # Interactive mode + check_deployment + show_menu +fi + diff --git a/deploy/tekton/eventlistener/triggerbinding.yaml b/deploy/tekton/eventlistener/triggerbinding.yaml new file mode 100644 index 00000000..fea5860e --- /dev/null +++ b/deploy/tekton/eventlistener/triggerbinding.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: TriggerBinding +metadata: + name: benchmark-mlop-binding + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + params: + # Extract submitter information + # ArgoCD: "argocd-prod-sync" + # Direct webhook: custom value + - name: submitted-by + value: $(body.submitted_by) + + # Extract trigger source for tracking + # Examples: "argocd", "webhook", "manual-test", "jenkins", etc. + - name: trigger-source + value: $(body.trigger_source) + + # Workflow image version for testing + - name: image-version + value: $(body.image_version) + + # DVC version parameters (required) + - name: dvc-nvr-version + value: $(body.dvc_nvr_version) + + - name: dvc-prompts-version + value: $(body.dvc_prompts_version) + + - name: dvc-known-false-positives-version + value: $(body.dvc_known_false_positives_version) + + - name: use-known-false-positive-file + value: $(body.use_known_false_positive_file) + diff --git a/deploy/tekton/eventlistener/triggertemplate.yaml b/deploy/tekton/eventlistener/triggertemplate.yaml new file mode 100644 index 00000000..4aedaa7c --- /dev/null +++ b/deploy/tekton/eventlistener/triggertemplate.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: TriggerTemplate +metadata: + name: benchmark-mlop-template + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop +spec: + params: + # Parameters from TriggerBinding + - name: submitted-by + description: "Trigger source identifier" + default: "eventlistener-webhook" + + - name: trigger-source + description: "Tool that triggered the EventListener (argocd, webhook, manual-test, etc.)" + default: "webhook" + + - name: image-version + description: "Workflow image version for testing (e.g., v2.1.0, sha-abc123)" + default: "latest" + + # DVC version parameters (required) + - name: dvc-nvr-version + description: "DVC NVR resource version" + + - name: dvc-prompts-version + description: "DVC prompts resource version" + + - name: dvc-known-false-positives-version + description: "DVC known false positives resource version" + + - name: use-known-false-positive-file + description: "Whether to use known false positive file" + default: "true" + + resourcetemplates: + - apiVersion: tekton.dev/v1 + kind: PipelineRun + metadata: + # Use generateName for unique PipelineRun names + # Kubernetes will append random suffix: benchmark-mlop-pipeline-abc123 + generateName: benchmark-mlop-pipeline- + labels: + app.kubernetes.io/name: sast-ai-workflow + app.kubernetes.io/component: benchmark-mlop + sast-ai.redhat.com/trigger-source: $(tt.params.trigger-source) + tekton.dev/pipeline: benchmark-mlop-pipeline + annotations: + sast-ai.redhat.com/submitted-by: $(tt.params.submitted-by) + sast-ai.redhat.com/trigger-source: $(tt.params.trigger-source) + spec: + pipelineRef: + name: benchmark-mlop-pipeline + + params: + # Pass parameters to pipeline + - name: submitted-by + value: $(tt.params.submitted-by) + + - name: image-version + value: $(tt.params.image-version) + + - name: dvc-nvr-version + value: $(tt.params.dvc-nvr-version) + + - name: dvc-prompts-version + value: $(tt.params.dvc-prompts-version) + + - name: dvc-known-false-positives-version + value: $(tt.params.dvc-known-false-positives-version) + + - name: use-known-false-positive-file + value: $(tt.params.use-known-false-positive-file) + + # Timeout for the entire pipeline + timeouts: + pipeline: "2h" + diff --git a/deploy/tekton/overlays/mlops/kustomization.yaml b/deploy/tekton/overlays/mlops/kustomization.yaml index 64befd36..544ad110 100644 --- a/deploy/tekton/overlays/mlops/kustomization.yaml +++ b/deploy/tekton/overlays/mlops/kustomization.yaml @@ -1,6 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# Add suffix to create separate mlops pipeline without overriding base +nameSuffix: -mlops + resources: - ../../base