Skip to content

Commit 0a53ebd

Browse files
RobGeadaRobGeada
authored
Guardrails Overhaul: Autoconfig, oauth, and TLS (#537)
* Add guardrails auto configurator * Add automatic TLS configuration to AutoConfig, run orchestrator by default as https * Add TLS certs to gateway * Update oauth templates * Configure oauth, fix watchers * Address comments * Undo params.env changes * Fix failing tests * Change gateway tls to edge
1 parent 40e7150 commit 0a53ebd

23 files changed

+2528
-89
lines changed

‎api/gorch/v1alpha1/guardrailsorchestrator_types.go‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,30 @@ import (
2525
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
2626
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
2727

28+
type AutoConfig struct {
29+
// The name of the inference service that provides the vLLM generation model to-be-guardrailed
30+
InferenceServiceToGuardrail string `json:"inferenceServiceToGuardrail"`
31+
32+
/* Label key to use when automatically identifying guardrail detector inference services.
33+
If provided, all inference services with the label `$detectorServiceLabelToMatch: true` will be used as a guardrails detector.
34+
If not provided, the default match label is `trustyai/guardrails`, and the autoconfig will use all inference services with the label `trustyai/guardrails: true` as detectors.
35+
*/
36+
// +optional
37+
DetectorServiceLabelToMatch string `json:"detectorServiceLabelToMatch,omitempty"`
38+
}
39+
2840
// GuardrailsOrchestratorSpec defines the desired state of GuardrailsOrchestrator.
2941
type GuardrailsOrchestratorSpec struct {
3042
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
3143
// Important: Run "make" to regenerate code after modifying this file
32-
3344
// Number of replicas
3445
Replicas int32 `json:"replicas"`
3546
// Name of configmap containing generator,detector,and chunker arguments
36-
OrchestratorConfig *string `json:"orchestratorConfig"`
47+
// +optional
48+
OrchestratorConfig *string `json:"orchestratorConfig,omitempty"`
49+
// Settings governing the automatic configuration of the orchestrator. Replaces `OrchestratorConfig`.
50+
// +optional
51+
AutoConfig *AutoConfig `json:"autoConfig,omitempty"`
3752
// Boolean flag to enable/disable built-in detectors
3853
// +optional
3954
EnableBuiltInDetectors bool `json:"enableBuiltInDetectors,omitempty"`
@@ -46,6 +61,9 @@ type GuardrailsOrchestratorSpec struct {
4661
// List of orchestrator enviroment variables for configuring the OTLP exporter
4762
// +optional
4863
OtelExporter OtelExporter `json:"otelExporter,omitempty"`
64+
// Set log level in the orchestrator deployment
65+
// +optional
66+
LogLevel *string `json:"logLevel,omitempty"`
4967
}
5068

5169
// OtelExporter defines the environment variables for configuring the OTLP exporter.
@@ -90,12 +108,34 @@ type Condition struct {
90108
LastTransitionTime metav1.Time `json:"lastTransitionTime" description:"last time the condition transit from one status to another"`
91109
}
92110

111+
type DetectedService struct {
112+
Name string `json:"name,omitempty"`
113+
Type string `json:"type,omitempty"` // e.g. "generator" or "detector"
114+
Scheme string `json:"scheme,omitempty"` //e.g., "http" or "https"
115+
Hostname string `json:"hostname,omitempty"`
116+
Port string `json:"port,omitempty"`
117+
TLSSecret string `json:"tlsSecret,omitempty"`
118+
}
119+
120+
type AutoConfigState struct {
121+
GeneratedConfigMap *string `json:"generatedConfigMap,omitempty"`
122+
GeneratedGatewayConfigMap *string `json:"generatedGatewayConfigMap,omitempty"`
123+
LastGenerated string `json:"lastGenerated,omitempty"`
124+
GenerationService DetectedService `json:"generationService,omitempty"`
125+
DetectorServices []DetectedService `json:"detectorServices,omitempty"`
126+
ConfigurationHash string `json:"configurationHash,omitempty"`
127+
Status string `json:"status,omitempty"`
128+
Message string `json:"message,omitempty"`
129+
}
130+
93131
type GuardrailsOrchestratorStatus struct {
94132
Phase string `json:"phase,omitempty"`
95-
96133
// Conditions describes the state of the GuardrailsOrchestrator resource.
97134
// +optional
98135
Conditions []Condition `json:"conditions,omitempty"`
136+
// AutoConfigState describes information about the generated autoconfiguration
137+
// +optional
138+
AutoConfigState *AutoConfigState `json:"autoConfigState,omitempty"`
99139
}
100140

101141
// +kubebuilder:object:root=true

‎api/gorch/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 76 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/base/params.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ lmes-allow-online=true
1313
lmes-allow-code-execution=true
1414
guardrails-orchestrator-image=quay.io/trustyai/ta-guardrails-orchestrator:latest
1515
guardrails-built-in-detector-image=quay.io/trustyai/regex-detector:latest
16-
guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest
16+
guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest

‎config/crd/bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,23 @@ spec:
4040
spec:
4141
description: GuardrailsOrchestratorSpec defines the desired state of GuardrailsOrchestrator.
4242
properties:
43+
autoConfig:
44+
description: Settings governing the automatic configuration of the
45+
orchestrator. Replaces `OrchestratorConfig`.
46+
properties:
47+
detectorServiceLabelToMatch:
48+
description: |-
49+
Label key to use when automatically identifying guardrail detector inference services.
50+
If provided, all inference services with the label `$detectorServiceLabelToMatch: true` will be used as a guardrails detector.
51+
If not provided, the default match label is `trustyai/guardrails`, and the autoconfig will use all inference services with the label `trustyai/guardrails: true` as detectors.
52+
type: string
53+
inferenceServiceToGuardrail:
54+
description: The name of the inference service that provides the
55+
vLLM generation model to-be-guardrailed
56+
type: string
57+
required:
58+
- inferenceServiceToGuardrail
59+
type: object
4360
enableBuiltInDetectors:
4461
description: Boolean flag to enable/disable built-in detectors
4562
type: boolean
@@ -51,6 +68,9 @@ spec:
5168
description: ' Name of the configmap containing guadrails sidecar
5269
gateway arguments'
5370
type: string
71+
logLevel:
72+
description: Set log level in the orchestrator deployment
73+
type: string
5474
orchestratorConfig:
5575
description: Name of configmap containing generator,detector,and chunker
5676
arguments
@@ -82,15 +102,66 @@ spec:
82102
type: string
83103
type: object
84104
replicas:
85-
description: Number of replicas
105+
description: |-
106+
INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
107+
Important: Run "make" to regenerate code after modifying this file
108+
Number of replicas
86109
format: int32
87110
type: integer
88111
required:
89-
- orchestratorConfig
90112
- replicas
91113
type: object
92114
status:
93115
properties:
116+
autoConfigState:
117+
description: AutoConfigState describes information about the generated
118+
autoconfiguration
119+
properties:
120+
configurationHash:
121+
type: string
122+
detectorServices:
123+
items:
124+
properties:
125+
hostname:
126+
type: string
127+
name:
128+
type: string
129+
port:
130+
type: string
131+
scheme:
132+
type: string
133+
tlsSecret:
134+
type: string
135+
type:
136+
type: string
137+
type: object
138+
type: array
139+
generatedConfigMap:
140+
type: string
141+
generatedGatewayConfigMap:
142+
type: string
143+
generationService:
144+
properties:
145+
hostname:
146+
type: string
147+
name:
148+
type: string
149+
port:
150+
type: string
151+
scheme:
152+
type: string
153+
tlsSecret:
154+
type: string
155+
type:
156+
type: string
157+
type: object
158+
lastGenerated:
159+
type: string
160+
message:
161+
type: string
162+
status:
163+
type: string
164+
type: object
94165
conditions:
95166
description: Conditions describes the state of the GuardrailsOrchestrator
96167
resource.

‎config/overlays/odh/params.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ lmes-allow-online=true
1313
lmes-allow-code-execution=true
1414
guardrails-orchestrator-image=quay.io/opendatahub/ta-guardrails-orchestrator:latest
1515
guardrails-built-in-detector-image=quay.io/opendatahub/regex-detector:latest
16-
guardrails-sidecar-gateway-image=quay.io/opendatahub/vllm-orchestrator-gateway:latest
16+
guardrails-sidecar-gateway-image=quay.io/opendatahub/vllm-orchestrator-gateway:latest

‎config/overlays/rhoai/params.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ lmes-allow-online=false
1313
lmes-allow-code-execution=false
1414
guardrails-orchestrator-image=quay.io/trustyai/ta-guardrails-orchestrator:latest
1515
guardrails-built-in-detector-image=quay.io/trustyai/regex-detector:latest
16-
guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest
16+
guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest

‎config/rbac/role.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,14 @@ rules:
208208
- patch
209209
- update
210210
- watch
211+
- apiGroups:
212+
- serving.kserve.io
213+
resources:
214+
- servingruntimes
215+
verbs:
216+
- get
217+
- list
218+
- watch
211219
- apiGroups:
212220
- trustyai.opendatahub.io
213221
resources:

0 commit comments

Comments
 (0)