Lgt 24152 added privacy filter service poc

chart/templates/helpers/_backend_crons_shared.tpl

@@ -150,6 +150,15 @@ Shared environment variables for backend and crons services
 - name: INTEGRATIONS_SIEM_STREAMING-SERVICE_URL
   value: "{{ include "http.scheme" . }}://{{ include "data_streamer.name" . }}:8080/events/post"
 {{- end }}
+{{- if .Values.general.privacy_filter.enabled }}
+- name: PRIVACY_FILTER_ENABLED
+  value: "true"
+- name: PRIVACY_FILTER_URL
+  value: "http://{{ include "privacy_filter.name" . }}:8786"
+{{- else }}
+- name: PRIVACY_FILTER_ENABLED
+  value: "false"
+{{- end }}
 {{- end -}}
 
 {{/*

chart/templates/helpers/_helpers.tpl

@@ -629,6 +629,51 @@ Container SecurityContext of lightrun data_streamer
 {{- end -}}
 {{- end -}}
 
+{{/*
+###################
+## Privacy filter ##
+###################
+*/}}
+
+
+{{- define "privacy_filter.name" -}}
+{{ include "lightrun.fullname" . }}-privacy-filter
+{{- end -}}
+
+
+
+{{/*
+Create the name of the lightrun privacy_filter service account to use
+*/}}
+{{- define "privacy_filter.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "privacy_filter.name" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Container SecurityContext of lightrun privacy_filter
+*/}}
+{{- define "privacy_filter.containerSecurityContext" -}}
+{{/*Define a local variable baseSecurityContext with the minimum required securityContext on the container level*/}}
+{{- $readOnlyRootFilesystem := dict "readOnlyRootFilesystem" (.Values.general.readOnlyRootFilesystem) -}}
+{{- $baseSecurityContext := include "baseSecurityContext" . | fromYaml -}}
+{{- $localSecurityContext := mustMerge $baseSecurityContext $readOnlyRootFilesystem -}}
+{{/*If user provided values for containerSecurityContext, merge them with the baseSecurityContext*/}}
+{{/*Values passed by user will override defaults*/}}
+{{- if .Values.deployments.privacy_filter.containerSecurityContext -}}
+{{- $mergedSecurityContext := mergeOverwrite  $localSecurityContext (.Values.deployments.privacy_filter.containerSecurityContext | default dict) -}}
+{{- $mergedSecurityContext | toYaml -}}
+{{- else if kindIs "invalid" .Values.deployments.privacy_filter.containerSecurityContext -}}
+{{ default dict | toYaml -}}
+{{- else -}}
+{{/*use default values from baseSecurityContext*/}}
+{{- $localSecurityContext | toYaml -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 ###################
 ## Artifacts ##

chart/templates/privacy_filter/deployment.yaml

@@ -0,0 +1,110 @@
+{{ if .Values.general.privacy_filter.enabled }}
+{{- $privacyFilter := .Values.deployments.privacy_filter }}
+{{- $nodeSelector := deepCopy (.Values.general.nodeSelector | default dict) }}
+{{- if $privacyFilter.nodeSelector }}
+{{- $nodeSelector = deepCopy $privacyFilter.nodeSelector }}
+{{- end }}
+{{- if $privacyFilter.instanceFamily }}
+{{- $_ := set $nodeSelector "karpenter.k8s.aws/instance-family" $privacyFilter.instanceFamily }}
+{{- end }}
+{{- $tolerations := concat (.Values.general.tolerations | default list) ($privacyFilter.tolerations | default list) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "privacy_filter.name" . }}
+  {{- with .Values.deployments.privacy_filter.annotations }}
+  annotations:
+    {{ toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: {{ .Values.deployments.privacy_filter.replicas }}
+  selector:
+    matchLabels:
+      app: {{ include "privacy_filter.name" . }}
+  template:
+    metadata:
+      {{- with .Values.deployments.privacy_filter.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        app: {{ include "privacy_filter.name" . }}
+        {{- with .Values.deployments.privacy_filter.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ template "privacy_filter.serviceAccountName" . }}
+        {{- if .Values.deployments.privacy_filter.podSecurityContext }}
+      securityContext:
+        {{- toYaml .Values.deployments.privacy_filter.podSecurityContext | nindent 8 }}
+        {{- end }}
+      {{- if .Values.secrets.defaults.dockerhub_config }}
+      imagePullSecrets:
+      {{- if .Values.secrets.defaults.dockerhub_config.existingSecret }}
+        - name: {{ .Values.secrets.defaults.dockerhub_config.existingSecret }}
+      {{- else }}
+        - name: {{ include "secrets.dockerhub.name" . }}
+      {{- end -}}
+      {{- end }}
+      containers:
+        - name: {{ include "privacy_filter.name" . }}
+          securityContext: {{- include "privacy_filter.containerSecurityContext" . | indent 12 }}
+          resources:
+            {{- toYaml .Values.deployments.privacy_filter.resources | nindent 12 }}
+          image: {{ .Values.deployments.privacy_filter.image.repository }}:{{ .Values.deployments.privacy_filter.image.tag }}
+          imagePullPolicy: {{ .Values.deployments.privacy_filter.image.pullPolicy }}
+          {{- with .Values.deployments.privacy_filter.extraEnvs }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: 8786
+              protocol: TCP
+          # Model warm-up runs during startup; the startupProbe guards that window
+          # before liveness/readiness take over.
+          startupProbe:
+            httpGet:
+              path: {{ .Values.deployments.privacy_filter.readinessProbe.path }}
+              port: http
+            periodSeconds: {{ .Values.deployments.privacy_filter.startupProbe.periodSeconds }}
+            failureThreshold: {{ .Values.deployments.privacy_filter.startupProbe.failureThreshold }}
+          livenessProbe:
+            initialDelaySeconds: {{ .Values.deployments.privacy_filter.livenessProbe.initialDelaySeconds }}
+            timeoutSeconds: {{ .Values.deployments.privacy_filter.livenessProbe.timeoutSeconds }}
+            periodSeconds: {{ .Values.deployments.privacy_filter.livenessProbe.periodSeconds }}
+            successThreshold: {{ .Values.deployments.privacy_filter.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.deployments.privacy_filter.livenessProbe.failureThreshold }}
+            httpGet:
+              path: {{ .Values.deployments.privacy_filter.livenessProbe.path }}
+              port: http
+          readinessProbe:
+            initialDelaySeconds: {{ .Values.deployments.privacy_filter.readinessProbe.initialDelaySeconds }}
+            timeoutSeconds: {{ .Values.deployments.privacy_filter.readinessProbe.timeoutSeconds }}
+            periodSeconds: {{ .Values.deployments.privacy_filter.readinessProbe.periodSeconds }}
+            successThreshold: {{ .Values.deployments.privacy_filter.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.deployments.privacy_filter.readinessProbe.failureThreshold }}
+            httpGet:
+              path: {{ .Values.deployments.privacy_filter.readinessProbe.path }}
+              port: http
+          {{- with .Values.deployments.privacy_filter.extraVolumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- if $tolerations }}
+      tolerations:
+        {{ toYaml $tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $nodeSelector }}
+      nodeSelector:
+        {{ toYaml $nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.deployments.privacy_filter.affinity }}
+      affinity:
+        {{ toYaml .Values.deployments.privacy_filter.affinity | nindent 8 }}
+      {{- end }}
+      {{- with .Values.deployments.privacy_filter.extraVolumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{ end }}

chart/templates/privacy_filter/service.yaml

@@ -0,0 +1,23 @@
+{{ if .Values.general.privacy_filter.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "privacy_filter.name" . }}
+  {{- with .Values.deployments.privacy_filter.service.annotations }}
+  annotations:
+    {{ toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.deployments.privacy_filter.service.labels }}
+  labels:
+    {{ toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: 8786
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: {{ include "privacy_filter.name" . }}
+{{ end }}

chart/templates/privacy_filter/serviceaccount.yaml

@@ -0,0 +1,10 @@
+{{- if and .Values.serviceAccount.create .Values.general.privacy_filter.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ include "privacy_filter.name" . }}
+  name: {{ template "privacy_filter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+{{- end }}

chart/values.yaml

@@ -125,6 +125,13 @@ general:
       ## List of response status codes upon which the data streamer will not retry
       - 403
 
+  #####################
+  ## Privacy filter
+  #####################
+  privacy_filter:
+    enabled: false
+    ## If set to `true`, will deploy a privacy filter pod for PII redaction
+
   #####################
   ## Crons
   #####################
@@ -1043,6 +1050,61 @@ deployments:
       failureThreshold: 3
       path: /ready
 
+  privacy_filter:
+    replicas: 1
+    rollout_strategy: "RollingUpdate"
+    image:
+      repository: lightruncom/privacy-filter
+      tag: "latest"
+      pullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: "1500m"
+        memory: "5Gi"
+      limits:
+        cpu: "2"
+        memory: "6Gi"
+        # Set nvidia.com/gpu to request GPU nodes.
+        # nvidia.com/gpu: "1"
+    instanceFamily: ""
+    # AWS example: c9g, c8i, or g5g.
+    # A non-empty selector replaces general.nodeSelector.
+    nodeSelector: {}
+    tolerations: []
+    podLabels: {}
+    podAnnotations: {}
+    annotations: {} # deployment annotations
+    extraEnvs: []
+    podSecurityContext: {}
+    containerSecurityContext: {}
+    service:
+      annotations: {}
+      labels: {}
+    extraVolumes: []
+    extraVolumeMounts: []
+    podDisruptionBudget: {} # [minAvailable|maxUnavailable] either integer or percentage
+    topologySpreadConstraints: []
+    affinity: {}
+    # Model warm-up happens on startup; startupProbe covers up to
+    # periodSeconds * failureThreshold seconds before liveness/readiness apply.
+    startupProbe:
+      periodSeconds: 5
+      failureThreshold: 60
+    livenessProbe:
+      initialDelaySeconds: 0
+      periodSeconds: 15
+      timeoutSeconds: 5
+      successThreshold: 1
+      failureThreshold: 3
+      path: /healthz
+    readinessProbe:
+      initialDelaySeconds: 0
+      periodSeconds: 10
+      timeoutSeconds: 5
+      successThreshold: 1
+      failureThreshold: 3
+      path: /readyz
+
   artifacts:
     loglevel: notice
     useJsonLogFormat: false