Skip to content

Commit 6eb5a70

Browse files
pinglinpinglin
committed
ci(compose,helm): add artifact-backend-worker (#1424)
This commit - adds artifact-backend-worker in the deployment suite.
1 parent 4be08e5 commit 6eb5a70

File tree

10 files changed

+283
-6
lines changed

10 files changed

+283
-6
lines changed

‎.env.secrets.component‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# Provide your API key for the AI vendors so that you can set the components up
22
# with default credentials.
33

4+
# Gemini API key is needed for the RAG feature in artifact-backend.
5+
CFG_RAG_MODEL_GEMINI_APIKEY=
6+
47
# OpenAI key is needed for the embedding feature in artifact-backend.
58
CFG_COMPONENT_SECRETS_OPENAI_APIKEY=
69

‎charts/core/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# core
22

3-
![Version: 0.1.65](https://img.shields.io/badge/Version-0.1.65-informational?style=flat-square) ![AppVersion: 0.54.0](https://img.shields.io/badge/AppVersion-0.54.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
3+
![Version: 0.1.70](https://img.shields.io/badge/Version-0.1.70-informational?style=flat-square) ![AppVersion: 0.58.0](https://img.shields.io/badge/AppVersion-0.58.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
44

55
The Helm chart of Instill Core
66

‎charts/core/templates/_helpers.tpl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,13 @@ artifact-backend
148148
{{- printf "%s-artifact-backend" (include "core.fullname" .) -}}
149149
{{- end -}}
150150

151+
{{/*
152+
artifact-backend-worker
153+
*/}}
154+
{{- define "core.artifactBackendWorker" -}}
155+
{{- printf "%s-artifact-backend-worker" (include "core.fullname" .) -}}
156+
{{- end -}}
157+
151158
{{/* artifact-backend service and container public port */}}
152159
{{- define "core.artifactBackend.publicPort" -}}
153160
{{- printf "8082" -}}

‎charts/core/templates/artifact-backend/configmap.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@ data:
6060
openfga:
6161
host: {{ include "core.openfga" . }}
6262
port: 8081
63+
temporal:
64+
hostport: {{ default (printf "%s:%s" (include "temporal.host" .) (include "temporal.frontend.grpcPort" .)) .Values.artifactBackend.temporal.hostPort }}
65+
namespace: {{ default "instill-core" .Values.artifactBackend.temporal.namespace }}
66+
apikey: {{ default "" .Values.artifactBackend.temporal.apiKey }}
67+
insecureskipverify: {{ default "true" .Values.artifactBackend.temporal.insecureSkipVerify }}
68+
metricsport: {{ include "temporal.frontend.metricsPort" . }}
6369
registry:
6470
host: {{ include "core.registry" . }}
6571
port: {{ include "core.registry.port" . }}

‎charts/core/templates/artifact-backend/deployment.yaml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,3 +182,128 @@ spec:
182182
tolerations:
183183
{{- toYaml . | nindent 8 }}
184184
{{- end }}
185+
---
186+
apiVersion: apps/v1
187+
kind: Deployment
188+
metadata:
189+
name: {{ include "core.artifactBackendWorker" . }}
190+
labels:
191+
{{- include "core.labels" . | nindent 4 }}
192+
app.kubernetes.io/component: artifact-backend-worker
193+
annotations:
194+
rollme: {{ randAlphaNum 5 | quote }}
195+
{{- with .Values.artifactBackendWorker.annotations }}
196+
{{- toYaml . | nindent 4 }}
197+
{{- end }}
198+
spec:
199+
strategy:
200+
type: {{ .Values.updateStrategy.type }}
201+
{{- if eq .Values.updateStrategy.type "RollingUpdate" }}
202+
rollingUpdate:
203+
maxSurge: {{ .Values.updateStrategy.rollingUpdate.maxSurge }}
204+
maxUnavailable: {{ .Values.updateStrategy.rollingUpdate.maxUnavailable }}
205+
{{- else}}
206+
rollingUpdate: null
207+
{{- end }}
208+
{{- if not .Values.artifactBackendWorker.autoscaling.enabled }}
209+
replicas: {{ .Values.artifactBackendWorker.replicaCount }}
210+
{{- end }}
211+
selector:
212+
matchLabels:
213+
{{- include "core.matchLabels" . | nindent 6 }}
214+
app.kubernetes.io/component: artifact-backend-worker
215+
template:
216+
metadata:
217+
labels:
218+
{{- include "core.matchLabels" . | nindent 8 }}
219+
app.kubernetes.io/component: artifact-backend-worker
220+
annotations:
221+
checksum/config: {{ include (print $.Template.BasePath "/artifact-backend/configmap.yaml") . | sha256sum }}
222+
{{- with .Values.artifactBackendWorker.podAnnotations }}
223+
{{- toYaml . | nindent 8 }}
224+
{{- end }}
225+
spec:
226+
# Distroless nonroot:nonroot is 65532:65532
227+
securityContext:
228+
runAsUser: 65532
229+
runAsGroup: 65532
230+
{{- if .Values.artifactBackend.serviceAccountName }}
231+
serviceAccountName: {{ .Values.artifactBackend.serviceAccountName }}
232+
{{- end }}
233+
automountServiceAccountToken: {{ .Values.artifactBackend.automountServiceAccountToken | default false }}
234+
terminationGracePeriodSeconds: 120
235+
{{- with .Values.imagePullSecrets }}
236+
imagePullSecrets:
237+
{{- toYaml . | nindent 8 }}
238+
{{- end }}
239+
initContainers:
240+
{{- with .Values.artifactBackendWorker.extraInitFirstContainers }}
241+
{{- toYaml . | nindent 8 }}
242+
{{- end }}
243+
{{- if .Values.database.enabled }}
244+
- name: wait-for-db
245+
image: {{ .Values.database.image.repository }}:{{ .Values.database.image.tag }}
246+
imagePullPolicy: {{ $.Values.database.image.pullPolicy }}
247+
command: ['sh', '-c', "until pg_isready; do echo waiting for db; sleep 2; done"]
248+
env:
249+
- name: PGHOST
250+
value: {{ include "core.database" . }}
251+
- name: PGUSER
252+
value: {{ include "core.database.username" . }}
253+
{{- end }}
254+
- name: wait-for-dependencies
255+
image: curlimages/curl:8.00.1
256+
command: ['sh', '-c']
257+
args:
258+
- >
259+
while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${ARTIFACT_BACKEND_HOST}:${ARTIFACT_BACKEND_PORT}/v1alpha/health/artifact)" != "200" ]]; do echo waiting for artifact-backend; sleep 1; done
260+
env:
261+
- name: ARTIFACT_BACKEND_HOST
262+
value: "{{ include "core.artifactBackend" . }}"
263+
- name: ARTIFACT_BACKEND_PORT
264+
value: "{{ include "core.artifactBackend.publicPort" . }}"
265+
{{- with .Values.artifactBackend.extraInitContainers }}
266+
{{- toYaml . | nindent 8 }}
267+
{{- end }}
268+
containers:
269+
- name: artifact-backend-worker
270+
image: {{ .Values.artifactBackend.image.repository }}:{{ .Values.artifactBackend.image.tag }}
271+
imagePullPolicy: {{ .Values.artifactBackend.image.pullPolicy }}
272+
livenessProbe:
273+
tcpSocket:
274+
port: rpc
275+
{{- if .Values.artifactBackendWorker.resources }}
276+
resources:
277+
{{- toYaml .Values.artifactBackendWorker.resources | nindent 12 }}
278+
{{- end }}
279+
command: ["./{{ .Values.artifactBackendWorker.commandName.worker }}"]
280+
env:
281+
{{- if .Values.artifactBackendWorker.extraEnv }}
282+
{{- toYaml .Values.artifactBackendWorker.extraEnv | nindent 12 }}
283+
{{- end }}
284+
volumeMounts:
285+
- name: config
286+
mountPath: {{ .Values.artifactBackend.configPath }}
287+
subPath: config.yaml
288+
{{- with .Values.artifactBackend.extraVolumeMounts }}
289+
{{- toYaml . | nindent 12 }}
290+
{{- end }}
291+
volumes:
292+
- name: config
293+
configMap:
294+
name: {{ include "core.artifactBackend" . }}
295+
{{- with .Values.artifactBackend.extraVolumes }}
296+
{{- toYaml . | nindent 8 }}
297+
{{- end }}
298+
{{- with .Values.artifactBackendWorker.nodeSelector }}
299+
nodeSelector:
300+
{{- toYaml . | nindent 8 }}
301+
{{- end }}
302+
{{- with .Values.artifactBackendWorker.affinity }}
303+
affinity:
304+
{{- toYaml . | nindent 8 }}
305+
{{- end }}
306+
{{- with .Values.artifactBackendWorker.tolerations }}
307+
tolerations:
308+
{{- toYaml . | nindent 8 }}
309+
{{- end }}

‎charts/core/templates/artifact-backend/hpa.yml‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ apiVersion: autoscaling/v2
33
kind: HorizontalPodAutoscaler
44
metadata:
55
name: {{ include "core.artifactBackend" . }}
6+
labels:
7+
{{- include "core.labels" . | nindent 4 }}
8+
app.kubernetes.io/component: artifact-backend
69
spec:
710
scaleTargetRef:
811
apiVersion: apps/v1
912
kind: Deployment
1013
name: {{ include "core.artifactBackend" . }}
11-
labels:
12-
{{- include "core.labels" . | nindent 4 }}
13-
app.kubernetes.io/component: artifact-backend
1414
minReplicas: {{ .Values.artifactBackend.autoscaling.minReplicas }}
1515
maxReplicas: {{ .Values.artifactBackend.autoscaling.maxReplicas }}
1616
metrics:
@@ -31,3 +31,37 @@ spec:
3131
averageUtilization: {{ . }}
3232
{{- end }}
3333
{{- end }}
34+
---
35+
{{- if .Values.artifactBackendWorker.autoscaling.enabled }}
36+
apiVersion: autoscaling/v2
37+
kind: HorizontalPodAutoscaler
38+
metadata:
39+
name: {{ include "core.artifactBackendWorker" . }}
40+
labels:
41+
{{- include "core.labels" . | nindent 4 }}
42+
app.kubernetes.io/component: artifact-backend-worker
43+
spec:
44+
scaleTargetRef:
45+
apiVersion: apps/v1
46+
kind: Deployment
47+
name: {{ include "core.artifactBackendWorker" . }}
48+
minReplicas: {{ .Values.artifactBackendWorker.autoscaling.minReplicas }}
49+
maxReplicas: {{ .Values.artifactBackendWorker.autoscaling.maxReplicas }}
50+
metrics:
51+
{{- with .Values.artifactBackendWorker.autoscaling.targetCPUUtilizationPercentage }}
52+
- type: Resource
53+
resource:
54+
name: cpu
55+
target:
56+
type: Utilization
57+
averageUtilization: {{ . }}
58+
{{- end }}
59+
{{- with .Values.artifactBackendWorker.autoscaling.targetMemoryUtilizationPercentage }}
60+
- type: Resource
61+
resource:
62+
name: memory
63+
target:
64+
type: Utilization
65+
averageUtilization: {{ . }}
66+
{{- end }}
67+
{{- end }}

‎charts/core/templates/pipeline-backend/configmap.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ data:
8989
namespace: {{ default "instill-core" .Values.pipelineBackend.temporal.namespace }}
9090
apikey: {{ default "" .Values.pipelineBackend.temporal.apiKey }}
9191
insecureskipverify: {{ default "true" .Values.pipelineBackend.temporal.insecureSkipVerify }}
92-
metricsport: 9090
92+
metricsport: {{ include "temporal.frontend.metricsPort" . }}
9393
otelcollector:
9494
enable: {{ index .Values "opentelemetry-collector" "enabled" }}
9595
host: {{ include "core.otel" . }}

‎charts/core/values.yaml‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,12 @@ artifactBackend:
620620
configPath: /artifact-backend/config/config.yaml
621621
# -- The host of instill-core
622622
instillCoreHost:
623+
# -- The configuration of Temporal
624+
temporal:
625+
hostPort:
626+
namespace:
627+
apiKey:
628+
insecureSkipVerify:
623629
# -- Set the service account to be used, default if left empty
624630
serviceAccountName: ""
625631
# -- Mount the service account token
@@ -679,6 +685,57 @@ artifactBackend:
679685
numberOfWorkers: 20
680686
blob:
681687
hostport: http://localhost:8080
688+
artifactBackendWorker:
689+
# -- Annotation for deployment
690+
annotations: {}
691+
# -- The command names to be executed
692+
commandName:
693+
worker: artifact-backend-worker
694+
# -- The path of configuration file for artifact-backend-worker
695+
configPath: /artifact-backend/config/config.yaml
696+
# -- The number of replica for artifact-backend-worker
697+
replicaCount: 1
698+
# -- Add extra env variables
699+
extraEnv: []
700+
# -- Mount external volumes
701+
# For example, mount a secret containing Certificate root CA to verify database
702+
# TLS connection.
703+
extraVolumes: []
704+
# - name: my-volume
705+
# secret:
706+
# secretName: my-secret
707+
extraVolumeMounts: []
708+
# - name: my-volume
709+
# mountPath: /etc/secrets/my-secret
710+
# readOnly: true
711+
# -- Add extra init first containers
712+
extraInitFirstContainers: []
713+
# extraInitFirstContainers:
714+
# - name: ...
715+
# image: ...
716+
# -- Add extra init containers
717+
extraInitContainers: []
718+
# extraInitContainers:
719+
# - name: ...
720+
# image: ...
721+
# -- Additional pod annotations
722+
podAnnotations: {}
723+
# -- The pod resource
724+
resources: {}
725+
nodeSelector: {}
726+
tolerations: []
727+
affinity: {}
728+
autoscaling:
729+
enabled: false
730+
minReplicas:
731+
maxReplicas:
732+
targetCPUUtilizationPercentage:
733+
targetMemoryUtilizationPercentage:
734+
podDisruptionBudget:
735+
enabled: false
736+
spec:
737+
minAvailable:
738+
maxUnavailable:
682739
# -- The configuration of console
683740
console:
684741
# -- Enable console deployment or not
@@ -1034,7 +1091,12 @@ openfga:
10341091
- name: wait-for-db
10351092
image: postgres:15-alpine
10361093
imagePullPolicy: IfNotPresent
1037-
command: ["sh", "-c", "until pg_isready; do echo waiting for db; sleep 2; done"]
1094+
command:
1095+
[
1096+
"sh",
1097+
"-c",
1098+
"until pg_isready; do echo waiting for db; sleep 2; done",
1099+
]
10381100
env:
10391101
- name: PGHOST
10401102
value: "core-database"

‎docker-compose-dev.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ services:
2828
- ${ARTIFACT_BACKEND_HOST_PRIVATEPORT}:${ARTIFACT_BACKEND_PRIVATEPORT}
2929
- ${ARTIFACT_BACKEND_HOST_PUBLICPORT}:${ARTIFACT_BACKEND_PUBLICPORT}
3030

31+
artifact_backend_worker:
32+
environment:
33+
CFG_SERVER_DEBUG: "true"
34+
CFG_SERVER_EDITION: ${EDITION}
35+
3136
model_backend:
3237
environment:
3338
CFG_SERVER_DEBUG: "true"

‎docker-compose.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ services:
164164
container_name: ${ARTIFACT_BACKEND_HOST}
165165
image: ${ARTIFACT_BACKEND_IMAGE}:${ARTIFACT_BACKEND_VERSION}
166166
restart: unless-stopped
167+
env_file: ${ENV_SECRETS_COMPONENT}
167168
environment:
168169
CFG_SERVER_PRIVATEPORT: ${ARTIFACT_BACKEND_PRIVATEPORT}
169170
CFG_SERVER_PUBLICPORT: ${ARTIFACT_BACKEND_PUBLICPORT}
@@ -212,6 +213,40 @@ services:
212213
pipeline_backend:
213214
condition: service_healthy
214215

216+
artifact_backend_worker:
217+
pull_policy: missing
218+
container_name: ${ARTIFACT_BACKEND_HOST}-worker
219+
image: ${ARTIFACT_BACKEND_IMAGE}:${ARTIFACT_BACKEND_VERSION}
220+
restart: unless-stopped
221+
env_file: ${ENV_SECRETS_COMPONENT}
222+
environment:
223+
CFG_SERVER_PRIVATEPORT: ${ARTIFACT_BACKEND_PRIVATEPORT}
224+
CFG_SERVER_PUBLICPORT: ${ARTIFACT_BACKEND_PUBLICPORT}
225+
CFG_SERVER_DEBUG: "false"
226+
CFG_SERVER_USAGE_ENABLED: ${USAGE_ENABLED}
227+
CFG_SERVER_INSTILLCOREHOST: http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT}
228+
CFG_SERVER_EDITION: ${EDITION}
229+
CFG_MGMTBACKEND_HOST: ${MGMT_BACKEND_HOST}
230+
CFG_MGMTBACKEND_PUBLICPORT: ${MGMT_BACKEND_PUBLICPORT}
231+
CFG_MGMTBACKEND_PRIVATEPORT: ${MGMT_BACKEND_PRIVATEPORT}
232+
CFG_PIPELINEBACKEND_HOST: ${PIPELINE_BACKEND_HOST}
233+
CFG_PIPELINEBACKEND_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT}
234+
CFG_PIPELINEBACKEND_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT}
235+
CFG_MODELBACKEND_HOST: ${MODEL_BACKEND_HOST}
236+
CFG_MODELBACKEND_PUBLICPORT: ${MODEL_BACKEND_PUBLICPORT}
237+
CFG_MODELBACKEND_PRIVATEPORT: ${MODEL_BACKEND_PRIVATEPORT}
238+
CFG_REGISTRY_HOST: ${REGISTRY_HOST}
239+
CFG_REGISTRY_PORT: ${REGISTRY_PORT}
240+
CFG_DATABASE_HOST: ${POSTGRESQL_HOST}
241+
CFG_DATABASE_PORT: ${POSTGRESQL_PORT}
242+
CFG_DATABASE_USERNAME: postgres
243+
CFG_DATABASE_PASSWORD: password
244+
CFG_INFLUXDB_URL: http://${INFLUXDB_HOST}:${INFLUXDB_PORT}
245+
entrypoint: ./artifact-backend-worker
246+
depends_on:
247+
artifact_backend:
248+
condition: service_healthy
249+
215250
model_backend:
216251
pull_policy: missing
217252
container_name: ${MODEL_BACKEND_HOST}

0 commit comments

Comments
 (0)