Merge pull request #213 from stfc/updates-to-galaxy

MAINT: make galaxy use secrets

Author: anish-mudaraddi

charts/dev/materials-galaxy/secrets-templates/materials-galaxy.yaml

@@ -0,0 +1,36 @@
+oauth2-proxy:
+
+  # IRIS IAM CLIENT CREDENTIALS
+  config:
+    clientID: iris-iam-client-id
+    clientSecret: iris-iam-secret
+    # Create a new secret with the following command
+    # openssl rand -base64 32 | head -c 32 | base64
+    cookieSecret: "XXXXXXXXXXXXXXXX"
+  
+  # if you want to setup galaxy where only specific emails have access
+  # USE EITHER THIS OR RESTRICT BY OIDC GROUP 
+  authenticatedEmailsFile:
+    # if you're using oidc - set this to false
+    enabled: true
+    restricted_access: |-
+      [email protected]
+      [email protected]
+  
+  # if you want to setup galaxy where only a specific IAM group has access
+  # USE EITHER THIS OR RESTRICT BY EMAILS ABOVE
+#  extraArgs:
+#    allowed-group: "stfc-cloud/admins"
+
+galaxy:
+  postgresql:
+      # a random consistent password for postgres galaxydbuser
+      # can't auto-generate a password because of argo-helm issues when it tries to reconcile
+      galaxyDatabasePassword: <random-password>
+  configs:
+    galaxy.yml:
+      galaxy:
+        # comma spaced list of admin emails
+        admin_users: "[email protected],[email protected]"
+
+

charts/dev/materials-galaxy/values.yaml

@@ -1,5 +1,4 @@
 galaxy:
-
   # RBAC is defined by admins once service is up
   rbac:
     enabled: true
@@ -98,8 +97,6 @@ galaxy:
         remote_user_secret: null
 
         # admin user config
-        # comma spaced list of admin emails - need to change this to have admins
-        admin_users: "[email protected]"
         allow_user_deletion: true
         allow_user_impersonation: true
 
@@ -350,11 +347,6 @@ galaxy:
 
 oauth2-proxy:
   # Oauth client configuration specifics
-  config:
-
-    #TODO: make this a secret
-    existingSecret: iris-iam-credentials
-
   extraArgs:
     upstream: "http://{{.Release.Name}}-nginx"
     redirect-url: "https://galaxy.example.com/oauth2/callback"

clusters/dev/galaxy/apps.yaml

@@ -29,6 +29,7 @@ metadata:
   namespace: argocd
 spec:
   goTemplate: true
+  goTemplateOptions: ["missingkey=invalid"]
   generators:
     - list:
         elements:
@@ -51,11 +52,6 @@ spec:
             chartName: longhorn
             namespace: longhorn-system
             valuesFile: ../../../clusters/dev/galaxy/argocd-setup-values.yaml
-          
-          - name: manila-csi
-            chartName: manila-csi
-            namespace: manila-csi
-            valuesFile: ../../../clusters/dev/galaxy/argocd-setup-values.yaml
 
           # disable galaxy - deploying manually to diagnose duplicate job bug
           # - name: materials-galaxy
@@ -81,6 +77,8 @@ spec:
         helm:
           valueFiles:
             - "{{.valuesFile}}"
+            # a hack to get optional secrets working
+            - secrets://{{ .secretsFile | default "../../../secrets/dummy-secret.yaml"}}
       destination:
         server: https://kubernetes.default.svc
         namespace: "{{.namespace}}"
@@ -92,19 +90,3 @@ spec:
           allowEmpty: true
         syncOptions:
           - CreateNamespace=true
-
-  templatePatch: |
-    {{- if eq .name "manila-csi" }}
-      spec:
-        ignoreDifferences:
-          - group: rbac.authorization.k8s.io
-            kind: ClusterRole
-            name: manila-csi-openstack-manila-csi-controllerplugin
-            jsonPointers:
-              - /rules
-          - group: rbac.authorization.k8s.io
-            kind: ClusterRole
-            name: manila-csi-openstack-manila-csi-nodeplugin
-            jsonPointers:
-              - /rules
-    {{- end }}

clusters/dev/galaxy/materials-galaxy-values.yaml

@@ -1,16 +1,10 @@
 oauth2-proxy:
+
+
   extraArgs:
-    # limit access to stfc-cloud-dev group
-    allowed-group: "stfc-cloud-dev"
     redirect-url: "https://galaxy.dev.nubes.stfc.ac.uk/oauth2/callback"
 
 galaxy:
-  configs:
-    galaxy.yml:
-      galaxy:
-        # comma spaced list of admin emails
-        admin_users: "[email protected]"
-
   ingress:
     hosts:
     - host: "galaxy.dev.nubes.stfc.ac.uk"

docs/apps/galaxy.md

@@ -88,15 +88,10 @@ This is because:
 
 3. Its seamless! Users will be redirected to IRIS IAM login page before accessing Galaxy.
 
-To set an allowed IAM Group you can add the following 
-```yaml
-oauth2-proxy:
-  extraArgs:
-    allowed-group: "my-group" # or name of your IRIS IAM group
-```
-
 Other oauth2 proxy config settings can be found here - https://oauth2-proxy.github.io/oauth2-proxy/configuration/overview/. Flags can be passed as `extraArgs`
 
+NOTE: see creating Sops secret about configuring oidc authorization to Galaxy 
+
 
 ### Configuring DNS + certs
 
@@ -128,41 +123,37 @@ galaxy:
 
 ## Pre-deployment steps
 
-### 1. Create Secret for IAM Credentials
+### 1a. Create Sops Secret (If using ArgoCD)
 
-Create a secret for IAM credentials you can do so by creating a file in `/tmp/iam-secret.yaml` and adding this config:
+Galaxy requires `iris-iam` client id and secret to be setup. 
 
-```yaml
-apiVersion: v1
-data:
-  client-id: "" # put client id here - remember to encode it in base64
-  client-secret: "" # put client secret here - remember to encode it in base64
-  cookie-secret: "" # any 32 digit alphanumeric
-kind: Secret
-metadata:
-  name: iris-iam-credentials
-  namespace: galaxy # make sure this matches namespace galaxy will be installed in 
-type: Opaque
-```	
-
-you can create a random cookie secret by running
-```bash
-openssl rand -base64 32 | head -c 32 | base64
-```
+Additionally, if you want to restrict access to galaxy using emails - these should be setup using a sops secret
+
+You can set these secrets using `sops` - template yaml files for setting these secrets can be found in `secret-templates`. See [secrets](../secrets.md) on how to set and encrypt these secrets using sops
+
+### 2. Set Secrets in sops (If not using ArgoCD)
+
+You'll need to configure IRIS-IAM credentials manually using the template yaml files. Copy these files to tmp directory to avoid committing secrets
 
-Then just apply the file 
 ```bash
-kubectl apply -f /tmp/iam-secret.yaml 
+cp charts/$env/$chartName/secret-templates/* /tmp/secret-templates
 ```
 
+### 3. Set Postgresql Galaxy User Password
+
+We need to set a consistent password for galaxy user to access the prosgresql database (under `galaxy.postgresql.galaxyDatabasePassword`)
+Alternatively you can use an existing secret in K8s using (`galaxy.postgresql.galaxyExistingSecret` and `galaxy.postgresql.galaxyExistingSecretKeyRef`)
+
+This is mandatory if you're using ArgoCD - see Common Problems No. 2
+
 ## Deployment 
 
 You can deploy the chart as standalone
 
 ```bash
 cd cloud-deployed-apps/charts/dev/materials-galaxy
 helm dependency upgrade .
-helm install my-galaxy-service . -n galaxy --create-namespace
+helm install my-materials-galaxy . -n materials-galaxy  --create-namespace -f /tmp/secret-templates/materials-galaxy.yaml
 ```
 
 or you can use argocd to install it - see [Deploying Apps](../deploying-apps.md)
@@ -173,3 +164,9 @@ or you can use argocd to install it - see [Deploying Apps](../deploying-apps.md)
 ### 1. Galaxy pods stuck initializing and db init job crashlooping
 
 **Solution**:  If you're using our longhorn chart, you will need to change `longhorn.persistence.migrateable` to `false` since RWX volumes are incompatible with this. Delete the pvc/pv and restart the job and it should work
+
+### 2. Galaxy spins up but reconciliation leads to various pods end up stuck in pending
+
+**Solution**: If you're using ArgoCD and not setting a consistent password for `postgresql.galaxyDatabasePassword` (or `postgresql.galaxyExistingSecret`) it will end up autogenerating a new password every time ArgoCD reconciles the Galaxy chart. 
+
+This will cause the init-db job to fail as it expects the old password to work. This issue can be mitigated by setting a consistent password. This relates to a helm issue around autogenerated secrets https://github.com/galaxyproject/galaxy-helm/issues/112

scripts/deploy.sh

@@ -14,7 +14,6 @@ if [ -z "$2" ]; then
   exit 1
 fi
 
-
 CLUSTER_NAME=$1
 ENVIRONMENT=$2
 
@@ -33,8 +32,7 @@ if ! kubectl get secret helm-secrets-private-keys -n argocd &> /dev/null; then
 fi
 
 # Installing cert-manager if it's not already installed (relevant for child clusters)
-kubectl get namespace cert-manager &> /dev/null || true
-if [[ $? -eq 0 ]]; then
+if kubectl get namespace cert-manager &> /dev/null; then
   echo "cert-manager already installed..."
 else
   echo "installing cert-manager"

secrets/dev/galaxy/materials-galaxy.yaml

@@ -0,0 +1,77 @@
+oauth2-proxy:
+    #ENC[AES256_GCM,data:mrdeIB3Nv/x4WLnuRtf/RimCebZCVUrawJHE7A==,iv:8bKF55aVjMHhwlW6Zrfc+vACeucfCCHnG3zJpBFc+yM=,tag:hHpGPxOCao3WwtPz5ah/Kg==,type:comment]
+    config:
+        clientID: ENC[AES256_GCM,data:fA2ePSbC0kd+GMXnamcGCjauMF4ICrrf1xHeKIQO6TMcP5Kk,iv:G3E626rJUNaunLCbqj9b1ieDhr63mxD6zxJBB/gu0yQ=,tag:xqwO+cHFZ81jLNZF1NuoCA==,type:str]
+        clientSecret: ENC[AES256_GCM,data:Z5yjU0gLBaqYGcPcKjjvJlbQY8jmseijt0Q0MZNXgZnNjAtW8GQJStZvIeJOlY2G14aWkYktCZyGfxVOzrMvjzRjI6uKjSKA9u8xMxD6xlr1uoLjFGE=,iv:IykhmrEfrbQxSw7v9b6jmOexdRkOFOrKK+4tGQ3cU0M=,tag:veeVZmwJ6B3LpicC36yumQ==,type:str]
+        #ENC[AES256_GCM,data:uassv+ORXcu726uwSNPzll+9ilpz6x19bkH5X4PgbdJc/uqLUm+gHbG8FJ0HQ6k=,iv:wTWpghowGnalACjsO6RWo6HagO8K+GYaNX5s2bkribk=,tag:AE7PfEzpUDxYy5mreEVWCw==,type:comment]
+        #ENC[AES256_GCM,data:bktxsU/8UwmxKaETZE/Q4r9xjLca1PC5pM4zXWo2TTQEjgB/W6zHu8fZolFx5w==,iv:hL+j99HxA9QOdUKj/nvgCsdFYQOJ4KzYx53gk5+JDUw=,tag:AimCeTrA8ouFLjh46VuRUw==,type:comment]
+        cookieSecret: ENC[AES256_GCM,data:l+pVAlzzHt1HkHBXOYQMMBRvEPPpgL0OlNAl1KLP+0onZTxpm/pEassWzgI=,iv:PzULUSFQvFPlvGyC3W2T0L6fcOYDom0cc1JXCbG7dSQ=,tag:tEH3dCg9CkJys891BQvVhg==,type:str]
+    authenticatedEmailsFile:
+        enabled: ENC[AES256_GCM,data:ZrYngw==,iv:jlvezGvNk9xYvLOvUOlTEIsCYGoovmm4TK8kGYDuW/k=,tag:Q2fTk9dmABVRwhznwFoIow==,type:bool]
+        restricted_access: ENC[AES256_GCM,data:naI9w7dNvnb88zOEdeQi4bmG8I6xmr2VRuzGBYU+/uze993/M6Fk9kqvkAC7cRgdhkuGzamnwsGauARcciY/AFZz/+aZzXwnpqfRFy3rtlx0bApc+pjtOqtBL/HAnzef/dq8j3eqbKPjTE4zsb/mAQvlH8VyAU2aF3/dR75YeUDcSRah8ersexk2g2r2KR0t7xJ4sQDWmZxi9EGsK99V5yrsrgw3FTAE1GLYUpw5UudCkC6dneJb3iyDqLi0LHT+dTwIBV3dQSgFV8mPV66mUrQT5sxJVlHSCyNvfclK/9miLgsiB/Kz1Z9W/iys6KLrCugtmsadATLH6ZsxVdDAz89tF0udrfVzmWstxxYGE8sy5yU6UAJnAxYhf/0CohToy/XjoJ0jqUXJAc3oQf1fB7MwtfXsp1xw/77RBYr6kZFS6awUIh7KnVi+UX2hDvQzWcrlZ2otJe1a/3kGi4UL9Py7sq7eZKmqYFqKycI+pYHn63W2V77OmtX41js5QVwUd6eCTKRpLb8LNtogQvhdUQnL6/b8syRRCfNvsubbisilZxgLS3rEcdqW7DzNOFk=,iv:mbl58rnmrwrz39bdhI/inoGGCaBcCAHj0dAELQ57JEo=,tag:JHTmTGBcNosqAjFl+Nygkg==,type:str]
+galaxy:
+    postgresql:
+        #ENC[AES256_GCM,data:MCh8ZsHTNCUMfGKsnHNv/y059jOchh3hD6pBvDDNOmQUvLCvaNzPMMqRYaxS+RtdJ8GL8R7Yfg==,iv:LxTMl3r6RGCkSinEjHef+2G6TDxiusZxmdOKZFuoHSM=,tag:urFSeeIH1qo6UI1bBblbiw==,type:comment]
+        #ENC[AES256_GCM,data:VYrPLdtahkqRMbw4uccj/xk695BIhSsSPGIZe1NWbNZ18ymTs7EwVcwvNy8xOgI9hCMSMiCXSA==,iv:h67d9d6C+ZTl+XxzMbPgtCMDCnuVfGNvZOXumh0ZAv4=,tag:UixwGTLQeuuG1MqVdl6Atw==,type:comment]
+        galaxyDatabasePassword: ENC[AES256_GCM,data:x7N59RQ5d51TQF3NOK1XqA==,iv:bUUD968sLAZeRn/v0wI5JqceFwtA04LpUrzBs3XkjOQ=,tag:mKzbT7IGxyHdpZTsaY/0xw==,type:str]
+    configs:
+        galaxy.yml:
+            galaxy:
+                #ENC[AES256_GCM,data:RmB2DIN1leHqA6+yDCL3r067j7zAzDceM05DhmLYGAZL3Q==,iv:KfBO4dzEKTybK2BLRkIqHdkFIfrYAlRzWXDTYrgEEP4=,tag:fu/Zv3L+f18sZIlX8RWWhg==,type:comment]
+                admin_users: ENC[AES256_GCM,data:SweYaLL+q/XaZYiQ4Aasa2B7+aGcloZAKjrxZheMLphf9uYWcz3/X2BfDuSJmiQbCl8hDBSVnipM0hssMHXGryiI5251qS7M/bPX0smOBd27DafIhobun5yb3cmg/13ppEKCcqo8vmB9RLupOTdo,iv:uDDS6pvnQ4bE4iqPxM8C+FJzxYzC5GrNaWdeL3fu9+U=,tag:q/IuLW7I8AwrM24BNQjCXg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1acqcungzwkt807d3jt94ngtdt0vhk9kec4ps4a22cpaah57jw4xsl7q4xc
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZdmpOYkltUHRnSUNoSi9W
+            UzRwN3NkS01pSmhEUGVUZzFCeERzTVZDY2hnCk5UbVZIUWo2VXpTTVNEaVdtaTIx
+            OHhCSXZsbjFxdzZWMnhoUTJPMlVDOTgKLS0tIDB3bHAvWDQvVVRMMExEODY2T3Fz
+            M2RJNE0rakFDR2hLNDYvN2wyZ1V4NjAKoYldmwafjnM+ZYjwtUEv3vGMzejfCk7W
+            Y95yjXtZxYad5XqPwVazYBGQCjTNyhbEgsSheQAHYZLLu3tL17clcg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1h3dmygqf4v6jg3nxk5sr9jkp27w3q83sqnqxdd5n92xf3w6fs5kshakrxn
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrVnBxZEs3WDMvNWpRa2Vy
+            cDNZWTRwbk5hdjh4QVJVQy9BZDRPTW1xS3hFCldVc1U1aEhMdHFPQXRwb0hjcElN
+            eWp6ZWtLNWtXRXRMbDRra3NsSVpVa2sKLS0tIGVCcmptZFN1bktoM0VpUmc3YkRN
+            ckg5My9tcFY4MUtMT2JTWkdqejEzSncKMCsx4V+4OJ61il6Q6CTQItuqzFoIOIgy
+            LkoT7afaqKfnTTvKHhDEHEdAEMDOkeOPL/BLlhXxmyeD1jEBxwhy4A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1xr298hh8ammzethfcdeh72c25wnrk3u2zlzxx78k4nfcq2rwpgqs9hljq8
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvSVhkUHBSODRjekpnT0FI
+            WlBSQlpacEU4d2htSll1bCtPYUhSWENzaW5rCmxmb0w3cEJoR0NDQXBpN29SbGND
+            Q3pMTkxnelVaVVA2R25iR0tlbXMwUW8KLS0tIFM2TTJydEJ5SmRzbW1XTEtDZnh3
+            cUIvN0dFWW9jMHBjdjJGSHd3aEZTeG8K9JtaswOcZljGLxIXKXVgG33TMRgViT6M
+            UOP6kZT3mgSLuERiP6qt+i/EYP/lHNsLrf/t5+EwXaYYITR/CeC3mw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12khufkd7z25eqgpjjyy0zcrq6kpjxzekmff5zhq7q54tajm4e58qul35x0
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNeGJjQXRZMVNxczdsQzM1
+            ZGU4ajN1MTE2dlhLcXN3bExpTVZIbHFlMlFnCjJNMEJtUXVJVjBMaXFkbDJVd1U1
+            ODNDK1JGR2wzWEk4SmowN1QvZlorQ0kKLS0tIDFraWpsay92SXhJcEFGSE5pNjFD
+            T3JtOGQyaDI0bllGSTFQVzRLWjVJR2sKSXluwjAnx9Bm19C9S6OVE/qpnACa4zy9
+            w5LKfHWeI3SU+ELrkppgRwo1OHCMZ06uZLYfUz6yDGvK93eVoxC/Pw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age16fufeddr0arrns268526gxethxgkh3g0euf8cn37kuwfmq3h23psutz4q8
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3M3JKSjlMK3lsQ3Noa24z
+            OTJkSWxHZ3hwUE5UQUJkYjViTFJNRmc4T2xnCmUxMEtFVEh4aTZoV3JUcEZoeFJr
+            bFpjcDA4UXRsekFuai9Db0JocjllVE0KLS0tIEpUZmJxbWQrRW1nQmx6eE1rbzBL
+            M3NjYVBUMkJwNXpRMms5dnRQU3FTeFEK3Xv8FnUAT2tzCwcq39/tRnLVCXyznrpq
+            ZvVqOqreF0cSflft3QSwfXHYLp8n/H8rX80eUUEtTf+veIgP99cZqQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-10-15T20:58:09Z"
+    mac: ENC[AES256_GCM,data:IwZVidMlOOCr3kcu5pgdm0I97Con30RrSZ6LuSbPLH4IPwttJEj6RiTC5kSos5+vAcy/mrlDtIcGrr3/t5CVs9f9+9PHasVOnEBts+moO3XISK1Ep+YbwOiF5lcHzq+rU2e2gJlnQ+QHjMB2yAzYI+DQBeEKc7+mG7cDcHgj+oA=,iv:HOL+/z3p8bXZtMYrjN9V9y/VazJd/PDovyYDQ+yfl5o=,tag:p1ulTRltYD1e4mih9gitnw==,type:str]
+    pgp: []
+    unencrypted_regex: ^(apiVersion|metadata|kind|type)$
+    version: 3.8.1

secrets/dummy-secret.yaml

@@ -0,0 +1,2 @@
+# This file is left purposefully empty to act as a "dummy" secrets file to 
+# get argocd appsets to function properly