fix(tmachine): reach the HA TLS gateway through its ClusterIP

Drop the port-forward restart removed by the ClusterIP change, resolve the Service DNS name covered by the server certificate to the ClusterIP, bound each authenticated API wait attempt, and collect diagnostics from every gateway Pod.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

Author: matthewgrossman

tests/ansible/playbooks/openshell-k3s-ha-tls.yaml

@@ -58,10 +58,13 @@
       environment:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
 
-    - name: Restart gateway forwarder after TLS upgrade
-      ansible.builtin.systemd_service:
-        name: openshell-k3s-port-forward.service
-        state: restarted
+    # The chart's server certificate covers the Service DNS name but not its
+    # ClusterIP, and the node resolver cannot reach cluster DNS.
+    - name: Resolve the gateway Service name to its ClusterIP
+      ansible.builtin.lineinfile:
+        path: /etc/hosts
+        regexp: '\sopenshell\.openshell\.svc\.cluster\.local$'
+        line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local"
 
     - name: Read chart client TLS Secret
       ansible.builtin.command:
@@ -99,12 +102,12 @@
         apply:
           become: false
       vars:
-        openshell_client_gateway_endpoint: https://localhost:17670
+        openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}"
 
     - name: Wait for authenticated gateway API
       become: false
       ansible.builtin.command:
-        argv: [/usr/local/bin/openshell, sandbox, list, --output, json]
+        argv: [timeout, 10s, /usr/local/bin/openshell, sandbox, list, --output, json]
       register: k3s_gateway_api
       until: k3s_gateway_api.rc == 0
       retries: 24

tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh

@@ -43,8 +43,13 @@ collect() {
   collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \
     -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions'
   collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp
-  collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps
-  collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps
+  # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer.
+  mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
+    --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null)
+  for pod in "${gateway_pods[@]}"; do
+    collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps
+    collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps
+  done
   collect 'Memory' free -m
   collect 'Disk' df -h / /var/lib/rancher/k3s
 } | sed -E \