Running more tests in parallel for faster presubmit outcome (#1256)

Now the tests finish < 2 hours for 2.1-deb11 which were earlier timing out in 4 hours, so atleast 2x faster.

kubectl run doesn't allow specifying CPU/Memory requirements in single line. The test cluster uses Autopilot so resources will be used only when tests are run.

Author: Deependra-Patel

cloudbuild/presubmit.sh

@@ -70,7 +70,6 @@ determine_tests_to_run() {
     changed_dir="${changed_dir%%/*}/"
     # Run all tests if common directories modified
     if [[ ${changed_dir} =~ ^(integration_tests|util|cloudbuild)/$ ]]; then
-      continue # remove this before squash/merge
       echo "All tests will be run: '${changed_dir}' was changed"
       TESTS_TO_RUN=(":DataprocInitActionsTestSuite")
       return 0
@@ -101,13 +100,13 @@ determine_tests_to_run() {
 }
 
 run_tests() {
-  local -r max_parallel_tests=10
+  local -r max_parallel_tests=20
   bazel test \
     --jobs="${max_parallel_tests}" \
     --local_test_jobs="${max_parallel_tests}" \
     --flaky_test_attempts=3 \
     --action_env="INTERNAL_IP_SSH=true" \
-    --test_output="errors" \
+    --test_output="all" \
     --noshow_progress \
     --noshow_loading_progress \
     --test_arg="--image_version=${IMAGE_VERSION}" \

cloudbuild/run-presubmit-on-k8s.sh

@@ -14,12 +14,32 @@ LOGS_SINCE_TIME=$(date --iso-8601=seconds)
 
 # This kubectl sometimes fails because services have not caught up.  Thread.yield()
 sleep 10s
-kubectl run "${POD_NAME}" \
-  --image="${IMAGE}" \
-  --restart=Never \
-  --env="COMMIT_SHA=${COMMIT_SHA}" \
-  --env="IMAGE_VERSION=${DATAPROC_IMAGE_VERSION}" \
-  --command -- bash /init-actions/cloudbuild/presubmit.sh
+
+readonly POD_CONFIG="pod.yaml"
+cat >$POD_CONFIG <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "${POD_NAME}"
+spec:
+  restartPolicy: Never
+  containers:
+  - name: "dataproc-test-runner"
+    image: "${IMAGE}"
+    resources:
+      requests:
+        memory: "4G"
+        cpu: "6000m"
+    env:
+    - name: COMMIT_SHA
+      value: "${COMMIT_SHA}"
+    - name: IMAGE_VERSION
+      value: "${DATAPROC_IMAGE_VERSION}"
+    command: ["bash"]
+    args: ["/init-actions/cloudbuild/presubmit.sh"]
+EOF
+
+kubectl apply -f $POD_CONFIG
 
 # Delete POD on exit and describe it before deletion if exit was unsuccessful
 trap '[[ $? != 0 ]] && kubectl describe "pod/${POD_NAME}"; kubectl delete pods "${POD_NAME}"' EXIT