test(monorepo): adds adhoc package selection for CI system tests

.kokoro/system.sh

@@ -161,6 +161,8 @@ reap_parallel_results() {
     fi
   done
 
+
+
   if [ "$failed_count" -gt 0 ]; then
     echo "=================================================="
     echo "@FAILED - DETAILED LOGS FOR FAILED PACKAGES"
@@ -183,6 +185,7 @@ reap_parallel_results() {
           cat "$LOG_DIR/$pkg.log"
         else
           echo "Warning: No log file found for failed package $pkg"
+
         fi
         echo ""
       fi
@@ -276,6 +279,61 @@ for path in `find 'packages' \
   fi
 done
 
+# --- Ad-hoc Testing Integration ---
+TRIGGER_ADHOC="false"
+if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then
+    echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..."
+    headers=(-H "User-Agent: Kokoro")
+    if [[ -n "${GITHUB_TOKEN:-${GH_TOKEN}}" ]]; then
+        headers+=(-H "Authorization: token ${GITHUB_TOKEN:-${GH_TOKEN}}")
+    fi
+    # Hardened curl call with || true to prevent script termination if network fails
+    LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels" || echo "[]")
+
+    # For this prototype:
+    # we use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone,
+    # and we cannot guarantee that tools like 'jq' or 'gh' are installed in the test environment.
+    # Python and its built-in 'json' module are guaranteed to be available in this repository.
+    IS_ADHOC=$(python3 -c "
+import json
+import sys
+try:
+    labels = json.loads(sys.argv[1])
+    if isinstance(labels, list):
+        if any(isinstance(l, dict) and l.get('name') == 'test:adhoc' for l in labels):
+            print('true')
+except Exception:
+    pass
+" "$LABELS_JSON")
+
+    if [[ "$IS_ADHOC" == "true" ]]; then
+        TRIGGER_ADHOC="true"
+        echo "Adhoc test label 'test:adhoc' found!"
+    else
+        echo "Adhoc test label not found or error occurred."
+    fi
+fi
+
+if [[ "$TRIGGER_ADHOC" == "true" ]]; then
+    echo "Running ad-hoc package selection..."
+    source ci/adhoc/adhoc_test_runner.sh
+
+    echo "Deduplicating packages..."
+    # Deduplication using Associative Arrays (Requires Bash 4+)
+    declare -A unique_packages
+    for pkg in "${PACKAGES_TO_TEST[@]}"; do
+        [[ -n "$pkg" ]] && unique_packages["$pkg"]=1
+    done
+    for pkg in $ADHOC_PACKAGES; do
+        [[ -n "$pkg" ]] && unique_packages["$pkg"]=1
+    done
+
+    PACKAGES_TO_TEST=("${!unique_packages[@]}")
+
+    echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}"
+fi
+# --- End Ad-hoc Testing Integration ---
+
 # Parallel Execution Logic
 MAX_JOBS=${MAX_JOBS:-4}
 
@@ -301,18 +359,20 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
 # Stream package names to xargs for parallel execution
 # -P "$MAX_JOBS" controls concurrency
 # -I {} replaces {} with the package name
-printf '%s\n' "${PACKAGES_TO_TEST[@]}" \
-  | xargs -n 1 -P "$MAX_JOBS" \
+printf '%s\0' "${PACKAGES_TO_TEST[@]}" \
+  | xargs -0 -n 1 -P "$MAX_JOBS" \
     bash -c '
       pkg="$0"
+
       # Determine log location: prefer Sponge artifacts directory if available
       if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then
         pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg"
-        mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; }
+        mkdir -p "$pkg_log_dir" || { echo "Failed to mkdir $pkg_log_dir"; touch "$LOG_DIR/$pkg.failed"; exit 1; }
         log_file="$pkg_log_dir/sponge_log.log"
       else
         log_file="$LOG_DIR/$pkg.log"
       fi
+      echo "Log file for $pkg: $log_file"
 
       # Run test; if it fails, create a .failed file to signal failure to the reaper
       run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"

ci/adhoc/.package_groups.txt

@@ -0,0 +1,4 @@
+handwritten: google-cloud-translate
+handwritten: google-cloud-logging
+core: google-api-core
+core: google-cloud-core

ci/adhoc/.standalone_package_list.txt

@@ -0,0 +1,3 @@
+package: google-cloud-logging
+package: google-cloud-dns
+group: handwritten

ci/adhoc/adhoc_test_runner.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Script to determine ad-hoc packages to test.
+# This script is intended to be sourced from main test scripts.
+#
+# Ensure we are in the project root if called directly,
+# but usually this is sourced and CWD is already project root.
+# For safety, we can use script location but if sourced $0 might be the parent script.
+# Let's assume CWD is project root as per system.sh behavior.
+
+ADHOC_DIR="ci/adhoc"
+STANDALONE_LIST="${ADHOC_DIR}/.standalone_package_list.txt"
+GROUPS_FILE="${ADHOC_DIR}/.package_groups.txt"
+
+if [[ ! -f "$STANDALONE_LIST" ]]; then
+    echo "Warning: $STANDALONE_LIST not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+if [[ ! -f "$GROUPS_FILE" ]]; then
+    echo "Warning: $GROUPS_FILE not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+# Grab individual packages
+adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true)
+
+# Grab requested groups
+requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true)
+
+# Expand groups
+for group in $requested_groups; do
+    group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs || true)
+    adhoc_packages="$adhoc_packages $group_pkgs"
+done
+
+# Convert to unique list (deduplicate our adhoc packages)
+ADHOC_PACKAGES=$(echo "$adhoc_packages" | tr ' ' '\n' | sort -u | xargs)
+
+export ADHOC_PACKAGES

packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py

@@ -13,6 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# EXPERIMENTAL: This is a temporary change to trigger Kokoro system tests.
+# Because Kokoro is currently configured to only watch specific package paths (like packages/google-cloud-speech/.*),
+# we must touch a file in one of those paths to wake it up.
+#
+# This file is GAPIC_AUTO, and this specific file is NOT one of the 5 tracked files (setup.py, etc.)
+# in system.sh, so this change will NOT launch google-cloud-speech's tests.
+#
+# If this ad-hoc testing prototype proves successful, we will update the internal Kokoro
+# JobConfigs to watch 'ci/adhoc/.*' instead, eliminating the need for this workaround.
+
 import json
 import logging as std_logging
 import os
@@ -45,9 +55,8 @@
 from google.auth.exceptions import MutualTLSChannelError  # type: ignore
 from google.auth.transport import mtls  # type: ignore
 from google.auth.transport.grpc import SslCredentials  # type: ignore
-from google.oauth2 import service_account  # type: ignore
-
 from google.cloud.speech_v1 import gapic_version as package_version
+from google.oauth2 import service_account  # type: ignore
 
 try:
     OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
@@ -67,9 +76,8 @@
 import google.api_core.operation_async as operation_async  # type: ignore
 import google.protobuf.duration_pb2 as duration_pb2  # type: ignore
 import google.rpc.status_pb2 as status_pb2  # type: ignore
-from google.longrunning import operations_pb2  # type: ignore
-
 from google.cloud.speech_v1.types import cloud_speech
+from google.longrunning import operations_pb2  # type: ignore
 
 from .transports.base import DEFAULT_CLIENT_INFO, SpeechTransport
 from .transports.grpc import SpeechGrpcTransport

packages/google-resumable-media/setup.py

@@ -16,7 +16,6 @@
 
 import setuptools
 
-
 PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))
 
 with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj:

packages/google-resumable-media/tests/system/requests/test_download.py

@@ -21,15 +21,13 @@
 
 import google.auth  # type: ignore
 import google.auth.transport.requests as tr_requests  # type: ignore
-import pytest  # type: ignore
-
-from google.resumable_media import common
 import google.resumable_media.requests as resumable_requests
-from google.resumable_media import _helpers
-from google.resumable_media.requests import _request_helpers
 import google.resumable_media.requests.download as download_mod
-from tests.system import utils
+import pytest  # type: ignore
+from google.resumable_media import _helpers, common
+from google.resumable_media.requests import _request_helpers
 
+from tests.system import utils
 
 CURR_DIR = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")

packages/google-resumable-media/tests/system/requests/test_dummy_failure.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Temporary dummy test to verify ad-hoc testing failure reporting.
+This file is for experimentation/prototyping only and will be removed before merge.
+We are using this to test packages that have legit changes (i.e. they would show up in `package_diff` naturally AND would have failing tests.
+"""
+
+
+def test_intentional_failure():
+    """Intentional failure to verify CI output formatting."""
+    assert False, "Intentional failure to verify ad-hoc testing behavior"