diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index afc29068e44..e52da3227d0 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -35,6 +35,7 @@ class JobSpec: services: t.Optional[t.List[str]] = None env: t.Optional[t.Dict[str, str]] = None parallelism: t.Optional[int] = None + venvs_per_job: t.Optional[int] = None retry: t.Optional[int] = None timeout: t.Optional[int] = None skip: bool = False @@ -129,6 +130,71 @@ def __str__(self) -> str: return "\n".join(lines) +def calculate_dynamic_parallelism(suite_name: str, suite_config: dict) -> t.Optional[int]: + """Calculate parallelism based on venvs_per_job configuration. + + Packs N venvs per parallel job, scaling automatically with venv count changes. + Only applies to riot runner suites with venvs_per_job configured. + + Args: + suite_name: The name of the test suite + suite_config: The suite configuration dict from suitespec + + Returns: + The calculated parallelism value (1 to 20), or None if venvs_per_job not configured + """ + # Only for riot suites + if suite_config.get("runner") != "riot": + return None + + # Check if venvs_per_job is configured + venvs_per_job = suite_config.get("venvs_per_job") + if venvs_per_job is None: + return None + + # Only import when needed + import math + + # Importing will load/evaluate the whole riotfile.py + import riotfile + + pattern = suite_config.get("pattern", suite_name) + try: + pattern_regex = re.compile(pattern) + except re.error: + LOGGER.warning("Invalid pattern for suite %s: %s", suite_name, pattern) + return None + + # Collect unique venv hashes by matching pattern (mimics riot's --hash-only logic) + venv_hashes = set() + for inst in riotfile.venv.instances(): # type: ignore[attr-defined] + if not inst.name or not inst.matches_pattern(pattern_regex): # type: ignore[attr-defined] + continue + venv_hashes.add(inst.short_hash) # type: ignore[attr-defined] + + venv_count = len(venv_hashes) + + if venv_count == 0: + LOGGER.warning("No riot venvs found for suite %s with pattern %s", suite_name, pattern) + return None + + # Calculate parallelism + calculated = math.ceil(venv_count / venvs_per_job) + + # Cap at 20 to avoid over-parallelization + MAX_PARALLELISM = 20 + calculated = min(calculated, MAX_PARALLELISM) + + LOGGER.debug( + "Suite %s: %d venvs, %d venvs_per_job -> parallelism %d", + suite_name, + venv_count, + venvs_per_job, + calculated, + ) + return calculated + + def gen_required_suites() -> None: """Generate the list of test suites that need to be run.""" from needs_testrun import extract_git_commit_selections @@ -193,6 +259,15 @@ def gen_required_suites() -> None: LOGGER.debug("Skipping suite %s", suite) continue + # Calculate dynamic parallelism for riot suites without explicit value + if jobspec.parallelism is None and suite_config.get("runner") == "riot": + calculated = calculate_dynamic_parallelism(suite, suite_config) + if calculated is not None: + jobspec.parallelism = calculated + LOGGER.info("Suite %s: calculated parallelism=%d", suite, calculated) + else: + LOGGER.debug("Suite %s: no venvs_per_job config, using GitLab default", suite) + print(str(jobspec), file=f) diff --git a/tests/appsec/suitespec.yml b/tests/appsec/suitespec.yml index 6c5a52ed7ff..0e4a3a56fd7 100644 --- a/tests/appsec/suitespec.yml +++ b/tests/appsec/suitespec.yml @@ -27,7 +27,7 @@ suites: runner: riot snapshot: true appsec_iast_default: - parallelism: 6 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -56,7 +56,7 @@ suites: runner: riot timeout: 30m appsec_iast_native: - parallelism: 6 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -81,7 +81,7 @@ suites: runner: riot timeout: 50m iast_tdd_propagation: - parallelism: 6 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -111,7 +111,7 @@ suites: env: TEST_POSTGRES_HOST: postgres TEST_MYSQL_HOST: mysql - parallelism: 6 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -127,7 +127,7 @@ suites: - postgres - mysql appsec_integrations_langchain: - parallelism: 15 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -140,7 +140,7 @@ suites: retry: 2 runner: riot appsec_integrations_flask: - parallelism: 13 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -155,7 +155,7 @@ suites: - testagent timeout: 40m appsec_integrations_django: - parallelism: 16 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -170,7 +170,7 @@ suites: - testagent timeout: 30m appsec_integrations_fastapi: - parallelism: 17 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -261,6 +261,7 @@ suites: - tests/appsec/ai_guard/api/* retry: 2 runner: riot + venvs_per_job: 1 ai_guard_langchain: paths: - '@bootstrap' @@ -272,3 +273,4 @@ suites: runner: riot services: - testagent + venvs_per_job: 2 diff --git a/tests/ci_visibility/suitespec.yml b/tests/ci_visibility/suitespec.yml index 6046b518c31..b630d6742a0 100644 --- a/tests/ci_visibility/suitespec.yml +++ b/tests/ci_visibility/suitespec.yml @@ -41,7 +41,7 @@ suites: runner: riot snapshot: true pytest: - parallelism: 12 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index c61b75638f7..e4d48669d0c 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -204,6 +204,7 @@ suites: env: TEST_MOTO_PORT: '3000' snapshot: true + venvs_per_job: 2 aiohttp: parallelism: 3 paths: @@ -269,6 +270,7 @@ suites: snapshot: true services: - postgres + venvs_per_job: 2 algoliasearch: parallelism: 2 paths: @@ -296,7 +298,7 @@ suites: - redis snapshot: true asgi: - parallelism: 6 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -486,6 +488,7 @@ suites: - rabbitmq - redis snapshot: true + venvs_per_job: 1 cherrypy: paths: - '@bootstrap' @@ -497,6 +500,7 @@ suites: - tests/snapshots/tests.{suite}.* runner: riot snapshot: true + venvs_per_job: 2 consul: parallelism: 1 paths: @@ -604,6 +608,7 @@ suites: - tests/contrib/dogpile_cache/* runner: riot snapshot: true + venvs_per_job: 3 dramatiq: env: TEST_REDIS_HOST: redis @@ -625,7 +630,7 @@ suites: env: TEST_ELASTICSEARCH_HOST: elasticsearch TEST_OPENSEARCH_HOST: opensearch - parallelism: 17 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -649,6 +654,7 @@ suites: - tests/contrib/falcon/* runner: riot snapshot: true + venvs_per_job: 2 fastapi: paths: - '@bootstrap' @@ -664,11 +670,12 @@ suites: - tests/snapshots/tests.{suite}.* runner: riot snapshot: true + venvs_per_job: 2 flask: env: TEST_MEMCACHED_HOST: memcached TEST_REDIS_HOST: redis - parallelism: 11 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -699,6 +706,7 @@ suites: - tests/contrib/gevent/* runner: riot snapshot: false + venvs_per_job: 2 graphql:graphene: parallelism: 1 paths: @@ -735,6 +743,7 @@ suites: - tests/snapshots/tests.contrib.grpc.* runner: riot snapshot: true + venvs_per_job: 3 gunicorn: parallelism: 6 paths: @@ -759,6 +768,7 @@ suites: services: - httpbin snapshot: true + venvs_per_job: 1 httpx: parallelism: 3 paths: @@ -872,6 +882,7 @@ suites: services: - mariadb snapshot: true + venvs_per_job: 2 molten: parallelism: 1 paths: @@ -938,6 +949,7 @@ suites: services: - postgres snapshot: true + venvs_per_job: 2 pylibmc: parallelism: 1 paths: @@ -976,6 +988,7 @@ suites: services: - mongo snapshot: true + venvs_per_job: 2 pymysql: parallelism: 1 paths: @@ -1109,6 +1122,7 @@ suites: - tests/snapshots/tests.contrib.sanic.* runner: riot snapshot: true + venvs_per_job: 2 snowflake: paths: - '@bootstrap' @@ -1121,6 +1135,7 @@ suites: - tests/snapshots/tests.contrib.snowflake.* runner: riot snapshot: true + venvs_per_job: 2 sourcecode: parallelism: 1 paths: @@ -1157,6 +1172,7 @@ suites: - tests/contrib/starlette/* runner: riot snapshot: true + venvs_per_job: 2 stdlib: parallelism: 2 paths: diff --git a/tests/llmobs/suitespec.yml b/tests/llmobs/suitespec.yml index 60ae476a054..b224a938e40 100644 --- a/tests/llmobs/suitespec.yml +++ b/tests/llmobs/suitespec.yml @@ -105,6 +105,7 @@ suites: - tests/snapshots/tests.contrib.litellm.* runner: riot snapshot: true + venvs_per_job: 1 llmobs: paths: - '@bootstrap' @@ -114,7 +115,7 @@ suites: - tests/llmobs/* runner: riot snapshot: true - parallelism: 5 + venvs_per_job: 1 mcp: parallelism: 1 paths: @@ -129,7 +130,7 @@ suites: runner: riot snapshot: true openai: - parallelism: 14 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' @@ -182,6 +183,7 @@ suites: - tests/snapshots/tests.contrib.openai_agents.* runner: riot snapshot: true + venvs_per_job: 2 pydantic_ai: paths: - '@bootstrap' @@ -194,3 +196,4 @@ suites: - tests/snapshots/tests.contrib.pydantic_ai.* runner: riot snapshot: true + venvs_per_job: 2 diff --git a/tests/profiling/suitespec.yml b/tests/profiling/suitespec.yml index 1b0ca074891..05ee118516c 100644 --- a/tests/profiling/suitespec.yml +++ b/tests/profiling/suitespec.yml @@ -81,8 +81,8 @@ suites: profile_v2: env: DD_TRACE_AGENT_URL: '' - # `riot list --hash-only profile-v2 | wc -l` = 26 - parallelism: 26 + # `riot list --hash-only profile-v2$ | wc -1` = 26 + venvs_per_job: 1 paths: - '@bootstrap' - '@core' diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 4358496c03b..bb67ba0d47b 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -253,7 +253,6 @@ suites: DD_TRACE_AGENT_URL: http://localhost:8126 KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" - parallelism: 14 paths: - '@tracing' - '@bootstrap' @@ -268,6 +267,7 @@ suites: - tests/snapshots/test_* retry: 2 runner: riot + venvs_per_job: 1 vendor: parallelism: 1 paths: