ci: suitespec riot venvs_per_job for gen_gitlab_config

scripts/gen_gitlab_config.py

@@ -35,6 +35,7 @@ class JobSpec:
     services: t.Optional[t.List[str]] = None
     env: t.Optional[t.Dict[str, str]] = None
     parallelism: t.Optional[int] = None
+    venvs_per_job: t.Optional[int] = None
     retry: t.Optional[int] = None
     timeout: t.Optional[int] = None
     skip: bool = False
@@ -129,6 +130,71 @@ def __str__(self) -> str:
         return "\n".join(lines)
 
 
+def calculate_dynamic_parallelism(suite_name: str, suite_config: dict) -> t.Optional[int]:
+    """Calculate parallelism based on venvs_per_job configuration.
+
+    Packs N venvs per parallel job, scaling automatically with venv count changes.
+    Only applies to riot runner suites with venvs_per_job configured.
+
+    Args:
+        suite_name: The name of the test suite
+        suite_config: The suite configuration dict from suitespec
+
+    Returns:
+        The calculated parallelism value (1 to 20), or None if venvs_per_job not configured
+    """
+    # Only for riot suites
+    if suite_config.get("runner") != "riot":
+        return None
+
+    # Check if venvs_per_job is configured
+    venvs_per_job = suite_config.get("venvs_per_job")
+    if venvs_per_job is None:
+        return None
+
+    # Only import when needed
+    import math
+
+    # Importing will load/evaluate the whole riotfile.py
+    import riotfile
+
+    pattern = suite_config.get("pattern", suite_name)
+    try:
+        pattern_regex = re.compile(pattern)
+    except re.error:
+        LOGGER.warning("Invalid pattern for suite %s: %s", suite_name, pattern)
+        return None
+
+    # Collect unique venv hashes by matching pattern (mimics riot's --hash-only logic)
+    venv_hashes = set()
+    for inst in riotfile.venv.instances():  # type: ignore[attr-defined]
+        if not inst.name or not inst.matches_pattern(pattern_regex):  # type: ignore[attr-defined]
+            continue
+        venv_hashes.add(inst.short_hash)  # type: ignore[attr-defined]
+
+    venv_count = len(venv_hashes)
+
+    if venv_count == 0:
+        LOGGER.warning("No riot venvs found for suite %s with pattern %s", suite_name, pattern)
+        return None
+
+    # Calculate parallelism
+    calculated = math.ceil(venv_count / venvs_per_job)
+
+    # Cap at 20 to avoid over-parallelization
+    MAX_PARALLELISM = 20
+    calculated = min(calculated, MAX_PARALLELISM)
+
+    LOGGER.debug(
+        "Suite %s: %d venvs, %d venvs_per_job -> parallelism %d",
+        suite_name,
+        venv_count,
+        venvs_per_job,
+        calculated,
+    )
+    return calculated
+
+
 def gen_required_suites() -> None:
     """Generate the list of test suites that need to be run."""
     from needs_testrun import extract_git_commit_selections
@@ -193,6 +259,15 @@ def gen_required_suites() -> None:
                 LOGGER.debug("Skipping suite %s", suite)
                 continue
 
+            # Calculate dynamic parallelism for riot suites without explicit value
+            if jobspec.parallelism is None and suite_config.get("runner") == "riot":
+                calculated = calculate_dynamic_parallelism(suite, suite_config)
+                if calculated is not None:
+                    jobspec.parallelism = calculated
+                    LOGGER.info("Suite %s: calculated parallelism=%d", suite, calculated)
+                else:
+                    LOGGER.debug("Suite %s: no venvs_per_job config, using GitLab default", suite)
+
             print(str(jobspec), file=f)
 
 

tests/appsec/suitespec.yml

@@ -27,7 +27,7 @@ suites:
     runner: riot
     snapshot: true
   appsec_iast_default:
-    parallelism: 6
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -56,7 +56,7 @@ suites:
     runner: riot
     timeout: 30m
   appsec_iast_native:
-    parallelism: 6
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -81,7 +81,7 @@ suites:
     runner: riot
     timeout: 50m
   iast_tdd_propagation:
-    parallelism: 6
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -111,7 +111,7 @@ suites:
     env:
       TEST_POSTGRES_HOST: postgres
       TEST_MYSQL_HOST: mysql
-    parallelism: 6
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -127,7 +127,7 @@ suites:
       - postgres
       - mysql
   appsec_integrations_langchain:
-    parallelism: 15
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -140,7 +140,7 @@ suites:
     retry: 2
     runner: riot
   appsec_integrations_flask:
-    parallelism: 13
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -155,7 +155,7 @@ suites:
       - testagent
     timeout: 40m
   appsec_integrations_django:
-    parallelism: 16
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -170,7 +170,7 @@ suites:
       - testagent
     timeout: 30m
   appsec_integrations_fastapi:
-    parallelism: 17
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -261,6 +261,7 @@ suites:
       - tests/appsec/ai_guard/api/*
     retry: 2
     runner: riot
+    venvs_per_job: 1
   ai_guard_langchain:
     paths:
       - '@bootstrap'
@@ -272,3 +273,4 @@ suites:
     runner: riot
     services:
       - testagent
+    venvs_per_job: 2

tests/ci_visibility/suitespec.yml

@@ -41,7 +41,7 @@ suites:
     runner: riot
     snapshot: true
   pytest:
-    parallelism: 12
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'

tests/contrib/suitespec.yml

@@ -204,6 +204,7 @@ suites:
     env:
       TEST_MOTO_PORT: '3000'
     snapshot: true
+    venvs_per_job: 2
   aiohttp:
     parallelism: 3
     paths:
@@ -269,6 +270,7 @@ suites:
     snapshot: true
     services:
       - postgres
+    venvs_per_job: 2
   algoliasearch:
     parallelism: 2
     paths:
@@ -296,7 +298,7 @@ suites:
       - redis
     snapshot: true
   asgi:
-    parallelism: 6
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -486,6 +488,7 @@ suites:
       - rabbitmq
       - redis
     snapshot: true
+    venvs_per_job: 1
   cherrypy:
     paths:
       - '@bootstrap'
@@ -497,6 +500,7 @@ suites:
       - tests/snapshots/tests.{suite}.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   consul:
     parallelism: 1
     paths:
@@ -604,6 +608,7 @@ suites:
       - tests/contrib/dogpile_cache/*
     runner: riot
     snapshot: true
+    venvs_per_job: 3
   dramatiq:
     env:
       TEST_REDIS_HOST: redis
@@ -625,7 +630,7 @@ suites:
     env:
       TEST_ELASTICSEARCH_HOST: elasticsearch
       TEST_OPENSEARCH_HOST: opensearch
-    parallelism: 17
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -649,6 +654,7 @@ suites:
       - tests/contrib/falcon/*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   fastapi:
     paths:
       - '@bootstrap'
@@ -664,11 +670,12 @@ suites:
       - tests/snapshots/tests.{suite}.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   flask:
     env:
       TEST_MEMCACHED_HOST: memcached
       TEST_REDIS_HOST: redis
-    parallelism: 11
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -699,6 +706,7 @@ suites:
       - tests/contrib/gevent/*
     runner: riot
     snapshot: false
+    venvs_per_job: 2
   graphql:graphene:
     parallelism: 1
     paths:
@@ -735,6 +743,7 @@ suites:
       - tests/snapshots/tests.contrib.grpc.*
     runner: riot
     snapshot: true
+    venvs_per_job: 3
   gunicorn:
     parallelism: 6
     paths:
@@ -759,6 +768,7 @@ suites:
     services:
       - httpbin
     snapshot: true
+    venvs_per_job: 1
   httpx:
     parallelism: 3
     paths:
@@ -872,6 +882,7 @@ suites:
     services:
       - mariadb
     snapshot: true
+    venvs_per_job: 2
   molten:
     parallelism: 1
     paths:
@@ -938,6 +949,7 @@ suites:
     services:
       - postgres
     snapshot: true
+    venvs_per_job: 2
   pylibmc:
     parallelism: 1
     paths:
@@ -976,6 +988,7 @@ suites:
     services:
       - mongo
     snapshot: true
+    venvs_per_job: 2
   pymysql:
     parallelism: 1
     paths:
@@ -1109,6 +1122,7 @@ suites:
       - tests/snapshots/tests.contrib.sanic.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   snowflake:
     paths:
       - '@bootstrap'
@@ -1121,6 +1135,7 @@ suites:
       - tests/snapshots/tests.contrib.snowflake.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   sourcecode:
     parallelism: 1
     paths:
@@ -1157,6 +1172,7 @@ suites:
       - tests/contrib/starlette/*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   stdlib:
     parallelism: 2
     paths:

tests/llmobs/suitespec.yml

@@ -105,6 +105,7 @@ suites:
       - tests/snapshots/tests.contrib.litellm.*
     runner: riot
     snapshot: true
+    venvs_per_job: 1
   llmobs:
     paths:
       - '@bootstrap'
@@ -114,7 +115,7 @@ suites:
       - tests/llmobs/*
     runner: riot
     snapshot: true
-    parallelism: 5
+    venvs_per_job: 1
   mcp:
     parallelism: 1
     paths:
@@ -129,7 +130,7 @@ suites:
     runner: riot
     snapshot: true
   openai:
-    parallelism: 14
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'
@@ -182,6 +183,7 @@ suites:
       - tests/snapshots/tests.contrib.openai_agents.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2
   pydantic_ai:
     paths:
       - '@bootstrap'
@@ -194,3 +196,4 @@ suites:
       - tests/snapshots/tests.contrib.pydantic_ai.*
     runner: riot
     snapshot: true
+    venvs_per_job: 2

tests/profiling/suitespec.yml

@@ -82,7 +82,7 @@ suites:
     env:
       DD_TRACE_AGENT_URL: ''
     # `riot list --hash-only profile-v2$ | wc -1` = 19
-    parallelism: 16
+    venvs_per_job: 1
     paths:
       - '@bootstrap'
       - '@core'