From d4a51228c398b47f2aa7283b1f093713bf3ac887 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Fri, 31 Jul 2026 11:57:46 -0700
Subject: [PATCH 01/13] [NVBUG-6481034][fix] Align perf launcher with pytest
 shard

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 jenkins/scripts/perf/submit.py             | 195 ++++++++++++++++++---
 tests/unittest/scripts/test_perf_submit.py |  60 +++++++
 2 files changed, 233 insertions(+), 22 deletions(-)

diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py
index 561f7425e469..5ee06ee187bf 100755
--- a/jenkins/scripts/perf/submit.py
+++ b/jenkins/scripts/perf/submit.py
@@ -33,9 +33,12 @@
 """
 
 import argparse
+import heapq
+import json
 import math
 import os
 import re
+import shlex
 import sys
 
 import yaml
@@ -63,26 +66,166 @@ def _import_precheck_config(llm_src):
 # --------------------------------------------------------------------------- #
 # Test list parsing
 # --------------------------------------------------------------------------- #
-def parse_test_case_name(test_list_path, llm_src, split_group=0):
-    """Parse the selected line of the test list.
-
-    Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode).
-    See the module docstring for the supported test name shapes.
-    """
+def _read_test_list_lines(test_list_path):
     with open(test_list_path, "r") as f:
         lines = [line.strip() for line in f if line.strip()]
-
     if not lines:
         raise ValueError(f"Test list is empty: {test_list_path}")
+    return lines
+
+
+def _pytest_command_tokens(script_prefix_lines):
+    pytest_command_line = next(
+        (line for line in script_prefix_lines if "export pytestCommand=" in line), ""
+    )
+    if not pytest_command_line:
+        return []
+    command = pytest_command_line.split("=", 1)[1].strip()
+    if len(command) >= 2 and command[0] == command[-1] and command[0] in ('"', "'"):
+        command = command[1:-1]
+    return shlex.split(command)
+
+
+def _pytest_option(tokens, option):
+    for index, token in enumerate(tokens):
+        if token == option:
+            return tokens[index + 1] if index + 1 < len(tokens) else None
+        if token.startswith(f"{option}="):
+            return token.split("=", 1)[1]
+    return None
+
+
+def _test_nodeid(test_line):
+    """Strip test-list markers from a line, matching pytest's selected nodeid."""
+    return re.split(
+        r"\s+(?:XFAIL|SKIP|UNSTABLE|TIMEOUT)(?:\s|$)",
+        test_line,
+        maxsplit=1,
+    )[0]
+
+
+def _load_pytest_split_durations(tokens, llm_src):
+    durations_option = _pytest_option(tokens, "--durations-path")
+    if durations_option:
+        durations_path = durations_option
+        if not os.path.exists(durations_path):
+            durations_path = os.path.join(
+                llm_src,
+                "tests",
+                "integration",
+                "defs",
+                os.path.basename(durations_option),
+            )
+    else:
+        durations_path = os.path.join(llm_src, "tests", "integration", "defs", ".test_durations")
+
+    try:
+        with open(durations_path, "r") as durations_file:
+            durations = json.load(durations_file)
+    except FileNotFoundError:
+        durations = {}
+    if isinstance(durations, list):
+        durations = dict(durations)
+    if not isinstance(durations, dict):
+        raise ValueError(f"Invalid pytest-split durations file: {durations_path}")
+    return durations, durations_path
+
+
+def _select_least_duration_group(lines, durations, splits, group):
+    """Mirror pytest-split's LeastDurationAlgorithm exactly."""
+    if splits < 1:
+        raise ValueError(f"pytest --splits must be >= 1, got {splits}")
+    if group < 1 or group > splits:
+        raise ValueError(f"pytest --group must be in [1, {splits}], got {group}")
+
+    nodeids = [_test_nodeid(line) for line in lines]
+    relevant_durations = {
+        nodeid: float(durations[nodeid]) for nodeid in nodeids if nodeid in durations
+    }
+    average_duration = (
+        sum(relevant_durations.values()) / len(relevant_durations) if relevant_durations else 1.0
+    )
+    items = [
+        (line, nodeid, relevant_durations.get(nodeid, average_duration), original_index)
+        for original_index, (line, nodeid) in enumerate(zip(lines, nodeids))
+    ]
 
-    if split_group > 0:
+    # pytest-split first sorts by item name, then performs a stable descending
+    # duration sort. It greedily places each item in the least-loaded group.
+    items.sort(key=lambda item: item[1])
+    items.sort(key=lambda item: item[2], reverse=True)
+    selected = [[] for _ in range(splits)]
+    group_heap = [(0.0, group_index) for group_index in range(splits)]
+    heapq.heapify(group_heap)
+    for line, _nodeid, duration, original_index in items:
+        group_duration, group_index = heapq.heappop(group_heap)
+        selected[group_index].append((original_index, line))
+        heapq.heappush(group_heap, (group_duration + duration, group_index))
+
+    return [line for _original_index, line in sorted(selected[group - 1], key=lambda item: item[0])]
+
+
+def select_test_case_line(test_list_path, llm_src, script_prefix_lines, split_group=0):
+    """Select the same test as the pytest-split shard in ``pytestCommand``."""
+    lines = _read_test_list_lines(test_list_path)
+    if split_group <= 0:
+        return lines[0]
+
+    tokens = _pytest_command_tokens(script_prefix_lines)
+    splits_option = _pytest_option(tokens, "--splits")
+    group_option = _pytest_option(tokens, "--group")
+    algorithm = _pytest_option(tokens, "--splitting-algorithm")
+    if splits_option is None or group_option is None:
         if split_group > len(lines):
             raise ValueError(
                 f"split_group {split_group} exceeds number of tests in test list ({len(lines)})"
             )
-        line = lines[split_group - 1]
+        return lines[split_group - 1]
+    if algorithm != "least_duration":
+        raise ValueError(
+            "Multi-node perf launcher only supports pytest-split's least_duration "
+            f"algorithm, got {algorithm!r}"
+        )
+
+    splits = int(splits_option)
+    pytest_group = int(group_option)
+    if pytest_group != split_group:
+        raise ValueError(
+            f"submit.py split_group={split_group} disagrees with pytest --group={pytest_group}"
+        )
+    durations, durations_path = _load_pytest_split_durations(tokens, llm_src)
+    selected = _select_least_duration_group(lines, durations, splits, pytest_group)
+    if len(selected) != 1:
+        raise ValueError(
+            "Multi-node perf launch requires exactly one test in each pytest-split "
+            f"group, but group {pytest_group}/{splits} selected {len(selected)} tests "
+            f"using {durations_path}: {selected}"
+        )
+    print(
+        f"Selected pytest-split group {pytest_group}/{splits} test using "
+        f"{durations_path}: {_test_nodeid(selected[0])}"
+    )
+    return selected[0]
+
+
+def parse_test_case_name(test_list_path, llm_src, split_group=0, selected_line=None):
+    """Parse the selected line of the test list.
+
+    Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode).
+    See the module docstring for the supported test name shapes.
+    """
+    if selected_line is not None:
+        line = selected_line
     else:
-        line = lines[0]
+        lines = _read_test_list_lines(test_list_path)
+        if split_group > 0:
+            if split_group > len(lines):
+                raise ValueError(
+                    f"split_group {split_group} exceeds number of tests in test list ({len(lines)})"
+                )
+            line = lines[split_group - 1]
+        else:
+            line = lines[0]
 
     if "[" not in line or "]" not in line:
         raise ValueError(f"Invalid test list format. Expected name with brackets: {line}")
@@ -484,7 +627,9 @@ def main():
         "--split-group",
         type=int,
         default=0,
-        help="1-indexed split group id. Selects the N-th test from the test list.",
+        help=(
+            "1-indexed pytest-split group id. Selects the same duration-balanced test as pytest."
+        ),
     )
     parser.add_argument("--stage-name", default="", help="Stage name (for logging / GPU detect)")
     parser.add_argument(
@@ -497,19 +642,29 @@ def main():
 
     args = parser.parse_args()
 
+    with open(args.script_prefix, "r") as f:
+        script_prefix_content = f.read()
+    script_prefix_lines = script_prefix_content.split("\n")
+
+    selected_test_line = select_test_case_line(
+        args.test_list,
+        args.llm_src,
+        script_prefix_lines,
+        args.split_group,
+    )
     config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name(
-        args.test_list, args.llm_src, args.split_group
+        args.test_list,
+        args.llm_src,
+        args.split_group,
+        selected_line=selected_test_line,
     )
 
     with open(config_yaml, "r") as f:
         config = yaml.safe_load(f)
 
-    # Recover test_case_name (the bracketed pytest test id) for the per-test
-    # output dir — same line/split logic as parse_test_case_name.
-    with open(args.test_list, "r") as f:
-        lines = [ln.strip() for ln in f if ln.strip()]
-    sel = lines[args.split_group - 1] if args.split_group > 0 else lines[0]
-    test_case_name = sel.split("[")[-1].split("]")[0] if "[" in sel else ""
+    test_case_name = (
+        selected_test_line.split("[")[-1].split("]")[0] if "[" in selected_test_line else ""
+    )
 
     hardware_config = get_hardware_config(config, runtime_mode, benchmark_mode, server_name)
     env_config = get_env_config(config, runtime_mode, benchmark_mode, server_name)
@@ -522,10 +677,6 @@ def main():
     print(f"Environment configuration: {env_config}")
     print(f"Benchmark configuration: {benchmark_config}")
 
-    with open(args.script_prefix, "r") as f:
-        script_prefix_content = f.read()
-    script_prefix_lines = script_prefix_content.split("\n")
-
     with open(args.srun_args, "r") as f:
         srun_args_content = f.read()
     srun_args_lines = srun_args_content.split()
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index 059b5d792527..0ebc92be3c4d 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import importlib.util
+import json
 from pathlib import Path
 from types import ModuleType
 
@@ -62,6 +63,11 @@ def test_get_benchmark_config_accepts_positive_integer(submit_module: ModuleType
     assert benchmark_config["concurrency"] == int(concurrency)
 
 
+@pytest.fixture
+def ci_submit_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+    return _load_module(SUBMIT_PATHS[0], monkeypatch)
+
+
 @pytest.mark.parametrize(
     "concurrency",
     (True, 1.5, [], {}, "0", 0, "-1", -1, "1.5", "not-an-integer", None),
@@ -102,3 +108,57 @@ def test_example_worker_environment_exports_positive_concurrency(example_submit_
     )
 
     assert worker_environment["TLLM_BENCHMARK_REQ_QUEUES_SIZE"] == "4301"
+
+
+def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_module, tmp_path):
+    test_lines = [
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1] TIMEOUT (90)",
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25] TIMEOUT (90)",
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1] TIMEOUT (90)",
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25] TIMEOUT (90)",
+    ]
+    test_list_path = tmp_path / "test_list.txt"
+    test_list_path.write_text("\n".join(test_lines), encoding="utf-8")
+
+    durations_dir = tmp_path / "tests" / "integration" / "defs"
+    durations_dir.mkdir(parents=True)
+    durations = {
+        ci_submit_module._test_nodeid(test_lines[0]): 836.268,
+        ci_submit_module._test_nodeid(test_lines[1]): 1462.754,
+        ci_submit_module._test_nodeid(test_lines[2]): 2211.1548,
+        ci_submit_module._test_nodeid(test_lines[3]): 2548.912,
+    }
+    (durations_dir / ".test_durations").write_text(json.dumps(durations), encoding="utf-8")
+    script_prefix_lines = [
+        'export pytestCommand="pytest --splitting-algorithm least_duration '
+        "--splits 4 --group 3 "
+        '--durations-path /remote/tests/integration/defs/.test_durations"'
+    ]
+
+    selected = ci_submit_module.select_test_case_line(
+        test_list_path,
+        tmp_path,
+        script_prefix_lines,
+        split_group=3,
+    )
+
+    assert selected == test_lines[1]
+
+
+def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path):
+    test_list_path = tmp_path / "test_list.txt"
+    test_list_path.write_text(
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n",
+        encoding="utf-8",
+    )
+    script_prefix_lines = [
+        'export pytestCommand="pytest --splitting-algorithm least_duration --splits 1 --group 1"'
+    ]
+
+    with pytest.raises(ValueError, match="disagrees with pytest --group"):
+        ci_submit_module.select_test_case_line(
+            test_list_path,
+            tmp_path,
+            script_prefix_lines,
+            split_group=2,
+        )

From 9b6d78c4eaaa7160bd4bc7fad4bd9d5499fc63d6 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Sat, 1 Aug 2026 18:27:01 -0700
Subject: [PATCH 02/13] [NVBUG-6481034][fix] Address shard selection review
 feedback

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 jenkins/scripts/perf/submit.py             |  6 ++++--
 requirements-dev.txt                       |  2 +-
 tests/unittest/scripts/test_perf_submit.py | 21 +++++++++++++++++++++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py
index 5ee06ee187bf..b9fffb125cd5 100755
--- a/jenkins/scripts/perf/submit.py
+++ b/jenkins/scripts/perf/submit.py
@@ -122,8 +122,10 @@ def _load_pytest_split_durations(tokens, llm_src):
     try:
         with open(durations_path, "r") as durations_file:
             durations = json.load(durations_file)
-    except FileNotFoundError:
-        durations = {}
+    except FileNotFoundError as error:
+        raise FileNotFoundError(
+            f"pytest-split durations file not found: {durations_path}"
+        ) from error
     if isinstance(durations, list):
         durations = dict(durations)
     if not isinstance(durations, dict):
diff --git a/requirements-dev.txt b/requirements-dev.txt
index d5dcc3a2bf7e..fe640557eb2a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -22,7 +22,7 @@ pytest-env
 pytest-forked
 pytest-xdist
 pytest-timeout
-pytest-split
+pytest-split==0.10.0
 pytest-mock
 pytest-threadleak
 pytest-unused-fixtures
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index 0ebc92be3c4d..3e3d116d7f9a 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -162,3 +162,24 @@ def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path):
             script_prefix_lines,
             split_group=2,
         )
+
+
+def test_ci_submit_rejects_missing_pytest_split_durations(ci_submit_module, tmp_path):
+    test_list_path = tmp_path / "test_list.txt"
+    test_list_path.write_text(
+        "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n",
+        encoding="utf-8",
+    )
+    script_prefix_lines = [
+        'export pytestCommand="pytest --splitting-algorithm least_duration '
+        '--splits 1 --group 1 --durations-path /remote/.test_durations"'
+    ]
+
+    expected_path = tmp_path / "tests" / "integration" / "defs" / ".test_durations"
+    with pytest.raises(FileNotFoundError, match=f"durations file not found: {expected_path}"):
+        ci_submit_module.select_test_case_line(
+            test_list_path,
+            tmp_path,
+            script_prefix_lines,
+            split_group=1,
+        )

From 63f27375eb84c59d12b17099b3d0c7d2860da2a5 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 10:56:27 -0700
Subject: [PATCH 03/13] [NVBUG-6481034][test] Guard pytest-split compatibility

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 requirements-dev.txt                       |  2 +-
 tests/unittest/scripts/test_perf_submit.py | 38 ++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index fe640557eb2a..d5dcc3a2bf7e 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -22,7 +22,7 @@ pytest-env
 pytest-forked
 pytest-xdist
 pytest-timeout
-pytest-split==0.10.0
+pytest-split
 pytest-mock
 pytest-threadleak
 pytest-unused-fixtures
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index 3e3d116d7f9a..2bf40a8df7a0 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -20,6 +20,7 @@
 from types import ModuleType
 
 import pytest
+from pytest_split.algorithms import LeastDurationAlgorithm
 
 REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
 SUBMIT_PATHS = (
@@ -29,6 +30,14 @@
 EXAMPLE_SUBMIT_PATH = REPO_ROOT / "examples" / "disaggregated" / "slurm" / "benchmark" / "submit.py"
 
 
+class _FakePytestItem:
+    def __init__(self, nodeid: str):
+        self.nodeid = nodeid
+
+    def __str__(self) -> str:
+        return self.nodeid
+
+
 def _load_module(path: Path, monkeypatch: pytest.MonkeyPatch) -> ModuleType:
     monkeypatch.syspath_prepend(str(path.parent))
     spec = importlib.util.spec_from_file_location(f"perf_submit_{path.parent.name}", path)
@@ -145,6 +154,35 @@ def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_m
     assert selected == test_lines[1]
 
 
+def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module):
+    lines = [
+        f"perf/test_perf_sanity.py::test_e2e[case-{case_name}] TIMEOUT (90)"
+        for case_name in ("zeta", "alpha", "gamma", "beta", "epsilon", "delta")
+    ]
+    nodeids = [ci_submit_module._test_nodeid(line) for line in lines]
+    items = [_FakePytestItem(nodeid) for nodeid in nodeids]
+    duration_sets = (
+        {nodeid: float(index + 1) for index, nodeid in enumerate(nodeids)},
+        dict.fromkeys(nodeids, 4.0),
+        {nodeids[1]: 8.0, nodeids[4]: 2.0, "irrelevant::test": 1000.0},
+        {},
+    )
+
+    for durations in duration_sets:
+        for splits in (2, 3, 4):
+            expected_groups = LeastDurationAlgorithm()(splits, items, durations)
+            for group, expected_group in enumerate(expected_groups, start=1):
+                selected = ci_submit_module._select_least_duration_group(
+                    lines,
+                    durations,
+                    splits,
+                    group,
+                )
+                assert [ci_submit_module._test_nodeid(line) for line in selected] == [
+                    item.nodeid for item in expected_group.selected
+                ]
+
+
 def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path):
     test_list_path = tmp_path / "test_list.txt"
     test_list_path.write_text(

From 9429527428c2d4dac8213bbcbe2cd54334fc43fc Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 11:03:38 -0700
Subject: [PATCH 04/13] [NVBUG-6481034][fix] Address selector contract review

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 jenkins/scripts/perf/submit.py             |  6 ++-
 tests/unittest/scripts/test_perf_submit.py | 59 ++++++++++++++++++++--
 2 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py
index b9fffb125cd5..259ef6c307c4 100755
--- a/jenkins/scripts/perf/submit.py
+++ b/jenkins/scripts/perf/submit.py
@@ -68,7 +68,11 @@ def _import_precheck_config(llm_src):
 # --------------------------------------------------------------------------- #
 def _read_test_list_lines(test_list_path):
     with open(test_list_path, "r") as f:
-        lines = [line.strip() for line in f if line.strip()]
+        lines = []
+        for line in f:
+            stripped_line = line.strip()
+            if stripped_line and not stripped_line.startswith("#"):
+                lines.append(stripped_line)
     if not lines:
         raise ValueError(f"Test list is empty: {test_list_path}")
     return lines
diff --git a/tests/unittest/scripts/test_perf_submit.py b/tests/unittest/scripts/test_perf_submit.py
index 2bf40a8df7a0..ed4fb8dedea7 100644
--- a/tests/unittest/scripts/test_perf_submit.py
+++ b/tests/unittest/scripts/test_perf_submit.py
@@ -31,7 +31,7 @@
 
 
 class _FakePytestItem:
-    def __init__(self, nodeid: str):
+    def __init__(self, nodeid: str) -> None:
         self.nodeid = nodeid
 
     def __str__(self) -> str:
@@ -119,7 +119,10 @@ def test_example_worker_environment_exports_positive_concurrency(example_submit_
     assert worker_environment["TLLM_BENCHMARK_REQ_QUEUES_SIZE"] == "4301"
 
 
-def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_module, tmp_path):
+def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(
+    ci_submit_module: ModuleType,
+    tmp_path: Path,
+) -> None:
     test_lines = [
         "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1] TIMEOUT (90)",
         "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25] TIMEOUT (90)",
@@ -154,7 +157,9 @@ def test_ci_submit_selects_same_least_duration_shard_as_pytest_split(ci_submit_m
     assert selected == test_lines[1]
 
 
-def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module):
+def test_ci_submit_selector_matches_installed_pytest_split(
+    ci_submit_module: ModuleType,
+) -> None:
     lines = [
         f"perf/test_perf_sanity.py::test_e2e[case-{case_name}] TIMEOUT (90)"
         for case_name in ("zeta", "alpha", "gamma", "beta", "epsilon", "delta")
@@ -183,7 +188,48 @@ def test_ci_submit_selector_matches_installed_pytest_split(ci_submit_module):
                 ]
 
 
-def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path):
+@pytest.mark.parametrize(
+    ("splits", "group", "match"),
+    (
+        (0, 1, "--splits"),
+        (2, 0, "--group"),
+        (2, 3, "--group"),
+    ),
+)
+def test_ci_submit_rejects_invalid_least_duration_groups(
+    ci_submit_module: ModuleType,
+    splits: int,
+    group: int,
+    match: str,
+) -> None:
+    with pytest.raises(ValueError, match=match):
+        ci_submit_module._select_least_duration_group(
+            ["perf/test_perf_sanity.py::test_e2e[case]"],
+            {},
+            splits,
+            group,
+        )
+
+
+def test_ci_submit_ignores_test_list_comments(
+    ci_submit_module: ModuleType,
+    tmp_path: Path,
+) -> None:
+    test_list_path = tmp_path / "test_list.txt"
+    test_list_path.write_text(
+        "# section comment\n\nperf/test_perf_sanity.py::test_e2e[case]\n",
+        encoding="utf-8",
+    )
+
+    assert ci_submit_module._read_test_list_lines(test_list_path) == [
+        "perf/test_perf_sanity.py::test_e2e[case]"
+    ]
+
+
+def test_ci_submit_rejects_split_group_disagreement(
+    ci_submit_module: ModuleType,
+    tmp_path: Path,
+) -> None:
     test_list_path = tmp_path / "test_list.txt"
     test_list_path.write_text(
         "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n",
@@ -202,7 +248,10 @@ def test_ci_submit_rejects_split_group_disagreement(ci_submit_module, tmp_path):
         )
 
 
-def test_ci_submit_rejects_missing_pytest_split_durations(ci_submit_module, tmp_path):
+def test_ci_submit_rejects_missing_pytest_split_durations(
+    ci_submit_module: ModuleType,
+    tmp_path: Path,
+) -> None:
     test_list_path = tmp_path / "test_list.txt"
     test_list_path.write_text(
         "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300-kimi]\n",

From 79f24c6a2b4f04469575e86514e10d38494371ca Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 11:10:05 -0700
Subject: [PATCH 05/13] [NVBUG-6481034][refactor] Use one shard selection path

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 jenkins/scripts/perf/submit.py | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py
index 259ef6c307c4..72b176df3a95 100755
--- a/jenkins/scripts/perf/submit.py
+++ b/jenkins/scripts/perf/submit.py
@@ -214,24 +214,13 @@ def select_test_case_line(test_list_path, llm_src, script_prefix_lines, split_gr
     return selected[0]
 
 
-def parse_test_case_name(test_list_path, llm_src, split_group=0, selected_line=None):
-    """Parse the selected line of the test list.
+def parse_test_case_name(llm_src, selected_line):
+    """Parse the selected test-list line.
 
     Returns (config_yaml_path, server_name, benchmark_mode, runtime_mode).
     See the module docstring for the supported test name shapes.
     """
-    if selected_line is not None:
-        line = selected_line
-    else:
-        lines = _read_test_list_lines(test_list_path)
-        if split_group > 0:
-            if split_group > len(lines):
-                raise ValueError(
-                    f"split_group {split_group} exceeds number of tests in test list ({len(lines)})"
-                )
-            line = lines[split_group - 1]
-        else:
-            line = lines[0]
+    line = selected_line
 
     if "[" not in line or "]" not in line:
         raise ValueError(f"Invalid test list format. Expected name with brackets: {line}")
@@ -659,10 +648,8 @@ def main():
         args.split_group,
     )
     config_yaml, server_name, benchmark_mode, runtime_mode = parse_test_case_name(
-        args.test_list,
         args.llm_src,
-        args.split_group,
-        selected_line=selected_test_line,
+        selected_test_line,
     )
 
     with open(config_yaml, "r") as f:

From 396255313a0b8b08e885a88f912123c7719b662b Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Fri, 31 Jul 2026 12:47:48 -0700
Subject: [PATCH 06/13] [https://nvbugs/6487038][fix] Bound GEN log sentinel
 wait

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 .../integration/defs/perf/test_perf_sanity.py | 75 ++++++++++++-------
 1 file changed, 46 insertions(+), 29 deletions(-)

diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py
index 4e86d5a34eba..f315e441e15c 100644
--- a/tests/integration/defs/perf/test_perf_sanity.py
+++ b/tests/integration/defs/perf/test_perf_sanity.py
@@ -104,6 +104,10 @@ def ensure_bench_serving_repo() -> str:
 # once EVERY ctx/gen worker has finished model load + autotune + warmup.
 AGG_SERVER_READY_TIMEOUT = 1800
 DISAGG_SERVER_READY_TIMEOUT = 3600
+# GEN workers normally reap within seconds after benchmark_status is written.
+# Keep this well below the whole-test timeout so a stuck multi-node srun cannot
+# turn the optional log-flush synchronization into a pytest/Slurm cancellation.
+GEN_LOG_SENTINEL_TIMEOUT = 120
 
 
 def server_ready_timeout(default: int, mode: str) -> int:
@@ -1348,7 +1352,11 @@ def wait_for_benchmark_ready(
                 )
             time.sleep(10)
 
-    def wait_for_gen_log_sentinels(self, poll_interval: float = 2.0) -> bool:
+    def wait_for_gen_log_sentinels(
+        self,
+        timeout: float = GEN_LOG_SENTINEL_TIMEOUT,
+        poll_interval: float = 2.0,
+    ) -> bool:
         """Block until every gen worker signals that its log is fully written.
 
         Each gen worker's srun in slurm_launch_draft.sh redirects all of its
@@ -1357,25 +1365,27 @@ def wait_for_gen_log_sentinels(self, poll_interval: float = 2.0) -> bool:
         flushed). The benchmark writes benchmark_status *before* calling this,
         which is what lets the gen srun exit — so this is not circular.
 
-        Returns True once all sentinels exist, or False if self.timeout is
-        reached first. On False the caller still parses whatever is on disk:
-        the sentinel is a correctness optimization against reading a
-        mid-flush log (nvbugs 6487036 / 6487040), never a hang risk for CI.
+        Returns True once all sentinels exist, or False if the dedicated
+        sentinel timeout is reached first. On False the caller still parses
+        whatever is on disk: the sentinel is a correctness optimization
+        against reading a mid-flush log, not a reason to consume the whole-test
+        timeout and trigger Slurm's kill-on-bad-exit cascade (nvbugs 6487036 /
+        6487040 / 6487038).
         """
         sentinels = [
             os.path.join(self.test_output_dir, f"gen_server_{i}.done")
             for i in range(self.num_gen_servers)
         ]
-        start_time = time.time()
+        start_time = time.monotonic()
         while True:
             missing = [p for p in sentinels if not os.path.exists(p)]
             if not missing:
                 print_info("All gen worker log sentinels present; log flush complete.")
                 return True
-            elapsed_time = time.time() - start_time
-            if elapsed_time > self.timeout:
+            elapsed_time = time.monotonic() - start_time
+            if elapsed_time > timeout:
                 print_info(
-                    f"Timeout ({self.timeout}s) waiting for gen worker log "
+                    f"Timeout ({timeout}s) waiting for gen worker log "
                     f"sentinels {missing}; parsing current log contents."
                 )
                 return False
@@ -1504,6 +1514,9 @@ def run_cmd(self, server_idx: int) -> List[str]:
             # the loop (as before) could read a truncated / not-yet-flushed log
             # and report a wrong mean (nvbugs 6487036 / 6487040).
             pending_device_step_time: List[dict] = []
+            collect_device_step_time = (
+                configs_for_idx is not None and configs_for_idx[2].benchmark_mode == "gen_only"
+            )
             try:
                 disagg_server_hostname, disagg_server_port = (
                     self._get_disagg_server_hostname_and_port(server_idx)
@@ -1535,11 +1548,15 @@ def run_cmd(self, server_idx: int) -> List[str]:
                         )
                         print_info(f"Starting benchmark. cmd is {client_cmd_with_port}")
 
-                        # Snapshot gen_server log sizes so the per-client
-                        # average covers only iterations driven by this client.
-                        gen_log_start_offsets = gen_worker_log_sizes(
-                            self.test_output_dir, self.num_gen_servers
-                        )
+                        # Snapshot gen_server log sizes so the gen_only
+                        # per-client average covers only iterations driven by
+                        # this client. Other modes do not emit this metric and
+                        # must not wait for the GEN teardown sentinel.
+                        gen_log_start_offsets = None
+                        if collect_device_step_time:
+                            gen_log_start_offsets = gen_worker_log_sizes(
+                                self.test_output_dir, self.num_gen_servers
+                            )
 
                         bench_env = copy.deepcopy(os.environ)
                         if client_config:
@@ -1554,16 +1571,17 @@ def run_cmd(self, server_idx: int) -> List[str]:
                             benchmark_ctx.write(output)
 
                         outputs.append(output)
-                        # Defer the gen-worker device-step-time parse until the
-                        # gen logs are flushed (see below); remember where to
-                        # write the summary back.
-                        pending_device_step_time.append(
-                            {
-                                "output_index": len(outputs) - 1,
-                                "benchmark_file_path": benchmark_file_path,
-                                "start_offsets": gen_log_start_offsets,
-                            }
-                        )
+                        if collect_device_step_time:
+                            # Defer the gen-worker device-step-time parse until
+                            # the gen logs are flushed (see below); remember
+                            # where to write the summary back.
+                            pending_device_step_time.append(
+                                {
+                                    "output_index": len(outputs) - 1,
+                                    "benchmark_file_path": benchmark_file_path,
+                                    "start_offsets": gen_log_start_offsets,
+                                }
+                            )
                     else:
                         print_info(
                             f"Skipping perf benchmark for client {client_idx}: "
@@ -1599,11 +1617,10 @@ def run_cmd(self, server_idx: int) -> List[str]:
 
             # benchmark_status is written, so the gen workers can now stop and
             # their srun will exit and drop gen_server_{i}.done. Wait once for
-            # those sentinels (bounded by self.timeout), then parse each
-            # benchmark client's gen-worker device step time a single time: the
-            # flushed log is complete, so no settle polling is needed. Only
-            # gen_only runs emit prev_device_step_time; other modes parse to
-            # None and skip the summary line.
+            # those sentinels (bounded independently of the whole-test timeout),
+            # then parse each benchmark client's gen-worker device step time a
+            # single time. Only gen_only runs populate this queue; other modes
+            # skip both the sentinel wait and device-step-time parsing.
             if pending_device_step_time:
                 self.wait_for_gen_log_sentinels()
                 for record in pending_device_step_time:

From d3e54b6ed3824e0df3b9cef424c44e937e0db65d Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Sat, 1 Aug 2026 18:27:14 -0700
Subject: [PATCH 07/13] [NVBUG-6487038][fix] Address perf review feedback

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 requirements-dev.txt                            |  2 +-
 tests/integration/defs/perf/test_perf_sanity.py | 15 +++++++--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index d5dcc3a2bf7e..fe640557eb2a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -22,7 +22,7 @@ pytest-env
 pytest-forked
 pytest-xdist
 pytest-timeout
-pytest-split
+pytest-split==0.10.0
 pytest-mock
 pytest-threadleak
 pytest-unused-fixtures
diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py
index f315e441e15c..ed562571d2a0 100644
--- a/tests/integration/defs/perf/test_perf_sanity.py
+++ b/tests/integration/defs/perf/test_perf_sanity.py
@@ -1366,11 +1366,11 @@ def wait_for_gen_log_sentinels(
         which is what lets the gen srun exit — so this is not circular.
 
         Returns True once all sentinels exist, or False if the dedicated
-        sentinel timeout is reached first. On False the caller still parses
-        whatever is on disk: the sentinel is a correctness optimization
-        against reading a mid-flush log, not a reason to consume the whole-test
-        timeout and trigger Slurm's kill-on-bad-exit cascade (nvbugs 6487036 /
-        6487040 / 6487038).
+        sentinel timeout is reached first. On False the caller skips the
+        device-step metric because the GEN logs may still be incomplete. The
+        bounded wait prevents a stuck multi-node srun from consuming the
+        whole-test timeout and triggering Slurm's kill-on-bad-exit cascade
+        (nvbugs 6487036 / 6487040 / 6487038).
         """
         sentinels = [
             os.path.join(self.test_output_dir, f"gen_server_{i}.done")
@@ -1386,7 +1386,7 @@ def wait_for_gen_log_sentinels(
             if elapsed_time > timeout:
                 print_info(
                     f"Timeout ({timeout}s) waiting for gen worker log "
-                    f"sentinels {missing}; parsing current log contents."
+                    f"sentinels {missing}; skipping the device-step metric."
                 )
                 return False
             print_info(
@@ -1621,8 +1621,7 @@ def run_cmd(self, server_idx: int) -> List[str]:
             # then parse each benchmark client's gen-worker device step time a
             # single time. Only gen_only runs populate this queue; other modes
             # skip both the sentinel wait and device-step-time parsing.
-            if pending_device_step_time:
-                self.wait_for_gen_log_sentinels()
+            if pending_device_step_time and self.wait_for_gen_log_sentinels():
                 for record in pending_device_step_time:
                     device_step_time_mean = parse_gen_worker_device_step_time(
                         self.test_output_dir,

From a469ce22cdf58418bbf9a5de67082512e195f6f1 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 10:29:16 -0700
Subject: [PATCH 08/13] [NVBUG-6487038][fix] Preserve GEN log metric fallback

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 .../integration/defs/perf/test_perf_sanity.py | 79 +++++++++++--------
 .../scripts/test_perf_sanity_helpers.py       | 75 ++++++++++++++++++
 2 files changed, 123 insertions(+), 31 deletions(-)
 create mode 100644 tests/unittest/scripts/test_perf_sanity_helpers.py

diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py
index ed562571d2a0..35d1fa872022 100644
--- a/tests/integration/defs/perf/test_perf_sanity.py
+++ b/tests/integration/defs/perf/test_perf_sanity.py
@@ -342,13 +342,14 @@ def parse_gen_worker_device_step_time(
     end-of-file are considered for gen_server_{i}.log — used to slice out a
     single client's iteration segment.
 
-    The log is read exactly once. The caller (DisaggTestCmds.run_cmd) blocks
-    on the gen_server_{i}.done sentinels before calling this, so every gen
-    srun has already exited and its &> aggregate log is fully flushed — there
-    is no partially-written tail to poll for. This replaces the earlier
-    settle-poll heuristic, which could return a mean over a truncated prefix
-    when it accepted the first repeated row count while the log was still
-    flushing across NFS (nvbugs 6487036 / 6487040).
+    The log is read exactly once. The caller (DisaggTestCmds.run_cmd) normally
+    waits for the gen_server_{i}.done sentinels first, so every gen srun has
+    exited and its &> aggregate log is fully flushed. If the dedicated
+    sentinel wait expires, the caller parses the current contents instead of
+    consuming the whole-test timeout; a missing metric then hard-fails before
+    upload. This replaces the earlier settle-poll heuristic, which could
+    accept a truncated prefix while the log was still flushing across NFS
+    (nvbugs 6487036 / 6487040 / 6487038).
     """
     per_file_scans, _total_count = _scan_gen_worker_device_step_time(
         output_dir, num_gen_servers, start_offsets
@@ -1366,11 +1367,10 @@ def wait_for_gen_log_sentinels(
         which is what lets the gen srun exit — so this is not circular.
 
         Returns True once all sentinels exist, or False if the dedicated
-        sentinel timeout is reached first. On False the caller skips the
-        device-step metric because the GEN logs may still be incomplete. The
-        bounded wait prevents a stuck multi-node srun from consuming the
-        whole-test timeout and triggering Slurm's kill-on-bad-exit cascade
-        (nvbugs 6487036 / 6487040 / 6487038).
+        sentinel timeout is reached first. On False the caller falls back to
+        parsing the current log contents. The bounded wait prevents a stuck
+        multi-node srun from consuming the whole-test timeout and triggering
+        Slurm's kill-on-bad-exit cascade (nvbugs 6487036 / 6487040 / 6487038).
         """
         sentinels = [
             os.path.join(self.test_output_dir, f"gen_server_{i}.done")
@@ -1386,7 +1386,7 @@ def wait_for_gen_log_sentinels(
             if elapsed_time > timeout:
                 print_info(
                     f"Timeout ({timeout}s) waiting for gen worker log "
-                    f"sentinels {missing}; skipping the device-step metric."
+                    f"sentinels {missing}; parsing current log contents."
                 )
                 return False
             print_info(
@@ -1394,6 +1394,36 @@ def wait_for_gen_log_sentinels(
             )
             time.sleep(poll_interval)
 
+    def _append_gen_worker_device_step_time(
+        self,
+        pending_device_step_time: List[dict],
+        outputs: List[str],
+    ) -> None:
+        """Wait for GEN log flush, then append each pending client metric.
+
+        A sentinel timeout is a bounded teardown fallback, not a reason to
+        discard metrics that are already present in the GEN logs. If the
+        fallback parse finds no usable metric, check_test_failure still fails
+        the gen_only run before results are uploaded.
+        """
+        if not pending_device_step_time:
+            return
+
+        self.wait_for_gen_log_sentinels()
+        for record in pending_device_step_time:
+            device_step_time_mean = parse_gen_worker_device_step_time(
+                self.test_output_dir,
+                self.num_gen_servers,
+                start_offsets=record["start_offsets"],
+            )
+            if device_step_time_mean is None:
+                continue
+            summary_line = f"Average Per Iter Device Step Time (ms): {device_step_time_mean:.2f}"
+            with open(record["benchmark_file_path"], "a") as benchmark_ctx:
+                benchmark_ctx.write(f"\n{summary_line}\n")
+            idx = record["output_index"]
+            outputs[idx] = f"{outputs[idx]}\n{summary_line}\n"
+
     def get_server_logs(self, server_idx: int) -> List[str]:
         server_logs = []
         for i in range(self.num_ctx_servers):
@@ -1507,7 +1537,7 @@ def run_cmd(self, server_idx: int) -> List[str]:
 
         elif self.disagg_serving_type == "BENCHMARK":
             # Perf-benchmark clients whose gen-worker device step time must be
-            # parsed once the gen logs are flushed. The parse is deferred out of
+            # parsed after the gen-log flush wait. The parse is deferred out of
             # the client loop because gen_server_*.log keeps being written until
             # the gen srun exits, and the gen srun only exits after
             # benchmark_status is written in the finally below. Parsing inside
@@ -1619,23 +1649,10 @@ def run_cmd(self, server_idx: int) -> List[str]:
             # their srun will exit and drop gen_server_{i}.done. Wait once for
             # those sentinels (bounded independently of the whole-test timeout),
             # then parse each benchmark client's gen-worker device step time a
-            # single time. Only gen_only runs populate this queue; other modes
-            # skip both the sentinel wait and device-step-time parsing.
-            if pending_device_step_time and self.wait_for_gen_log_sentinels():
-                for record in pending_device_step_time:
-                    device_step_time_mean = parse_gen_worker_device_step_time(
-                        self.test_output_dir,
-                        self.num_gen_servers,
-                        start_offsets=record["start_offsets"],
-                    )
-                    if device_step_time_mean is not None:
-                        summary_line = (
-                            f"Average Per Iter Device Step Time (ms): {device_step_time_mean:.2f}"
-                        )
-                        with open(record["benchmark_file_path"], "a") as benchmark_ctx:
-                            benchmark_ctx.write(f"\n{summary_line}\n")
-                        idx = record["output_index"]
-                        outputs[idx] = f"{outputs[idx]}\n{summary_line}\n"
+            # single time. A timeout falls back to the current log contents.
+            # Only gen_only runs populate this queue; other modes skip both the
+            # sentinel wait and device-step-time parsing.
+            self._append_gen_worker_device_step_time(pending_device_step_time, outputs)
 
         return outputs
 
diff --git a/tests/unittest/scripts/test_perf_sanity_helpers.py b/tests/unittest/scripts/test_perf_sanity_helpers.py
new file mode 100644
index 000000000000..9f8a112ee54f
--- /dev/null
+++ b/tests/unittest/scripts/test_perf_sanity_helpers.py
@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("torch")
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.insert(0, str(REPO_ROOT / "tests" / "integration"))
+
+from defs.perf import test_perf_sanity as perf_sanity  # noqa: E402
+
+
+def test_sentinel_timeout_falls_back_to_current_gen_logs(monkeypatch, tmp_path):
+    benchmark_log = tmp_path / "trtllm-benchmark.0.0.log"
+    benchmark_log.write_text("benchmark output", encoding="utf-8")
+    outputs = ["benchmark output"]
+    pending = [
+        {
+            "output_index": 0,
+            "benchmark_file_path": str(benchmark_log),
+            "start_offsets": [10, 20],
+        }
+    ]
+    commands = perf_sanity.DisaggTestCmds(
+        server_cmds=[],
+        client_cmds={},
+        timeout=1,
+        hostname="localhost",
+        disagg_serving_type="BENCHMARK",
+        num_ctx_servers=1,
+        num_gen_servers=2,
+        output_dir=str(tmp_path),
+        test_output_dir=str(tmp_path),
+    )
+
+    monkeypatch.setattr(
+        perf_sanity.DisaggTestCmds,
+        "wait_for_gen_log_sentinels",
+        lambda self: False,
+    )
+    parse_calls = []
+
+    def parse_device_step_time(output_dir, num_gen_servers, start_offsets):
+        parse_calls.append((output_dir, num_gen_servers, start_offsets))
+        return 7.25
+
+    monkeypatch.setattr(
+        perf_sanity,
+        "parse_gen_worker_device_step_time",
+        parse_device_step_time,
+    )
+
+    commands._append_gen_worker_device_step_time(pending, outputs)
+
+    assert parse_calls == [(str(tmp_path), 2, [10, 20])]
+    assert outputs == ["benchmark output\nAverage Per Iter Device Step Time (ms): 7.25\n"]
+    assert benchmark_log.read_text(encoding="utf-8").endswith(
+        "\nAverage Per Iter Device Step Time (ms): 7.25\n"
+    )

From e8ab8ad1f1b9c43a029e8fe09b4769aac6267802 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 10:58:55 -0700
Subject: [PATCH 09/13] [NVBUG-6487038][test] Preserve pytest-split contract

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 requirements-dev.txt                               | 2 +-
 tests/unittest/scripts/test_perf_sanity_helpers.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index fe640557eb2a..d5dcc3a2bf7e 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -22,7 +22,7 @@ pytest-env
 pytest-forked
 pytest-xdist
 pytest-timeout
-pytest-split==0.10.0
+pytest-split
 pytest-mock
 pytest-threadleak
 pytest-unused-fixtures
diff --git a/tests/unittest/scripts/test_perf_sanity_helpers.py b/tests/unittest/scripts/test_perf_sanity_helpers.py
index 9f8a112ee54f..60c172199ed0 100644
--- a/tests/unittest/scripts/test_perf_sanity_helpers.py
+++ b/tests/unittest/scripts/test_perf_sanity_helpers.py
@@ -18,7 +18,7 @@
 
 import pytest
 
-pytest.importorskip("torch")
+pytest.importorskip("torch._inductor")
 
 REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
 sys.path.insert(0, str(REPO_ROOT / "tests" / "integration"))

From 1cefe9c803be3c996f9488d88259d55c0e4d3673 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 3 Aug 2026 11:10:11 -0700
Subject: [PATCH 10/13] [NVBUG-6487038][test] Type sentinel fallback test

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 tests/unittest/scripts/test_perf_sanity_helpers.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/tests/unittest/scripts/test_perf_sanity_helpers.py b/tests/unittest/scripts/test_perf_sanity_helpers.py
index 60c172199ed0..f344529c2f35 100644
--- a/tests/unittest/scripts/test_perf_sanity_helpers.py
+++ b/tests/unittest/scripts/test_perf_sanity_helpers.py
@@ -26,7 +26,10 @@
 from defs.perf import test_perf_sanity as perf_sanity  # noqa: E402
 
 
-def test_sentinel_timeout_falls_back_to_current_gen_logs(monkeypatch, tmp_path):
+def test_sentinel_timeout_falls_back_to_current_gen_logs(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
     benchmark_log = tmp_path / "trtllm-benchmark.0.0.log"
     benchmark_log.write_text("benchmark output", encoding="utf-8")
     outputs = ["benchmark output"]
@@ -54,9 +57,13 @@ def test_sentinel_timeout_falls_back_to_current_gen_logs(monkeypatch, tmp_path):
         "wait_for_gen_log_sentinels",
         lambda self: False,
     )
-    parse_calls = []
+    parse_calls: list[tuple[str, int, list[int]]] = []
 
-    def parse_device_step_time(output_dir, num_gen_servers, start_offsets):
+    def parse_device_step_time(
+        output_dir: str,
+        num_gen_servers: int,
+        start_offsets: list[int],
+    ) -> float:
         parse_calls.append((output_dir, num_gen_servers, start_offsets))
         return 7.25
 

From ce69d75ba58a1ab521fa61624bec9173cbb7eaad Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 27 Jul 2026 16:48:54 -0700
Subject: [PATCH 11/13] [https://nvbugs/6487038][fix] extend GB300 Kimi KV
 transfer timeout

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 ...4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
index 7bbe295f8371..12b245ec091c 100644
--- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
+++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
@@ -64,6 +64,7 @@ worker_config:
       max_tokens_in_buffer: 16384
       backend: NIXL
       transceiver_runtime: PYTHON
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     trust_remote_code: true
     num_postprocess_workers: 4
@@ -90,5 +91,6 @@ worker_config:
       max_tokens_in_buffer: 16384
       backend: NIXL
       transceiver_runtime: PYTHON
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     trust_remote_code: true

From dd8607bd47736643bfd4751db01cd76d4174bd98 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Mon, 27 Jul 2026 16:56:31 -0700
Subject: [PATCH 12/13] [https://nvbugs/6487038][test] unwaive GB300 Kimi KV
 timeout cases

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 tests/integration/test_lists/waives.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt
index 647729a6c0da..91815d8f8938 100644
--- a/tests/integration/test_lists/waives.txt
+++ b/tests/integration/test_lists/waives.txt
@@ -380,7 +380,6 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6426890)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6550133)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6550133)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6550133)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6374872)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6550133)

From e4f6185c312a30b2e5dad03c0efce1e4c30263f5 Mon Sep 17 00:00:00 2001
From: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
Date: Thu, 30 Jul 2026 15:16:48 -0700
Subject: [PATCH 13/13] [https://nvbugs/6487038][test] validate GB300 Kimi with
 default KV timeout

Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
---
 ...4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
index 12b245ec091c..7bbe295f8371 100644
--- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
+++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL.yaml
@@ -64,7 +64,6 @@ worker_config:
       max_tokens_in_buffer: 16384
       backend: NIXL
       transceiver_runtime: PYTHON
-      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     trust_remote_code: true
     num_postprocess_workers: 4
@@ -91,6 +90,5 @@ worker_config:
       max_tokens_in_buffer: 16384
       backend: NIXL
       transceiver_runtime: PYTHON
-      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     trust_remote_code: true