From 648b5a5f068839e13eafe36cc2648e65767677e0 Mon Sep 17 00:00:00 2001 From: Asfiya Baig Date: Mon, 13 Jul 2026 16:12:35 -0700 Subject: [PATCH 1/3] [TRTLLM-13948][test] Add DeepSeek R1/V3.2/V3-Lite disaggregated accuracy tests Signed-off-by: Asfiya Baig --- .../accuracy/test_disaggregated_serving.py | 133 ++++++++++++++++-- .../test_lists/qa/llm_function_core.txt | 7 +- .../test_lists/test-db/l0_dgx_b300.yml | 7 +- 3 files changed, 136 insertions(+), 11 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 67ca753bef1f..6a373460c6b6 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1345,13 +1345,15 @@ def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker): @pytest.mark.skip_less_device(2) @pytest.mark.skip_less_device_memory(60000) @skip_pre_hopper - def test_kv_cache_v2_nixl_python(self): - """Test with use_kv_cache_manager_v2=True, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], + ids=["cache_mgr_v1", "cache_mgr_v2"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" ctx_server_config = { "disable_overlap_scheduler": True, "kv_cache_config": { "enable_block_reuse": False, - "use_kv_cache_manager_v2": True + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 }, "cache_transceiver_config": { "backend": "NIXL", @@ -1362,7 +1364,7 @@ def test_kv_cache_v2_nixl_python(self): "disable_overlap_scheduler": True, "kv_cache_config": { "enable_block_reuse": False, - "use_kv_cache_manager_v2": True + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 }, "cache_transceiver_config": { "backend": "NIXL", @@ -1371,15 +1373,12 @@ def test_kv_cache_v2_nixl_python(self): } disaggregated_server_config = { "hostname": "localhost", - "port": 8000, "backend": "pytorch", "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] + "num_instances": 1 }, "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] + "num_instances": 1 } } with launch_disaggregated_llm(disaggregated_server_config, @@ -1843,6 +1842,61 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, max_workers=128) as llm: run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + @pytest.mark.skip_less_device(4) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], + ids=["cache_mgr_v1", "cache_mgr_v2"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} + ctx_server_config = { + "disable_overlap_scheduler": True, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + gen_server_config = { + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1, + }, + "generation_servers": { + "num_instances": 1, + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + @pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) @skip_pre_hopper @@ -2754,3 +2808,64 @@ def test_auto_dtype(self): server_waiting_timeout=3600) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm, is_integration_test=True) + + +@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) +@skip_pre_blackwell +@pytest.mark.skip_less_device_memory(80000) +class TestDeepSeekR1(LlmapiAccuracyTestHarness): + MODEL_NAME = "deepseek-ai/DeepSeek-R1" + MODEL_PATH = f"{llm_models_root()}/DeepSeek-R1-0528-FP4-v2" + + @pytest.mark.skip_less_device(4) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], + ids=["cache_mgr_v1", "cache_mgr_v2"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} + ctx_server_config = { + "disable_overlap_scheduler": True, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + } + gen_server_config = { + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1, + }, + "generation_servers": { + "num_instances": 1, + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 0be227bc27e8..ec1fae851905 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -1,8 +1,12 @@ +accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1dp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp2tp1cp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] @@ -24,7 +28,8 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b300.yml b/tests/integration/test_lists/test-db/l0_dgx_b300.yml index 5b53885685f9..4e2db110f307 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b300.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b300.yml @@ -64,8 +64,13 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend + - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] + - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v2] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python + - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v1] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] From 6e02fbd843eb8355d6995a3b4176fcf359ef5eb9 Mon Sep 17 00:00:00 2001 From: Asfiya Baig Date: Tue, 14 Jul 2026 10:56:10 -0700 Subject: [PATCH 2/3] changes: 1. Revert changes to DeepSeekV3-Lite test 2. Update DeepSeekR1 and DeepSeekV3.2 tests to test only cache_mgr_v1 Signed-off-by: Asfiya Baig --- .../accuracy/test_disaggregated_serving.py | 22 +++++++++---------- .../test_lists/qa/llm_function_core.txt | 5 +---- .../test_lists/test-db/l0_dgx_b300.yml | 5 +---- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 6a373460c6b6..fc6738396768 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1345,15 +1345,13 @@ def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker): @pytest.mark.skip_less_device(2) @pytest.mark.skip_less_device_memory(60000) @skip_pre_hopper - @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], - ids=["cache_mgr_v1", "cache_mgr_v2"]) - def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): - """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + def test_kv_cache_v2_nixl_python(self): + """Test with use_kv_cache_manager_v2=True, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" ctx_server_config = { "disable_overlap_scheduler": True, "kv_cache_config": { "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + "use_kv_cache_manager_v2": True }, "cache_transceiver_config": { "backend": "NIXL", @@ -1364,7 +1362,7 @@ def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): "disable_overlap_scheduler": True, "kv_cache_config": { "enable_block_reuse": False, - "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + "use_kv_cache_manager_v2": True }, "cache_transceiver_config": { "backend": "NIXL", @@ -1843,10 +1841,10 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) @pytest.mark.skip_less_device(4) - @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], - ids=["cache_mgr_v1", "cache_mgr_v2"]) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): - """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" max_num_tokens = 8192 moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} ctx_server_config = { @@ -2818,10 +2816,10 @@ class TestDeepSeekR1(LlmapiAccuracyTestHarness): MODEL_PATH = f"{llm_models_root()}/DeepSeek-R1-0528-FP4-v2" @pytest.mark.skip_less_device(4) - @pytest.mark.parametrize("use_kv_cache_manager_v2", [False, True], - ids=["cache_mgr_v1", "cache_mgr_v2"]) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): - """Test with KV cache manager v1 and v2, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" max_num_tokens = 8192 moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} ctx_server_config = { diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index ec1fae851905..4e3bd088ee17 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -1,12 +1,10 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1dp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp2tp1cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] @@ -28,8 +26,7 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b300.yml b/tests/integration/test_lists/test-db/l0_dgx_b300.yml index 4e2db110f307..64f4d5632310 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b300.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b300.yml @@ -65,12 +65,9 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python[cache_mgr_v2] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] From d555a539b14861540fe7f55238ffa6998139070d Mon Sep 17 00:00:00 2001 From: Asfiya Baig Date: Tue, 14 Jul 2026 18:31:04 -0700 Subject: [PATCH 3/3] disable autotuner for DeepSeekR1 test Signed-off-by: Asfiya Baig --- tests/integration/defs/accuracy/test_disaggregated_serving.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index fc6738396768..21518fb61cc9 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2836,6 +2836,7 @@ def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): }, "tensor_parallel_size": 2, "moe_expert_parallel_size": 2, + "enable_autotuner": False, } gen_server_config = { "disable_overlap_scheduler": False, @@ -2852,6 +2853,7 @@ def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): }, "tensor_parallel_size": 2, "moe_expert_parallel_size": 2, + "enable_autotuner": False, } disaggregated_server_config = { "hostname": "localhost",