diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 67ca753bef1f..21518fb61cc9 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1371,15 +1371,12 @@ def test_kv_cache_v2_nixl_python(self): } disaggregated_server_config = { "hostname": "localhost", - "port": 8000, "backend": "pytorch", "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] + "num_instances": 1 }, "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] + "num_instances": 1 } } with launch_disaggregated_llm(disaggregated_server_config, @@ -1843,6 +1840,61 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, max_workers=128) as llm: run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + @pytest.mark.skip_less_device(4) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} + ctx_server_config = { + "disable_overlap_scheduler": True, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + gen_server_config = { + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1, + }, + "generation_servers": { + "num_instances": 1, + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + @pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) @skip_pre_hopper @@ -2754,3 +2806,66 @@ def test_auto_dtype(self): server_waiting_timeout=3600) as llm: task = MMLU(self.MODEL_NAME) task.evaluate(llm, is_integration_test=True) + + +@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) +@skip_pre_blackwell +@pytest.mark.skip_less_device_memory(80000) +class TestDeepSeekR1(LlmapiAccuracyTestHarness): + MODEL_NAME = "deepseek-ai/DeepSeek-R1" + MODEL_PATH = f"{llm_models_root()}/DeepSeek-R1-0528-FP4-v2" + + @pytest.mark.skip_less_device(4) + @pytest.mark.parametrize("use_kv_cache_manager_v2", [False], + ids=["cache_mgr_v1"]) + def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2): + """Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" + max_num_tokens = 8192 + moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens} + ctx_server_config = { + "disable_overlap_scheduler": True, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + gen_server_config = { + "disable_overlap_scheduler": False, + "moe_config": moe_config, + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + "use_kv_cache_manager_v2": use_kv_cache_manager_v2 + }, + "cache_transceiver_config": { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + "max_tokens_in_buffer": 4096 + }, + "tensor_parallel_size": 2, + "moe_expert_parallel_size": 2, + "enable_autotuner": False, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1, + }, + "generation_servers": { + "num_instances": 1, + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, gen_server_config, + self.MODEL_PATH) as llm: + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 0be227bc27e8..4e3bd088ee17 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -1,8 +1,10 @@ +accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1dp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp1cp4] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp1tp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype_with_helix[fifo-cudagraph:with_padding-pp2tp1cp2] +accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b300.yml b/tests/integration/test_lists/test-db/l0_dgx_b300.yml index 5b53885685f9..64f4d5632310 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b300.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b300.yml @@ -64,6 +64,8 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend + - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] + - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python