Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions tests/integration/defs/.test_durations
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 585.7382,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False]": 618.2120759493671,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 669.1686,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=False]": 278.947,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True]": 155.35775,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False]": 211.017,
Expand All @@ -10,13 +9,10 @@
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[adp-mtp2]": 320.45259493670886,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_first[noadp-mtp0]": 467.45696296296296,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec": 174.1415,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync[python]": 272.31831645569616,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0]": 144.22060000000002,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2]": 142.88979999999998,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0]": 276.96827272727273,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2]": 178.90498823529413,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_kv_cache_v2_nixl_python": 285.9667195121951,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_nixl_backend": 274.9563766233766,
"accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype": 681.8583145161291,
"accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1]": 992.9112333333333,
"accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1]": 333.1872,
Expand Down Expand Up @@ -630,7 +626,6 @@
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8]": 144.85461842105263,
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8]": 120.784,
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]": 138.068,
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_transceiver_runtime_python[DeepSeek-V3-Lite-fp8]": 119.97518421052632,
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8]": 95.34589473684211,
"disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8]": 135.01195707070707,
"disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0]": 71.34835714285714,
Expand Down
234 changes: 51 additions & 183 deletions tests/integration/defs/accuracy/test_disaggregated_serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -1044,62 +1044,25 @@ class TestDeepSeekV3Lite(LlmapiAccuracyTestHarness):
@pytest.mark.skip_less_device(2)
@pytest.mark.skip_less_device_memory(60000)
@skip_no_hopper
def test_nixl_backend(self):
ctx_server_config = {
"disable_overlap_scheduler": True,
"cache_transceiver_config": {
"backend": "NIXL",
"max_tokens_in_buffer": 4096
}
}
gen_server_config = {
"disable_overlap_scheduler": True,
"cache_transceiver_config": {
"backend": "NIXL",
"max_tokens_in_buffer": 4096
}
}
disaggregated_server_config = {
"hostname": "localhost",
"backend": "pytorch",
"context_servers": {
"num_instances": 1
},
"generation_servers": {
"num_instances": 1
}
}
with launch_disaggregated_llm(disaggregated_server_config,
ctx_server_config, gen_server_config,
self.MODEL_PATH) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"])

@pytest.mark.skip_less_device(2)
@pytest.mark.skip_less_device_memory(60000)
@skip_no_hopper
@pytest.mark.parametrize("transceiver_runtime", ["PYTHON", "CPP"],
ids=["python", "cpp"])
def test_gen_only_sync(self, transceiver_runtime):
"""Test gen-only synchronous KV transfer with each NIXL runtime.
def test_gen_only_sync(self):
Comment thread
asfiyab-nvidia marked this conversation as resolved.
"""Test gen-only synchronous KV transfer with PYTHON NIXL runtime.

Sets TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 so the gen worker calls
the blocking request_and_receive_sync path. The C++ variant uses a
bounded client timeout so a stuck transfer fails this test instead of
waiting for its outer one-hour timeout.
the blocking request_and_receive_sync path.
"""
ctx_server_config = {
"disable_overlap_scheduler": True,
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": transceiver_runtime,
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096,
},
}
gen_server_config = {
"disable_overlap_scheduler": True,
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": transceiver_runtime,
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096,
},
}
Expand All @@ -1120,10 +1083,8 @@ def test_gen_only_sync(self, transceiver_runtime):
self.MODEL_PATH,
# Apply to both servers: gen worker uses sync receive path.
extra_env={"TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP": "1"},
request_timeout_s=(120 if transceiver_runtime == "CPP" else
DEFAULT_REQUEST_TIMEOUT_S),
request_max_retries=(0
if transceiver_runtime == "CPP" else None),
request_timeout_s=DEFAULT_REQUEST_TIMEOUT_S,
request_max_retries=None,
) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"])

Expand Down Expand Up @@ -1160,47 +1121,6 @@ def test_gen_only_spec_dec(self):
tensor_parallel_size=4) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"])

@pytest.mark.skip_less_device(8)
@parametrize_with_ids("overlap_scheduler", [True, False])
@parametrize_with_ids("mtp_nextn", [0, 2])
@pytest.mark.skip_less_device(8)
def test_auto_dtype(self, overlap_scheduler, mtp_nextn):
ctx_server_config = {"disable_overlap_scheduler": True}
gen_server_config = {"disable_overlap_scheduler": not overlap_scheduler}
ctx_server_config["cache_transceiver_config"] = {
"backend": "DEFAULT",
"max_tokens_in_buffer": 4096
}
gen_server_config["cache_transceiver_config"] = {
"backend": "DEFAULT",
"max_tokens_in_buffer": 4096
}
if mtp_nextn > 0:
ctx_server_config["speculative_config"] = {
"decoding_type": "MTP",
"max_draft_len": mtp_nextn
}
gen_server_config["speculative_config"] = {
"decoding_type": "MTP",
"max_draft_len": mtp_nextn
}
disaggregated_server_config = {
"hostname": "localhost",
"backend": "pytorch",
"context_servers": {
"num_instances": 1
},
"generation_servers": {
"num_instances": 1
}
}
with launch_disaggregated_llm(disaggregated_server_config,
ctx_server_config,
gen_server_config,
self.MODEL_PATH,
tensor_parallel_size=4) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"])

@skip_pre_blackwell
@pytest.mark.skip_less_device(8)
@pytest.mark.parametrize("gen_pp,gen_tp,gen_cp,enable_attention_dp", [
Expand Down Expand Up @@ -1286,33 +1206,22 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config,
self.MODEL_PATH) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"])

@pytest.mark.skip_less_device(2)
@pytest.mark.skip_less_device_memory(60000)
@pytest.mark.skip_less_device(8)
@parametrize_with_ids("overlap_scheduler", [True, False])
@parametrize_with_ids("mtp_nextn", [0, 2])
@pytest.mark.parametrize("backend", ["xgrammar", "llguidance"])
def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker):
mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"})
ctx_server_config = {
"disable_overlap_scheduler": True,
"kv_cache_config": {
"free_gpu_memory_fraction": 0.8,
},
"guided_decoding_backend": backend,
"cache_transceiver_config": {
"backend": "DEFAULT",
"max_tokens_in_buffer": 4096
}
@pytest.mark.skip_less_device(8)
def test_auto_dtype(self, overlap_scheduler, mtp_nextn):
ctx_server_config = {"disable_overlap_scheduler": True}
gen_server_config = {"disable_overlap_scheduler": not overlap_scheduler}
ctx_server_config["cache_transceiver_config"] = {
"backend": "NIXL",
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
}
gen_server_config = {
"disable_overlap_scheduler": False,
"kv_cache_config": {
"free_gpu_memory_fraction": 0.8,
},
"guided_decoding_backend": backend,
"cache_transceiver_config": {
"backend": "DEFAULT",
"max_tokens_in_buffer": 4096
}
gen_server_config["cache_transceiver_config"] = {
"backend": "NIXL",
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
}
if mtp_nextn > 0:
ctx_server_config["speculative_config"] = {
Expand All @@ -1334,37 +1243,51 @@ def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker):
}
}
with launch_disaggregated_llm(disaggregated_server_config,
ctx_server_config, gen_server_config,
self.MODEL_PATH) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["JsonModeEval"])
ctx_server_config,
gen_server_config,
self.MODEL_PATH,
tensor_parallel_size=4) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["MMLU", "GSM8K"])

@pytest.mark.skip_less_device(2)
@pytest.mark.skip_less_device_memory(60000)
@skip_pre_hopper
def test_kv_cache_v2_nixl_python(self):
"""Test with use_kv_cache_manager_v2=True, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON."""
@parametrize_with_ids("mtp_nextn", [0, 2])
@pytest.mark.parametrize("backend", ["xgrammar", "llguidance"])
def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker):
mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"})
ctx_server_config = {
"disable_overlap_scheduler": True,
"kv_cache_config": {
"enable_block_reuse": False,
"use_kv_cache_manager_v2": True
"free_gpu_memory_fraction": 0.8,
},
"guided_decoding_backend": backend,
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": "PYTHON"
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
}
}
gen_server_config = {
"disable_overlap_scheduler": True,
"disable_overlap_scheduler": False,
"kv_cache_config": {
"enable_block_reuse": False,
"use_kv_cache_manager_v2": True
"free_gpu_memory_fraction": 0.8,
},
"guided_decoding_backend": backend,
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": "PYTHON"
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
}
}
if mtp_nextn > 0:
ctx_server_config["speculative_config"] = {
"decoding_type": "MTP",
"max_draft_len": mtp_nextn
}
gen_server_config["speculative_config"] = {
"decoding_type": "MTP",
"max_draft_len": mtp_nextn
}
disaggregated_server_config = {
"hostname": "localhost",
"backend": "pytorch",
Expand All @@ -1378,7 +1301,7 @@ def test_kv_cache_v2_nixl_python(self):
with launch_disaggregated_llm(disaggregated_server_config,
ctx_server_config, gen_server_config,
self.MODEL_PATH) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"])
run_accuracy_test(llm, self.MODEL_NAME, ["JsonModeEval"])

@pytest.mark.skip_less_device(4)
@pytest.mark.skip_less_device_memory(60000)
Expand Down Expand Up @@ -1680,7 +1603,8 @@ class TestDeepSeekV32Exp(LlmapiAccuracyTestHarness):
@pytest.mark.parametrize("overlap_scheduler", [False])
def test_auto_dtype(self, overlap_scheduler):
cache_transceiver_config = {
"backend": "DEFAULT",
"backend": "NIXL",
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
}
max_num_tokens = 8192
Expand Down Expand Up @@ -1740,62 +1664,6 @@ def test_auto_dtype(self, overlap_scheduler):
model_name=self.MODEL_NAME,
test_sets=["MMLU", "GSM8K"])

@pytest.mark.skip_less_device(4)
@pytest.mark.skip_less_device_memory(200000)
@pytest.mark.parametrize("use_kv_cache_manager_v2", [False],
ids=["cache_mgr_v1"])
def test_kv_cache_v2_nixl_python(self, use_kv_cache_manager_v2):
"""Test with KV cache manager v1, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON."""
max_num_tokens = 8192
moe_config = {"backend": "TRTLLM", "max_num_tokens": max_num_tokens}
ctx_server_config = {
"disable_overlap_scheduler": True,
"kv_cache_config": {
"free_gpu_memory_fraction": 0.5,
"enable_block_reuse": False,
"use_kv_cache_manager_v2": use_kv_cache_manager_v2
},
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
},
"tensor_parallel_size": 2,
"moe_expert_parallel_size": 2,
"enable_autotuner": False,
}
gen_server_config = {
"disable_overlap_scheduler": False,
"moe_config": moe_config,
"kv_cache_config": {
"free_gpu_memory_fraction": 0.5,
"enable_block_reuse": False,
"use_kv_cache_manager_v2": use_kv_cache_manager_v2
},
"cache_transceiver_config": {
"backend": "NIXL",
"transceiver_runtime": "PYTHON",
"max_tokens_in_buffer": 4096
},
"tensor_parallel_size": 2,
"moe_expert_parallel_size": 2,
"enable_autotuner": False,
}
disaggregated_server_config = {
"hostname": "localhost",
"backend": "pytorch",
"context_servers": {
"num_instances": 1,
},
"generation_servers": {
"num_instances": 1,
}
}
with launch_disaggregated_llm(disaggregated_server_config,
ctx_server_config, gen_server_config,
self.MODEL_PATH) as llm:
run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"])


@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT)
@skip_pre_hopper
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ context_servers:
event_buffer_max_size: 1024
free_gpu_memory_fraction: 0.1
cache_transceiver_config:
backend: DEFAULT
backend: NIXL
transceiver_runtime: PYTHON
generation_servers:
num_instances: 2
router:
Expand All @@ -29,4 +30,5 @@ generation_servers:
event_buffer_max_size: 1024
free_gpu_memory_fraction: 0.1
cache_transceiver_config:
backend: DEFAULT
backend: NIXL
transceiver_runtime: PYTHON
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ context_servers:
enable_partial_reuse: true
event_buffer_max_size: 1024
cache_transceiver_config:
backend: DEFAULT
backend: NIXL
transceiver_runtime: PYTHON
generation_servers:
num_instances: 1
tensor_parallel_size: 1
Expand All @@ -30,4 +31,5 @@ generation_servers:
event_buffer_max_size: 1024
free_gpu_memory_fraction: 0.05
cache_transceiver_config:
backend: DEFAULT
backend: NIXL
transceiver_runtime: PYTHON
Loading
Loading