diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py index 4ddecb15fc..d4e35b47d4 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py @@ -62,6 +62,12 @@ _WEIGHTS_MOUNT = "/model-store" _SCRATCH_MOUNT = "/scratch" _LORA_MOUNT = "/scratch/loras" +# The adapters sidecar's `nemo services run` writes instance state under $XDG_STATE_HOME +# (default ~/.local/state), but the pod runs it as the vLLM uid (2000), which does not own +# the nmp-api image's $HOME (/home/nvs, uid 1000) -- so the default path is unwritable and +# the sidecar crash-loops. Redirect it to the writable scratch volume, outside the +# /scratch/loras subtree the adapters controller GCs. +_LORA_SIDECAR_STATE_DIR = f"{_SCRATCH_MOUNT}/.nmp-state" _SCRATCH_VOLUME_SIZE = "1Gi" @@ -145,6 +151,7 @@ def _lora_sidecar( "NMP_MODEL_ENTITY_WORKSPACE": entity_workspace, "NMP_MODEL_ENTITY_NAME": entity_name, "NMP_BASE_URL": platform.base_url, + "XDG_STATE_HOME": _LORA_SIDECAR_STATE_DIR, } if engine == ENGINE_VLLM: sidecar_env["VLLM_LORA_BASE_MODEL_OVERRIDE"] = MODEL_STORE_PATH diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py index 8d4334379a..e7bd907ad1 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py @@ -300,6 +300,9 @@ def test_lora_uses_native_sidecar_on_k8s_and_container_on_docker() -> None: assert sidecar.image == "registry/nmp-api:tag" env = {item.name: item.value for item in sidecar.env} assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" + # Sidecar `nemo services run` state must land on the writable scratch volume, not the + # nmp-api image's $HOME (unwritable under the pod's vLLM uid). See _lora_sidecar. + assert env["XDG_STATE_HOME"] == "/scratch/.nmp-state" assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" assert env["NMP_BASE_URL"] == "http://platform.example:8080" assert env["VLLM_ENDPOINT"] == "http://127.0.0.1:8000"