fix(models): fix default model_puller image for NIMCache

docs/set-up/config-reference.mdx

@@ -603,6 +603,8 @@ models:
         default_vllm_group_id: 0
         # Kubernetes secret name for Files service authentication (HF_TOKEN) | default: 'nemo-models-files-token'
         files_auth_secret: nemo-models-files-token
+        # Image the NIM operator runs to fill the NIMCache from Files. We need this specific image because the operator runs `download-to-cache` as the entrypoint, and that binary isn't in the nmp-api image. Override the registry/tag if you're mirroring it. | default: 'nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10'
+        huggingface_model_puller: nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10
         # The name of the image pull secret for the modelPuller image | default: 'nvcrimagepullsecret'
         huggingface_model_puller_image_pull_secret: nvcrimagepullsecret
         # BusyBox image repository used by plugin init containers. Fully qualified (docker.io/library/...) so it resolves on container runtimes that enforce fully-qualified image names (short names like 'busybox' fail there). | default: 'docker.io/library/busybox'

services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py

@@ -105,7 +105,6 @@ def init(self) -> None:
             dynamic_client=self._dynamic_client,
             backend_config=self._backend_config,
             k8s_namespace=self._k8s_namespace,
-            huggingface_model_puller=self._huggingface_model_puller,
             status=self._status_projector,
             deleter=self._resource_deleter,
         )

services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py

@@ -103,6 +103,13 @@ class K8sNimOperatorConfig(BaseModel):
         default="nemo-models-files-token",
         description="Kubernetes secret name for Files service authentication (HF_TOKEN)",
     )
+    huggingface_model_puller: str = Field(
+        default="nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10",
+        description=(
+            "Image the NIM operator runs to fill the NIMCache from Files service. We need this specific "
+            "image because the operator runs `download-to-cache` as the entrypoint, which nmp-api doesn't support."
+        ),
+    )
     huggingface_model_puller_image_pull_secret: str = Field(
         default="nvcrimagepullsecret",
         description="The name of the image pull secret for the modelPuller image",

services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/nimservice_compiler.py

@@ -115,7 +115,6 @@ def compile_nimcache(
     model_namespace: str,
     model_name: str,
     pvc_size: str,
-    huggingface_model_puller: str,
     model_revision: Optional[str] = None,
 ) -> NIMCache:
     """Generate a NIMCache CR for models whose weights are served via the Files service HF-compatible API.
@@ -177,7 +176,10 @@ def compile_nimcache(
                     endpoint=files_full_url,
                     namespace=model_namespace,
                     authSecret=backend_config.files_auth_secret,
-                    modelPuller=huggingface_model_puller,
+                    # The operator runs `download-to-cache` in here, which nmp-api doesn't have,
+                    # so we use the huggingface-cli image that ships it.
+                    # value is set in the NIMService compiler, see K8sNimOperatorConfig.huggingface_model_puller.
+                    modelPuller=backend_config.huggingface_model_puller,
                     pullSecret=backend_config.huggingface_model_puller_image_pull_secret,
                     modelName=model_name,
                     revision=model_revision,

services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py

@@ -67,14 +67,12 @@ def __init__(
         dynamic_client: DynamicClient,
         backend_config: K8sNimOperatorConfig,
         k8s_namespace: str,
-        huggingface_model_puller: str,
         status: StatusProjector,
         deleter: ResourceDeleter,
     ) -> None:
         self._dynamic_client = dynamic_client
         self._backend_config = backend_config
         self._k8s_namespace = k8s_namespace
-        self._huggingface_model_puller = huggingface_model_puller
         self._status = status
         self._deleter = deleter
 
@@ -113,7 +111,7 @@ async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate:
                 resource_name=resource_name,
                 nimcache_name=nimcache_name,
                 model_entity=model_entity,
-                huggingface_model_puller=self._huggingface_model_puller,
+                huggingface_model_puller=self._backend_config.huggingface_model_puller,
             )
 
             nimservice_api = self._dynamic_client.resources.get(
@@ -189,7 +187,7 @@ async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate:
                 resource_name=resource_name,
                 nimcache_name=nimcache_name,
                 model_entity=model_entity,
-                huggingface_model_puller=self._huggingface_model_puller,
+                huggingface_model_puller=self._backend_config.huggingface_model_puller,
             )
 
             nimservice_api = self._dynamic_client.resources.get(
@@ -358,7 +356,6 @@ async def _ensure_nimcache(
                 model_namespace=model_namespace,
                 model_name=model_name,
                 pvc_size=pvc_size,
-                huggingface_model_puller=self._huggingface_model_puller,
                 model_revision=resolved.model_revision,
             )
             await self._create_nimcache(nimcache)

services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py

@@ -786,6 +786,9 @@ async def test_create_model_deployment_with_sft_model(k8s_backend, sample_deploy
     k8s_backend._backend_config.default_storage_class = "local-storage"
     k8s_backend._backend_config.default_pvc_size = "200Gi"
     k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token"
+    k8s_backend._backend_config.huggingface_model_puller = (
+        "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10"
+    )
     k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret"
     k8s_backend._backend_config.default_user_id = None
     k8s_backend._backend_config.default_group_id = None
@@ -889,6 +892,9 @@ async def test_create_model_deployment_with_files_service_model_triggers_nimcach
     k8s_backend._backend_config.default_storage_class = "local-storage"
     k8s_backend._backend_config.default_pvc_size = "200Gi"
     k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token"
+    k8s_backend._backend_config.huggingface_model_puller = (
+        "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10"
+    )
     k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret"
     k8s_backend._backend_config.default_user_id = None
     k8s_backend._backend_config.default_group_id = None
@@ -1054,6 +1060,9 @@ async def test_create_model_deployment_with_sft_model_and_revision(k8s_backend,
     k8s_backend._backend_config.default_user_id = 1000
     k8s_backend._backend_config.default_group_id = 1000
     k8s_backend._backend_config.files_auth_secret = "files-api-token"
+    k8s_backend._backend_config.huggingface_model_puller = (
+        "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10"
+    )
     k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcrimagepullsecret"
     k8s_backend._backend_config.default_resources = None
     k8s_backend._backend_config.default_tolerations = None
@@ -1162,6 +1171,9 @@ async def test_create_model_deployment_nimcache_uses_fileset_not_entity_name(
     k8s_backend._backend_config.default_storage_class = "local-storage"
     k8s_backend._backend_config.default_pvc_size = "200Gi"
     k8s_backend._backend_config.files_auth_secret = "nemo-models-files-token"
+    k8s_backend._backend_config.huggingface_model_puller = (
+        "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10"
+    )
     k8s_backend._backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret"
     k8s_backend._backend_config.default_user_id = None
     k8s_backend._backend_config.default_group_id = None

services/core/models/tests/unit/controllers/test_nimservice_compiler.py

@@ -1555,6 +1555,7 @@ def test_compile_nimcache_basic(backend_config):
     # Configure backend config
     backend_config.default_storage_class = "local-storage"
     backend_config.files_auth_secret = "nemo-models-files-token"
+    backend_config.huggingface_model_puller = "nvcr.io/nvidia/model-puller:latest"
     backend_config.huggingface_model_puller_image_pull_secret = "nvcr-secret"
     backend_config.default_user_id = 1000
     backend_config.default_group_id = 1000
@@ -1578,7 +1579,6 @@ def test_compile_nimcache_basic(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision="v1",
         )
 
@@ -1604,6 +1604,35 @@ def test_compile_nimcache_basic(backend_config):
         assert nimcache.spec.source.hf.pullSecret == "nvcr-secret"
 
 
+def test_compile_nimcache_default_puller_ships_download_to_cache(backend_config):
+    """Regression (nvbug 6480466): the NIMCache puller must default to an image that ships
+    `download-to-cache` (the entrypoint the k8s-nim-operator runs), NOT the nmp-api image.
+
+    Uses a bare K8sNimOperatorConfig() so the field default is what production Helm installs
+    resolve to when no override is set.
+    """
+    platform_config = PlatformConfig(  # type: ignore[abstract]
+        service_discovery={"files": "http://files-service:8000"},
+    )
+    with patch(
+        "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config",
+        return_value=platform_config,
+    ):
+        nimcache = compile_nimcache(
+            backend_config=backend_config,
+            k8s_namespace="default",
+            resource_name="test-deployment",
+            model_namespace="test-ns",
+            model_name="test-model",
+            pvc_size="200Gi",
+            model_revision=None,
+        )
+
+    assert nimcache.spec.source.hf is not None
+    assert nimcache.spec.source.hf.modelPuller == "nvcr.io/nvidia/nemo-microservices/nds-v2-huggingface-cli:25.10"
+    assert "nmp-api" not in nimcache.spec.source.hf.modelPuller
+
+
 def test_compile_nimcache_without_v2hf_suffix(backend_config):
     """Test NIMCache CR generation when files service URL does NOT include /v2/hf suffix."""
     # Configure backend config
@@ -1628,7 +1657,6 @@ def test_compile_nimcache_without_v2hf_suffix(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )
 
@@ -1661,7 +1689,6 @@ def test_compile_nimcache_with_v2hf_suffix(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )
 
@@ -1694,7 +1721,6 @@ def test_compile_nimcache_with_custom_auth_secret(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )
 
@@ -1735,7 +1761,6 @@ def test_compile_nimcache_default_resources_tolerations_node_selector(backend_co
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )
 
@@ -1784,7 +1809,6 @@ def test_compile_nimcache_no_defaults_when_unset(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )
 
@@ -1815,7 +1839,6 @@ def test_compile_nimcache_default_labels_and_annotations(backend_config):
             model_namespace="test-ns",
             model_name="test-model",
             pvc_size="200Gi",
-            huggingface_model_puller="nvcr.io/nvidia/model-puller:latest",
             model_revision=None,
         )