[None][perf] Optimize Video Hashing Speed

tensorrt_llm/inputs/media_io.py

@@ -36,6 +36,7 @@
 import requests
 import soundfile
 import torch
+from blake3 import blake3
 from packaging.version import Version
 from PIL import Image
 
@@ -409,6 +410,7 @@ def _load_video_by_cv2(
     device: str = "cpu",
     extract_audio: bool = False,
     cv2_backend: Optional[int] = None,
+    raw_bytes_hash: Optional[str] = None,
 ) -> VideoData:
     """Decode a video and return sampled frames as a list.
 
@@ -542,7 +544,12 @@ def _load_video_by_cv2(
             else:
                 raise
 
-    return VideoData(frames=loaded_frames, metadata=metadata, audio=audio)
+    return VideoData(
+        frames=loaded_frames,
+        metadata=metadata,
+        audio=audio,
+        raw_bytes_hash=raw_bytes_hash,
+    )
 
 
 def _normalize_file_uri(uri: str) -> str:
@@ -760,6 +767,7 @@ def load_bytes(self, data: bytes) -> VideoData:
         # is unlinked on context exit; the inode survives until cv2 closes
         # its own fd (Linux semantics), so the decode inside the `with`
         # block reads safely.
+        raw_bytes_hash = blake3(data).hexdigest()
         cv2_backend = _select_cv2_stream_buffered_backend()
         if cv2_backend is not None:
             return _load_video_by_cv2(
@@ -770,6 +778,7 @@ def load_bytes(self, data: bytes) -> VideoData:
                 self._device,
                 extract_audio=self._extract_audio,
                 cv2_backend=cv2_backend,
+                raw_bytes_hash=raw_bytes_hash,
             )
         with tempfile.NamedTemporaryFile(suffix=".mp4", dir=_VIDEO_TEMPFILE_DIR) as tmp:
             tmp.write(data)
@@ -781,20 +790,14 @@ def load_bytes(self, data: bytes) -> VideoData:
                 self._format,
                 self._device,
                 extract_audio=self._extract_audio,
+                raw_bytes_hash=raw_bytes_hash,
             )
 
     def load_base64(self, media_type: str, data: str) -> VideoData:
         return self.load_bytes(base64.b64decode(data))
 
     def load_file(self, url: str) -> VideoData:
-        return _load_video_by_cv2(
-            _normalize_file_uri(url),
-            self._num_frames,
-            self._fps,
-            self._format,
-            self._device,
-            extract_audio=self._extract_audio,
-        )
+        return self.load_bytes(Path(_normalize_file_uri(url)).read_bytes())
 
 
 MEDIA_IO_REGISTRY: Mapping[MediaModality, Type[BaseMediaIO]] = MappingProxyType(

tensorrt_llm/inputs/multimodal_data.py

@@ -154,11 +154,19 @@ class VideoData(BaseModalityData):
             - duration: Duration of the video in seconds
             - frames_indices: List of indices of the sampled frames
         audio: Structured audio payload from the video, when extracted.
+        raw_bytes_hash: BLAKE3 hex digest of the source video bytes when
+            available. Populated by media loaders that hold the source
+            (e.g. `VideoMediaIO`); `None` when the `VideoData` is
+            constructed from a frame list directly. When set, it acts as
+            the source anchor in `update_hash` — combined with the
+            sampling metadata, it uniquely identifies the decoded frame
+            content without walking pixels.
     """
 
     frames: list[Image.Image] | list[torch.Tensor]
     metadata: dict[str, Any]
     audio: AudioData | None = None
+    raw_bytes_hash: str | None = None
 
     def __post_init__(self) -> None:
         if not self.frames:
@@ -169,10 +177,21 @@ def __post_init__(self) -> None:
     def update_hash(self, hasher: ContentHasher) -> None:
         hasher.update(b"<video>")
         # Sampling metadata is part of the model-visible cache identity.
+        # `frames_indices` is a deterministic function of source bytes plus
+        # media IO `num_frames`/`fps` kwargs, so per-request kwarg
+        # overrides land here implicitly.
         meta = {k: self.metadata[k] for k in _VIDEO_HASH_METADATA_FIELDS if k in self.metadata}
         hasher.update(serialize_item(meta))
-        for frame in self.frames:
-            hasher.update(b"<frame>")
-            hasher.update(serialize_item(frame))
+        if self.raw_bytes_hash is not None:
+            # Source anchor: metadata + source digest is equivalent to
+            # hashing decoded frames, since cv2 decoding is deterministic
+            # given source bytes and sampling is captured in `meta`.
+            hasher.update(b"<raw_bytes>")
+            hasher.update(self.raw_bytes_hash.encode("utf-8"))
+        else:
+            # No source anchor available (frame list constructed directly).
+            for frame in self.frames:
+                hasher.update(b"<frame>")
+                hasher.update(serialize_item(frame))
         if self.audio is not None:
             self.audio.update_hash(hasher)

tests/unittest/inputs/test_video_data_hashing.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+"""Regression tests for `VideoData.update_hash`.
+
+Cover both branches:
+  * `raw_bytes_hash` set  -> source-anchor path (must not walk frames)
+  * `raw_bytes_hash` None -> frame-walk fallback (must include every frame)
+
+Plus sampling-metadata and audio contributions to cache identity.
+"""
+
+import torch
+from blake3 import blake3
+
+from tensorrt_llm.inputs import multimodal_data as md
+from tensorrt_llm.inputs.multimodal_data import AudioData, VideoData
+
+
+def _hex(video: VideoData) -> str:
+    h = blake3()
+    video.update_hash(h)
+    return h.hexdigest()
+
+
+def _make_video(raw_bytes_hash=None, audio=None, frames=None, meta=None):
+    frames = frames or [torch.zeros((3, 8, 8), dtype=torch.float32) for _ in range(3)]
+    meta = meta or {
+        "frames_indices": [0, 1, 2],
+        "fps": 30.0,
+        "duration": 1.0,
+        "total_num_frames": 3,
+    }
+    return VideoData(frames=frames, metadata=meta, audio=audio, raw_bytes_hash=raw_bytes_hash)
+
+
+def _tensor_serialize_calls(spy) -> int:
+    return sum(1 for call in spy.call_args_list if isinstance(call.args[0], torch.Tensor))
+
+
+def test_source_anchor_skips_frame_walk(mocker):
+    """When `raw_bytes_hash` is set the frames must not be serialized."""
+    spy = mocker.spy(md, "serialize_item")
+    _hex(_make_video(raw_bytes_hash="a" * 64))
+    assert _tensor_serialize_calls(spy) == 0
+
+
+def test_frame_walk_fallback_serializes_every_frame(mocker):
+    """When `raw_bytes_hash` is None the fallback hashes each frame."""
+    spy = mocker.spy(md, "serialize_item")
+    _hex(_make_video(raw_bytes_hash=None))
+    assert _tensor_serialize_calls(spy) == 3
+
+
+def test_source_anchor_hash_stable_across_frame_content():
+    """Frame contents must not affect the digest when source anchor is set.
+
+    Given identical `raw_bytes_hash` + metadata, the source anchor is
+    authoritative and pixel data must not participate in the hash.
+    """
+    v1 = _make_video(
+        raw_bytes_hash="deadbeef" * 8, frames=[torch.zeros((3, 4, 4)) for _ in range(2)]
+    )
+    v2 = _make_video(
+        raw_bytes_hash="deadbeef" * 8, frames=[torch.ones((3, 4, 4)) for _ in range(2)]
+    )
+    assert _hex(v1) == _hex(v2)
+
+
+def test_metadata_change_alters_hash():
+    """`frames_indices` / `fps` are part of cache identity."""
+    base_meta = {"frames_indices": [0, 1, 2], "fps": 30.0}
+    v1 = _make_video(raw_bytes_hash="a" * 64, meta=dict(base_meta))
+    v2 = _make_video(raw_bytes_hash="a" * 64, meta={**base_meta, "frames_indices": [0, 2, 4]})
+    v3 = _make_video(raw_bytes_hash="a" * 64, meta={**base_meta, "fps": 15.0})
+    assert _hex(v1) != _hex(v2)
+    assert _hex(v1) != _hex(v3)
+
+
+def test_audio_contributes_to_hash():
+    """Adding an audio track changes the digest."""
+    audio = AudioData(samples=torch.zeros(1024), sample_rate=16000)
+    without_audio = _make_video(raw_bytes_hash="a" * 64)
+    with_audio = _make_video(raw_bytes_hash="a" * 64, audio=audio)
+    assert _hex(without_audio) != _hex(with_audio)
+
+
+def test_different_source_hashes_differ():
+    """Different `raw_bytes_hash` inputs must yield different digests."""
+    v1 = _make_video(raw_bytes_hash="a" * 64)
+    v2 = _make_video(raw_bytes_hash="b" * 64)
+    assert _hex(v1) != _hex(v2)