Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 26 additions & 14 deletions swift/dev/template/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,22 @@ def encode_prompt(
raise ValueError("encode_prompt requires one unsplit prompt feature.")
prompt_inputs = sample if isinstance(sample, TemplateInputs) else TemplateInputs.from_dict(sample)
engine_ids = list(engine_feature["input_ids"])
media = MediaBundle.from_sources(sample, feature)
# Serving engines retain resized images here. Transformers templates may
# use the same key for tensors consumed by the model.
engine_images = engine_feature.get("images") if backend in ("vllm", "sglang", "lmdeploy") else None
media = MediaBundle.from_sources({"images": engine_images}, sample, feature)
processor_kwargs = dict(engine_feature.get("mm_processor_kwargs") or {})
if backend == "vllm" and media.videos:
# The rollout uploads the original videos, which vLLM must process.
if processor_kwargs.get("do_resize") is False:
processor_kwargs.pop("do_resize")
if engine_images:
image_kwargs = dict(processor_kwargs.get("images_kwargs") or {})
image_kwargs.setdefault("do_resize", False)
processor_kwargs["images_kwargs"] = image_kwargs
if processor_kwargs.get("do_sample_frames") is False:
processor_kwargs.pop("do_sample_frames")
processor_kwargs.pop("video_metadata", None)
if self.is_multimodal:
self._check_engine_placeholders(engine_ids, media)
state = RolloutState(
Expand All @@ -857,6 +872,7 @@ def encode_prompt(
tools=tuple(deepcopy(prompt_inputs.chosen.tools or ())),
reasoning_started=self._reasoning_started(feature["input_ids"]),
media=media,
mm_processor_kwargs=processor_kwargs,
)
return PromptEncoding(state)

Expand Down Expand Up @@ -937,6 +953,7 @@ def append_completion(
tools=state.tools,
reasoning_started=bool(response.reasoning_started),
media=state.media,
mm_processor_kwargs=state.mm_processor_kwargs,
)
return CompletionEncoding(next_state, response)

Expand Down Expand Up @@ -2643,7 +2660,7 @@ def concat_tensor(batch: List[Dict[str, Any]], attr_name: str, dim: int) -> Opti
res = []
for b in batch:
if b.get(attr_name) is not None:
res.append(b.pop(attr_name))
res.append(torch.as_tensor(b.pop(attr_name)))
return torch.concat(res, dim=dim) if res else None

def _rlhf_data_collator(
Expand Down Expand Up @@ -3015,19 +3032,14 @@ def create_mm_token_type_ids(self, input_ids: List[int], mm_mask: Optional[List[
return mm_token_type_ids

def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
# multimodal
# Encoded media from HTTP arrives as nested lists.
res = {}
pixel_values = [b["pixel_values"] for b in batch if b.get("pixel_values") is not None]
if len(pixel_values) > 0:
res["pixel_values"] = torch.concat(pixel_values)

image_sizes = [b["image_sizes"] for b in batch if b.get("image_sizes") is not None]
if len(image_sizes) > 0:
res["image_sizes"] = torch.concat(image_sizes)

pixel_values_videos = [b["pixel_values_videos"] for b in batch if b.get("pixel_values_videos") is not None]
if len(pixel_values_videos) > 0:
res["pixel_values_videos"] = torch.concat(pixel_values_videos)
for key in ("pixel_values", "image_sizes", "pixel_values_videos"):
if key == "image_sizes" and "pixel_values" not in res:
continue
values = [torch.as_tensor(row[key]) for row in batch if row.get(key) is not None]
if values:
res[key] = torch.concat(values)

for media_type in ["image", "video"]:
grid_thw = self.concat_tensor(batch, f"{media_type}_grid_thw", 0)
Expand Down
4 changes: 3 additions & 1 deletion swift/dev/template_contract/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,13 @@ class RolloutState:
reasoning_started: bool
media: MediaBundle
render_cache: Optional[Any] = None
mm_processor_kwargs: Dict[str, Any] = dataclasses.field(default_factory=dict)

def __reduce__(self):
return (
type(self),
(self.feature, self.engine_input_ids, self.messages, self.tools, self.reasoning_started, self.media, None),
(self.feature, self.engine_input_ids, self.messages, self.tools, self.reasoning_started, self.media, None,
self.mm_processor_kwargs),
)


Expand Down
53 changes: 53 additions & 0 deletions swift/dev/tests/component/processor/test_template_intent.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,59 @@ def test_encode_prompt_uses_native_engine_intent():
assert list(prompt.state.engine_input_ids) == expected_engine["input_ids"]


@pytest.mark.parametrize("backend", ["transformers", "vllm", "sglang", "lmdeploy"])
def test_encode_prompt_selects_images_for_backend(backend, monkeypatch):
import torch

template = _template()
sample = {"messages": [{"role": "user", "content": "question"}], "images": ["original.png"]}
tensor = torch.ones(1, 3, 2, 2)
engine_image = tensor if backend == "transformers" else "resized.png"

def encode(_sample, *, intent, engine):
assert engine == backend
image = tensor if intent.expand_media else engine_image
feature = {"input_ids": [1, 2], "images": [image]}
if not intent.expand_media and backend == "vllm":
feature["mm_processor_kwargs"] = {"do_resize": False}
return feature

monkeypatch.setattr(template, "encode", encode)
state = template.encode_prompt(sample, backend=backend).state

assert state.media.images == (("original.png" if backend == "transformers" else "resized.png"),)
assert state.feature["images"][0] is tensor
assert state.mm_processor_kwargs == ({"do_resize": False} if backend == "vllm" else {})
assert "mm_processor_kwargs" not in state.feature


@pytest.mark.parametrize("with_image", [False, True])
def test_vllm_encode_prompt_keeps_resize_override_only_for_processed_images(with_image, monkeypatch):
template = _template()
sample = {"messages": [{"role": "user", "content": "question"}], "videos": ["original.mp4"]}
engine_kwargs = {"do_resize": False, "do_sample_frames": False, "video_metadata": [], "fps": 2}

def encode(_sample, *, intent, engine):
if intent.expand_media:
return {"input_ids": [1, 2]}
return {
"input_ids": [1, 2],
"images": ["resized.png"] if with_image else [],
"videos": ["sampled_video"],
"mm_processor_kwargs": engine_kwargs,
}

monkeypatch.setattr(template, "encode", encode)
state = template.encode_prompt(sample, backend="vllm").state
assert state.media.videos == ("original.mp4",)
assert state.media.images == (("resized.png",) if with_image else ())
expected = {"fps": 2}
if with_image:
expected["images_kwargs"] = {"do_resize": False}
assert state.mm_processor_kwargs == expected
assert engine_kwargs["do_resize"] is False and engine_kwargs["do_sample_frames"] is False


@pytest.mark.parametrize("intent", [TRAIN, SCORE])
@pytest.mark.parametrize("return_length", [False, True])
def test_split_finalizes_every_real_encoder_chunk_once(intent, return_length):
Expand Down
Loading