Skip to content

Commit cb1c1f3

Browse files
committed
Add synchronized early stopping for ZeRO-3 rollout
ZeRO-3 gathers parameters during every decode forward, so all data-parallel ranks must execute the same number of decode iterations. Today a single rank reaching EOS forces the remaining ranks to keep decoding to `max_new_tokens`, and letting each rank stop locally would desynchronize the parameter-gather collectives. Add a stopping criterion that rendezvouses the local work flag over `optimizer.dp_process_group`, which is the group used by ZeRO-3 parameter gathers rather than the default world group. Generation now stops at the latest rank-local EOS and each sequence keeps its own response length. Non-ZeRO-3 generation is unchanged and keeps the native EOS path. The criterion runs inside the Transformers decode loop, so also opt out of the `synced_gpus` default that Transformers derives from the default world size when it can observe ZeRO-3. That default issues an ungrouped world all-reduce, which a single data-parallel subgroup entering rollout would block on forever, and which unnecessarily couples concurrently active subgroups. Signed-off-by: LiRunGuo <li19107254665@gmail.com>
1 parent 3b80932 commit cb1c1f3

2 files changed

Lines changed: 309 additions & 3 deletions

File tree

‎deepspeed/runtime/rollout/hybrid_engine_rollout.py‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
import torch
2424

25+
import deepspeed.comm as dist
2526
from deepspeed.accelerator import get_accelerator
2627
from deepspeed.runtime.rollout.base import RolloutBatch, RolloutEngine, RolloutRequest, SamplingConfig
2728
from deepspeed.runtime.rollout.continuous_batching import ContinuousBatchRequest, ContinuousBatchScheduler
@@ -75,6 +76,39 @@ def _elapsed_time(self, start, end):
7576
return start.elapsed_time(end)
7677

7778

79+
class GlobalWorkRemaining:
80+
"""Rendezvous a local work flag over a distributed process group."""
81+
82+
def __init__(self, process_group=None):
83+
self.process_group = process_group
84+
85+
def __call__(self, local_work_remaining):
86+
work_remaining = torch.as_tensor(local_work_remaining, dtype=torch.int32).clone()
87+
if dist.is_initialized() and dist.get_world_size(group=self.process_group) > 1:
88+
dist.all_reduce(work_remaining, op=dist.ReduceOp.MAX, group=self.process_group)
89+
return bool(work_remaining.item())
90+
91+
92+
class _GlobalEosStoppingCriteria:
93+
"""Stop all local sequences only after every rank has observed EOS."""
94+
95+
def __init__(self, response_start, eos_token_id, work_remaining):
96+
self.response_start = response_start
97+
self.eos_token_ids = torch.as_tensor(eos_token_id).flatten()
98+
self.work_remaining = work_remaining
99+
100+
def __call__(self, input_ids, scores, **kwargs): # noqa: ARG002
101+
response_ids = input_ids[:, self.response_start:]
102+
eos_token_ids = self.eos_token_ids.to(device=input_ids.device, dtype=input_ids.dtype)
103+
sequence_has_eos = (response_ids.unsqueeze(-1) == eos_token_ids).any(dim=-1).any(dim=-1)
104+
local_work_remaining = (~sequence_has_eos).any().to(dtype=torch.int32)
105+
global_work_remaining = self.work_remaining(local_work_remaining)
106+
# This is a rank-global decision, so a scalar preserves the same
107+
# behavior for every local sequence and remains compatible with older
108+
# Transformers releases that apply Python ``any`` to criterion results.
109+
return not global_work_remaining
110+
111+
78112
@dataclass
79113
class HybridEngineRolloutConfig:
80114
"""Configuration for HybridEngineRollout.
@@ -175,12 +209,28 @@ def generate(self, request: RolloutRequest, sampling: SamplingConfig) -> Rollout
175209
else:
176210
temperature = max(sampling.temperature, 1e-8)
177211
do_sample = not is_greedy
212+
zero3_sync = self._zero3_work_remaining()
213+
stopping_criteria = None
214+
if zero3_sync is not None and self.tokenizer.eos_token_id is not None:
215+
stopping_criteria = [
216+
_GlobalEosStoppingCriteria(prompt_len, self.tokenizer.eos_token_id, zero3_sync)
217+
]
178218
generate_kwargs = {
179219
"attention_mask": prompt_attn,
180220
"max_new_tokens": max_new_tokens,
181-
# ZeRO-3 gathers parameters during each decode forward, so every
182-
# data-parallel rank must execute the same number of iterations.
183-
"eos_token_id": None,
221+
# ZeRO-3 gathers parameters during each decode forward. Defer
222+
# EOS handling to the group-aware criterion so every parameter
223+
# partition rank executes the same number of iterations.
224+
"eos_token_id": None if zero3_sync is not None else self.tokenizer.eos_token_id,
225+
"stopping_criteria": stopping_criteria,
226+
# Transformers enables synced_gpus from the default world
227+
# size when it can observe ZeRO-3, and the decode loop then
228+
# issues an ungrouped world all-reduce. A data-parallel
229+
# subgroup that enters rollout alone would block on that
230+
# collective. The criterion above already synchronizes the
231+
# stop decision over the parameter-gather group, so opt out
232+
# of the native synchronization for the ZeRO-3 path only.
233+
"synced_gpus": False if zero3_sync is not None else None,
184234
"do_sample": do_sample,
185235
"temperature": temperature if do_sample else 1.0,
186236
"top_p": sampling.top_p if do_sample else 1.0,
@@ -897,6 +947,20 @@ def _build_continuous_batch(self, request, responses):
897947
device=input_ids.device),
898948
)
899949

950+
def _zero3_work_remaining(self):
951+
"""Return a rendezvous over the group used for ZeRO-3 parameter gathers."""
952+
is_zero3 = getattr(self.engine, "zero_optimization_partition_weights", None)
953+
if not callable(is_zero3) or not is_zero3():
954+
return None
955+
956+
process_group = None
957+
if dist.is_initialized():
958+
optimizer = getattr(self.engine, "optimizer", None)
959+
process_group = getattr(optimizer, "dp_process_group", None)
960+
if process_group is None:
961+
raise RuntimeError("ZeRO-3 synchronized rollout requires optimizer.dp_process_group")
962+
return GlobalWorkRemaining(process_group)
963+
900964
def get_last_profile(self):
901965
"""Return the most recent profiling snapshot for this rollout instance."""
902966
return self._last_profile

‎tests/unit/runtime/rollout/test_hybrid_engine_rollout.py‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,26 @@
88
"""
99

1010
from types import SimpleNamespace
11+
from types import MethodType
1112
from unittest.mock import MagicMock, patch
1213

1314
import pytest
1415
import torch
1516
from transformers import PretrainedConfig
1617

18+
import deepspeed
19+
import deepspeed.comm as dist
1720
from deepspeed.ops.transformer.inference.op_binding.workspace import WorkspaceOp
1821
from deepspeed.runtime.hybrid_engine import DeepSpeedHybridEngine
1922
from deepspeed.runtime.rollout.base import RolloutRequest, SamplingConfig
2023
from deepspeed.runtime.rollout.hybrid_engine_rollout import (
24+
GlobalWorkRemaining,
2125
HybridEngineRollout,
2226
HybridEngineRolloutConfig,
27+
_GlobalEosStoppingCriteria,
2328
)
2429
from deepspeed.utils.static_cache import DeepSpeedStaticCache
30+
from unit.common import DistributedTest
2531

2632

2733
def _make_engine():
@@ -550,6 +556,242 @@ def test_continuous_generation_treats_none_repetition_penalty_as_one():
550556

551557
assert next_tokens.tolist() == [[0]]
552558

559+
def test_global_work_remaining_uses_requested_group(monkeypatch):
560+
process_group = object()
561+
all_reduce = MagicMock(side_effect=lambda flag, **kwargs: flag.fill_(1))
562+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True)
563+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.get_world_size", lambda group: 2)
564+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.all_reduce", all_reduce)
565+
566+
assert GlobalWorkRemaining(process_group)(torch.tensor(0)) is True
567+
assert all_reduce.call_args.kwargs["group"] is process_group
568+
569+
570+
def test_global_eos_stops_only_when_no_global_work_remains():
571+
rendezvous = MagicMock(side_effect=[True, False])
572+
criterion = _GlobalEosStoppingCriteria(response_start=2, eos_token_id=2, work_remaining=rendezvous)
573+
574+
keep_going = criterion(torch.tensor([[10, 11, 2], [10, 11, 5]]), scores=None)
575+
stop = criterion(torch.tensor([[10, 11, 2], [10, 11, 2]]), scores=None)
576+
577+
assert keep_going is False
578+
assert stop is True
579+
# Transformers 4.32 applies Python any() to criterion results.
580+
assert any([keep_going]) is False
581+
assert any([stop]) is True
582+
assert [call.args[0].item() for call in rendezvous.call_args_list] == [1, 0]
583+
584+
585+
def test_zero3_rendezvous_uses_optimizer_dp_group(monkeypatch):
586+
engine = _make_engine()
587+
process_group = object()
588+
engine.zero_optimization_partition_weights.return_value = True
589+
engine.optimizer.dp_process_group = process_group
590+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True)
591+
592+
rendezvous = HybridEngineRollout(engine, _make_tokenizer())._zero3_work_remaining()
593+
594+
assert rendezvous.process_group is process_group
595+
596+
597+
def test_non_zero3_generation_uses_native_eos():
598+
engine = _make_engine()
599+
engine.zero_optimization_partition_weights.return_value = False
600+
engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]])
601+
rollout = HybridEngineRollout(engine, _make_tokenizer())
602+
603+
rollout.generate(_make_request(), _make_sampling())
604+
605+
assert engine.module.generate.call_args.kwargs["eos_token_id"] == 2
606+
assert engine.module.generate.call_args.kwargs["stopping_criteria"] is None
607+
# Non-ZeRO-3 generation must keep the Transformers default.
608+
assert engine.module.generate.call_args.kwargs["synced_gpus"] is None
609+
610+
611+
def test_zero3_generation_disables_native_synced_gpus(monkeypatch):
612+
engine = _make_engine()
613+
engine.zero_optimization_partition_weights.return_value = True
614+
engine.optimizer.dp_process_group = object()
615+
engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]])
616+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True)
617+
618+
HybridEngineRollout(engine, _make_tokenizer()).generate(_make_request(), _make_sampling())
619+
620+
kwargs = engine.module.generate.call_args.kwargs
621+
# Transformers would otherwise derive synced_gpus from the default world
622+
# size and issue an ungrouped world all-reduce inside the decode loop.
623+
assert kwargs["synced_gpus"] is False
624+
assert kwargs["eos_token_id"] is None
625+
assert kwargs["stopping_criteria"] is not None
626+
627+
628+
class TestGlobalWorkRemainingSubgroups(DistributedTest):
629+
world_size = 4
630+
backend = "gloo"
631+
requires_cuda_env = False
632+
init_distributed = False
633+
634+
def test(self):
635+
# This is a CPU/Gloo topology test. Avoid binding the Gloo default
636+
# group to the host's visible accelerator when running on a GPU node.
637+
with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator:
638+
mock_accelerator.return_value.device_name.return_value = "cpu"
639+
deepspeed.init_distributed(dist_backend="gloo")
640+
first_group = dist.new_group(ranks=[0, 1])
641+
second_group = dist.new_group(ranks=[2, 3])
642+
rank = dist.get_rank()
643+
process_group = first_group if rank < 2 else second_group
644+
645+
# Only rank 0 has local work. Its DP sibling must continue, while the
646+
# independent DP subgroup must be able to stop.
647+
local_work_remaining = torch.tensor(rank == 0, dtype=torch.int32)
648+
global_work_remaining = GlobalWorkRemaining(process_group)(local_work_remaining)
649+
650+
assert global_work_remaining is (rank < 2)
651+
652+
653+
class TestZero3SubgroupGeneration(DistributedTest):
654+
"""ZeRO-3 rollout inside one data-parallel subgroup must not touch WORLD.
655+
656+
When Transformers can observe ZeRO-3 it derives ``synced_gpus`` from the
657+
default world size, and the decode loop then issues an ungrouped world
658+
all-reduce. A subgroup that enters rollout while the remaining WORLD ranks
659+
stay out would block forever on that collective. Here ranks 0-1 generate
660+
inside one subgroup and ranks 2-3 never enter rollout at all.
661+
"""
662+
world_size = 4
663+
backend = "gloo"
664+
requires_cuda_env = False
665+
init_distributed = False
666+
667+
def test(self):
668+
from transformers import Qwen2Config, Qwen2ForCausalLM
669+
from transformers.integrations.deepspeed import HfDeepSpeedConfig, is_deepspeed_zero3_enabled
670+
671+
# This is a CPU/Gloo topology test. Avoid binding the Gloo default
672+
# group to the host's visible accelerator when running on a GPU node.
673+
with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator:
674+
mock_accelerator.return_value.device_name.return_value = "cpu"
675+
deepspeed.init_distributed(dist_backend="gloo")
676+
first_group = dist.new_group(ranks=[0, 1])
677+
dist.new_group(ranks=[2, 3])
678+
rank = dist.get_rank()
679+
dist.barrier()
680+
681+
# Make ZeRO-3 visible to Transformers so the native default for
682+
# synced_gpus would be True. Keep the config alive for the process.
683+
deepspeed_config = HfDeepSpeedConfig({"zero_optimization": {"stage": 3}})
684+
assert is_deepspeed_zero3_enabled()
685+
686+
if rank >= 2:
687+
# Non-participating WORLD ranks must not be required by rollout.
688+
del deepspeed_config
689+
return
690+
691+
model = Qwen2ForCausalLM(
692+
Qwen2Config(
693+
vocab_size=32,
694+
max_position_embeddings=16,
695+
hidden_size=32,
696+
intermediate_size=64,
697+
num_hidden_layers=2,
698+
num_attention_heads=4,
699+
num_key_value_heads=4,
700+
bos_token_id=1,
701+
eos_token_id=2,
702+
pad_token_id=0,
703+
))
704+
engine = SimpleNamespace(
705+
module=model,
706+
zero_optimization_partition_weights=lambda: True,
707+
optimizer=SimpleNamespace(dp_process_group=first_group),
708+
)
709+
rollout = HybridEngineRollout(engine, SimpleNamespace(pad_token_id=0, eos_token_id=2))
710+
request = RolloutRequest(
711+
prompt_ids=torch.tensor([[5, 6]]),
712+
prompt_attention_mask=torch.ones(1, 2, dtype=torch.long),
713+
)
714+
715+
result = rollout.generate(request, SamplingConfig(max_new_tokens=3, temperature=0))
716+
717+
assert result.input_ids.shape == (1, 5)
718+
assert result.attention_mask.shape == (1, 5)
719+
assert result.input_ids[0, :2].tolist() == [5, 6]
720+
del deepspeed_config
721+
722+
723+
class TestZero3SynchronizedEarlyStop(DistributedTest):
724+
world_size = 2
725+
726+
def test(self):
727+
"""Different local EOS steps stop at the latest EOS, not max length."""
728+
from transformers import Qwen2Config, Qwen2ForCausalLM
729+
730+
rank = dist.get_rank()
731+
device = torch.device(f"cuda:{rank}")
732+
model = Qwen2ForCausalLM(
733+
Qwen2Config(
734+
vocab_size=32,
735+
max_position_embeddings=16,
736+
hidden_size=32,
737+
intermediate_size=64,
738+
num_hidden_layers=2,
739+
num_attention_heads=4,
740+
num_key_value_heads=4,
741+
bos_token_id=1,
742+
eos_token_id=2,
743+
pad_token_id=0,
744+
))
745+
ds_config = {
746+
"train_micro_batch_size_per_gpu": 1,
747+
"optimizer": {
748+
"type": "AdamW",
749+
"params": {
750+
"lr": 1e-5,
751+
"torch_adam": True
752+
},
753+
},
754+
"zero_optimization": {
755+
"stage": 3
756+
},
757+
"hybrid_engine": {
758+
"enabled": True,
759+
"inference_tp_size": 1,
760+
"pin_parameters": True,
761+
},
762+
}
763+
engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=ds_config)
764+
engine.eval()
765+
766+
target_eos_step = 2 if rank == 0 else 5
767+
forward_count = 0
768+
original_forward = engine.module.forward
769+
770+
def scripted_forward(_module, *args, **kwargs):
771+
nonlocal forward_count
772+
output = original_forward(*args, **kwargs)
773+
forward_count += 1
774+
scripted_token = 2 if forward_count >= target_eos_step else 3
775+
logits = torch.full_like(output.logits, torch.finfo(output.logits.dtype).min)
776+
logits[:, -1, scripted_token] = 0
777+
output.logits = logits
778+
return output
779+
780+
engine.module.forward = MethodType(scripted_forward, engine.module)
781+
tokenizer = SimpleNamespace(pad_token_id=0, eos_token_id=2)
782+
rollout = HybridEngineRollout(engine, tokenizer)
783+
request = RolloutRequest(
784+
prompt_ids=torch.tensor([[5, 6]], device=device),
785+
prompt_attention_mask=torch.ones(1, 2, dtype=torch.long, device=device),
786+
)
787+
result = rollout.generate(request, SamplingConfig(max_new_tokens=8, temperature=0))
788+
789+
counts = [torch.zeros(1, dtype=torch.int32, device=device) for _ in range(2)]
790+
dist.all_gather(counts, torch.tensor([forward_count], dtype=torch.int32, device=device))
791+
assert [count.item() for count in counts] == [5, 5]
792+
assert int(result.attention_mask[:, 2:].sum()) == target_eos_step
793+
assert result.input_ids[0, 1 + target_eos_step].item() == 2
794+
553795

554796
@patch("deepspeed.runtime.rollout.hybrid_engine_rollout.time.perf_counter")
555797
@patch("deepspeed.runtime.rollout.hybrid_engine_rollout.get_accelerator")

0 commit comments

Comments
 (0)