Skip to content

Commit d8a5b17

Browse files
committed
Add synchronized early stopping for ZeRO-3 rollout
Signed-off-by: LiRunGuo <li19107254665@gmail.com>
1 parent 56de570 commit d8a5b17

2 files changed

Lines changed: 213 additions & 3 deletions

File tree

‎deepspeed/runtime/rollout/hybrid_engine_rollout.py‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import torch
1919

20+
import deepspeed.comm as dist
2021
from deepspeed.accelerator import get_accelerator
2122
from deepspeed.runtime.rollout.base import RolloutBatch, RolloutEngine, RolloutRequest, SamplingConfig
2223

@@ -69,6 +70,39 @@ def _elapsed_time(self, start, end):
6970
return start.elapsed_time(end)
7071

7172

73+
class GlobalWorkRemaining:
74+
"""Rendezvous a local work flag over a distributed process group."""
75+
76+
def __init__(self, process_group=None):
77+
self.process_group = process_group
78+
79+
def __call__(self, local_work_remaining):
80+
work_remaining = torch.as_tensor(local_work_remaining, dtype=torch.int32).clone()
81+
if dist.is_initialized() and dist.get_world_size(group=self.process_group) > 1:
82+
dist.all_reduce(work_remaining, op=dist.ReduceOp.MAX, group=self.process_group)
83+
return bool(work_remaining.item())
84+
85+
86+
class _GlobalEosStoppingCriteria:
87+
"""Stop all local sequences only after every rank has observed EOS."""
88+
89+
def __init__(self, response_start, eos_token_id, work_remaining):
90+
self.response_start = response_start
91+
self.eos_token_ids = torch.as_tensor(eos_token_id).flatten()
92+
self.work_remaining = work_remaining
93+
94+
def __call__(self, input_ids, scores, **kwargs): # noqa: ARG002
95+
response_ids = input_ids[:, self.response_start:]
96+
eos_token_ids = self.eos_token_ids.to(device=input_ids.device, dtype=input_ids.dtype)
97+
sequence_has_eos = (response_ids.unsqueeze(-1) == eos_token_ids).any(dim=-1).any(dim=-1)
98+
local_work_remaining = (~sequence_has_eos).any().to(dtype=torch.int32)
99+
global_work_remaining = self.work_remaining(local_work_remaining)
100+
# This is a rank-global decision, so a scalar preserves the same
101+
# behavior for every local sequence and remains compatible with older
102+
# Transformers releases that apply Python ``any`` to criterion results.
103+
return not global_work_remaining
104+
105+
72106
@dataclass
73107
class HybridEngineRolloutConfig:
74108
"""Configuration for HybridEngineRollout."""
@@ -148,13 +182,21 @@ def generate(self, request: RolloutRequest, sampling: SamplingConfig) -> Rollout
148182
else:
149183
temperature = max(sampling.temperature, 1e-8)
150184
do_sample = not is_greedy
185+
zero3_sync = self._zero3_work_remaining()
186+
stopping_criteria = None
187+
if zero3_sync is not None and self.tokenizer.eos_token_id is not None:
188+
stopping_criteria = [
189+
_GlobalEosStoppingCriteria(prompt_len, self.tokenizer.eos_token_id, zero3_sync)
190+
]
151191
output_ids = module.generate(
152192
prompt_ids,
153193
attention_mask=prompt_attn,
154194
max_new_tokens=max_new_tokens,
155-
# ZeRO-3 gathers parameters during each decode forward, so every
156-
# data-parallel rank must execute the same number of iterations.
157-
eos_token_id=None,
195+
# ZeRO-3 gathers parameters during each decode forward. Defer
196+
# EOS handling to the group-aware criterion so every parameter
197+
# partition rank executes the same number of iterations.
198+
eos_token_id=None if zero3_sync is not None else self.tokenizer.eos_token_id,
199+
stopping_criteria=stopping_criteria,
158200
do_sample=do_sample,
159201
temperature=temperature if do_sample else 1.0,
160202
top_p=sampling.top_p if do_sample else 1.0,
@@ -236,6 +278,20 @@ def generate(self, request: RolloutRequest, sampling: SamplingConfig) -> Rollout
236278

237279
return rollout_batch
238280

281+
def _zero3_work_remaining(self):
282+
"""Return a rendezvous over the group used for ZeRO-3 parameter gathers."""
283+
is_zero3 = getattr(self.engine, "zero_optimization_partition_weights", None)
284+
if not callable(is_zero3) or not is_zero3():
285+
return None
286+
287+
process_group = None
288+
if dist.is_initialized():
289+
optimizer = getattr(self.engine, "optimizer", None)
290+
process_group = getattr(optimizer, "dp_process_group", None)
291+
if process_group is None:
292+
raise RuntimeError("ZeRO-3 synchronized rollout requires optimizer.dp_process_group")
293+
return GlobalWorkRemaining(process_group)
294+
239295
def get_last_profile(self):
240296
"""Return the most recent profiling snapshot for this rollout instance."""
241297
return self._last_profile

‎tests/unit/runtime/rollout/test_hybrid_engine_rollout.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,24 @@
99
"""
1010

1111
from types import SimpleNamespace
12+
from types import MethodType
1213
from unittest.mock import MagicMock, patch
1314

1415
import pytest
1516
import torch
1617

18+
import deepspeed
19+
import deepspeed.comm as dist
1720
from deepspeed.ops.transformer.inference.op_binding.workspace import WorkspaceOp
1821
from deepspeed.runtime.hybrid_engine import DeepSpeedHybridEngine
1922
from deepspeed.runtime.rollout.base import RolloutRequest, SamplingConfig
2023
from deepspeed.runtime.rollout.hybrid_engine_rollout import (
24+
GlobalWorkRemaining,
2125
HybridEngineRollout,
2226
HybridEngineRolloutConfig,
27+
_GlobalEosStoppingCriteria,
2328
)
29+
from unit.common import DistributedTest
2430

2531

2632
def _make_engine():
@@ -80,6 +86,154 @@ def test_constructor_defaults_without_cfg():
8086
assert rollout.use_shared_prefill is False
8187

8288

89+
def test_global_work_remaining_uses_requested_group(monkeypatch):
90+
process_group = object()
91+
all_reduce = MagicMock(side_effect=lambda flag, **kwargs: flag.fill_(1))
92+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True)
93+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.get_world_size", lambda group: 2)
94+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.all_reduce", all_reduce)
95+
96+
assert GlobalWorkRemaining(process_group)(torch.tensor(0)) is True
97+
assert all_reduce.call_args.kwargs["group"] is process_group
98+
99+
100+
def test_global_eos_stops_only_when_no_global_work_remains():
101+
rendezvous = MagicMock(side_effect=[True, False])
102+
criterion = _GlobalEosStoppingCriteria(response_start=2, eos_token_id=2, work_remaining=rendezvous)
103+
104+
keep_going = criterion(torch.tensor([[10, 11, 2], [10, 11, 5]]), scores=None)
105+
stop = criterion(torch.tensor([[10, 11, 2], [10, 11, 2]]), scores=None)
106+
107+
assert keep_going is False
108+
assert stop is True
109+
# Transformers 4.32 applies Python any() to criterion results.
110+
assert any([keep_going]) is False
111+
assert any([stop]) is True
112+
assert [call.args[0].item() for call in rendezvous.call_args_list] == [1, 0]
113+
114+
115+
def test_zero3_rendezvous_uses_optimizer_dp_group(monkeypatch):
116+
engine = _make_engine()
117+
process_group = object()
118+
engine.zero_optimization_partition_weights.return_value = True
119+
engine.optimizer.dp_process_group = process_group
120+
monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True)
121+
122+
rendezvous = HybridEngineRollout(engine, _make_tokenizer())._zero3_work_remaining()
123+
124+
assert rendezvous.process_group is process_group
125+
126+
127+
def test_non_zero3_generation_uses_native_eos():
128+
engine = _make_engine()
129+
engine.zero_optimization_partition_weights.return_value = False
130+
engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]])
131+
rollout = HybridEngineRollout(engine, _make_tokenizer())
132+
133+
rollout.generate(_make_request(), _make_sampling())
134+
135+
assert engine.module.generate.call_args.kwargs["eos_token_id"] == 2
136+
assert engine.module.generate.call_args.kwargs["stopping_criteria"] is None
137+
138+
139+
class TestGlobalWorkRemainingSubgroups(DistributedTest):
140+
world_size = 4
141+
backend = "gloo"
142+
requires_cuda_env = False
143+
init_distributed = False
144+
145+
def test(self):
146+
# This is a CPU/Gloo topology test. Avoid binding the Gloo default
147+
# group to the host's visible accelerator when running on a GPU node.
148+
with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator:
149+
mock_accelerator.return_value.device_name.return_value = "cpu"
150+
deepspeed.init_distributed(dist_backend="gloo")
151+
first_group = dist.new_group(ranks=[0, 1])
152+
second_group = dist.new_group(ranks=[2, 3])
153+
rank = dist.get_rank()
154+
process_group = first_group if rank < 2 else second_group
155+
156+
# Only rank 0 has local work. Its DP sibling must continue, while the
157+
# independent DP subgroup must be able to stop.
158+
local_work_remaining = torch.tensor(rank == 0, dtype=torch.int32)
159+
global_work_remaining = GlobalWorkRemaining(process_group)(local_work_remaining)
160+
161+
assert global_work_remaining is (rank < 2)
162+
163+
164+
class TestZero3SynchronizedEarlyStop(DistributedTest):
165+
world_size = 2
166+
167+
def test(self):
168+
"""Different local EOS steps stop at the latest EOS, not max length."""
169+
from transformers import Qwen2Config, Qwen2ForCausalLM
170+
171+
rank = dist.get_rank()
172+
device = torch.device(f"cuda:{rank}")
173+
model = Qwen2ForCausalLM(
174+
Qwen2Config(
175+
vocab_size=32,
176+
max_position_embeddings=16,
177+
hidden_size=32,
178+
intermediate_size=64,
179+
num_hidden_layers=2,
180+
num_attention_heads=4,
181+
num_key_value_heads=4,
182+
bos_token_id=1,
183+
eos_token_id=2,
184+
pad_token_id=0,
185+
))
186+
ds_config = {
187+
"train_micro_batch_size_per_gpu": 1,
188+
"optimizer": {
189+
"type": "AdamW",
190+
"params": {
191+
"lr": 1e-5,
192+
"torch_adam": True
193+
},
194+
},
195+
"zero_optimization": {
196+
"stage": 3
197+
},
198+
"hybrid_engine": {
199+
"enabled": True,
200+
"inference_tp_size": 1,
201+
"pin_parameters": True,
202+
},
203+
}
204+
engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=ds_config)
205+
engine.eval()
206+
207+
target_eos_step = 2 if rank == 0 else 5
208+
forward_count = 0
209+
original_forward = engine.module.forward
210+
211+
def scripted_forward(_module, *args, **kwargs):
212+
nonlocal forward_count
213+
output = original_forward(*args, **kwargs)
214+
forward_count += 1
215+
scripted_token = 2 if forward_count >= target_eos_step else 3
216+
logits = torch.full_like(output.logits, torch.finfo(output.logits.dtype).min)
217+
logits[:, -1, scripted_token] = 0
218+
output.logits = logits
219+
return output
220+
221+
engine.module.forward = MethodType(scripted_forward, engine.module)
222+
tokenizer = SimpleNamespace(pad_token_id=0, eos_token_id=2)
223+
rollout = HybridEngineRollout(engine, tokenizer)
224+
request = RolloutRequest(
225+
prompt_ids=torch.tensor([[5, 6]], device=device),
226+
prompt_attention_mask=torch.ones(1, 2, dtype=torch.long, device=device),
227+
)
228+
result = rollout.generate(request, SamplingConfig(max_new_tokens=8, temperature=0))
229+
230+
counts = [torch.zeros(1, dtype=torch.int32, device=device) for _ in range(2)]
231+
dist.all_gather(counts, torch.tensor([forward_count], dtype=torch.int32, device=device))
232+
assert [count.item() for count in counts] == [5, 5]
233+
assert int(result.attention_mask[:, 2:].sum()) == target_eos_step
234+
assert result.input_ids[0, 1 + target_eos_step].item() == 2
235+
236+
83237
@patch("deepspeed.runtime.rollout.hybrid_engine_rollout.time.perf_counter")
84238
@patch("deepspeed.runtime.rollout.hybrid_engine_rollout.get_accelerator")
85239
def test_generate_records_profile_when_enabled(mock_get_accelerator, mock_perf_counter):

0 commit comments

Comments
 (0)