|
9 | 9 | """ |
10 | 10 |
|
11 | 11 | from types import SimpleNamespace |
| 12 | +from types import MethodType |
12 | 13 | from unittest.mock import MagicMock, patch |
13 | 14 |
|
14 | 15 | import pytest |
15 | 16 | import torch |
16 | 17 |
|
| 18 | +import deepspeed |
| 19 | +import deepspeed.comm as dist |
17 | 20 | from deepspeed.ops.transformer.inference.op_binding.workspace import WorkspaceOp |
18 | 21 | from deepspeed.runtime.hybrid_engine import DeepSpeedHybridEngine |
19 | 22 | from deepspeed.runtime.rollout.base import RolloutRequest, SamplingConfig |
20 | 23 | from deepspeed.runtime.rollout.hybrid_engine_rollout import ( |
| 24 | + GlobalWorkRemaining, |
21 | 25 | HybridEngineRollout, |
22 | 26 | HybridEngineRolloutConfig, |
| 27 | + _GlobalEosStoppingCriteria, |
23 | 28 | ) |
| 29 | +from unit.common import DistributedTest |
24 | 30 |
|
25 | 31 |
|
26 | 32 | def _make_engine(): |
@@ -80,6 +86,154 @@ def test_constructor_defaults_without_cfg(): |
80 | 86 | assert rollout.use_shared_prefill is False |
81 | 87 |
|
82 | 88 |
|
| 89 | +def test_global_work_remaining_uses_requested_group(monkeypatch): |
| 90 | + process_group = object() |
| 91 | + all_reduce = MagicMock(side_effect=lambda flag, **kwargs: flag.fill_(1)) |
| 92 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True) |
| 93 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.get_world_size", lambda group: 2) |
| 94 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.all_reduce", all_reduce) |
| 95 | + |
| 96 | + assert GlobalWorkRemaining(process_group)(torch.tensor(0)) is True |
| 97 | + assert all_reduce.call_args.kwargs["group"] is process_group |
| 98 | + |
| 99 | + |
| 100 | +def test_global_eos_stops_only_when_no_global_work_remains(): |
| 101 | + rendezvous = MagicMock(side_effect=[True, False]) |
| 102 | + criterion = _GlobalEosStoppingCriteria(response_start=2, eos_token_id=2, work_remaining=rendezvous) |
| 103 | + |
| 104 | + keep_going = criterion(torch.tensor([[10, 11, 2], [10, 11, 5]]), scores=None) |
| 105 | + stop = criterion(torch.tensor([[10, 11, 2], [10, 11, 2]]), scores=None) |
| 106 | + |
| 107 | + assert keep_going is False |
| 108 | + assert stop is True |
| 109 | + # Transformers 4.32 applies Python any() to criterion results. |
| 110 | + assert any([keep_going]) is False |
| 111 | + assert any([stop]) is True |
| 112 | + assert [call.args[0].item() for call in rendezvous.call_args_list] == [1, 0] |
| 113 | + |
| 114 | + |
| 115 | +def test_zero3_rendezvous_uses_optimizer_dp_group(monkeypatch): |
| 116 | + engine = _make_engine() |
| 117 | + process_group = object() |
| 118 | + engine.zero_optimization_partition_weights.return_value = True |
| 119 | + engine.optimizer.dp_process_group = process_group |
| 120 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True) |
| 121 | + |
| 122 | + rendezvous = HybridEngineRollout(engine, _make_tokenizer())._zero3_work_remaining() |
| 123 | + |
| 124 | + assert rendezvous.process_group is process_group |
| 125 | + |
| 126 | + |
| 127 | +def test_non_zero3_generation_uses_native_eos(): |
| 128 | + engine = _make_engine() |
| 129 | + engine.zero_optimization_partition_weights.return_value = False |
| 130 | + engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]]) |
| 131 | + rollout = HybridEngineRollout(engine, _make_tokenizer()) |
| 132 | + |
| 133 | + rollout.generate(_make_request(), _make_sampling()) |
| 134 | + |
| 135 | + assert engine.module.generate.call_args.kwargs["eos_token_id"] == 2 |
| 136 | + assert engine.module.generate.call_args.kwargs["stopping_criteria"] is None |
| 137 | + |
| 138 | + |
| 139 | +class TestGlobalWorkRemainingSubgroups(DistributedTest): |
| 140 | + world_size = 4 |
| 141 | + backend = "gloo" |
| 142 | + requires_cuda_env = False |
| 143 | + init_distributed = False |
| 144 | + |
| 145 | + def test(self): |
| 146 | + # This is a CPU/Gloo topology test. Avoid binding the Gloo default |
| 147 | + # group to the host's visible accelerator when running on a GPU node. |
| 148 | + with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator: |
| 149 | + mock_accelerator.return_value.device_name.return_value = "cpu" |
| 150 | + deepspeed.init_distributed(dist_backend="gloo") |
| 151 | + first_group = dist.new_group(ranks=[0, 1]) |
| 152 | + second_group = dist.new_group(ranks=[2, 3]) |
| 153 | + rank = dist.get_rank() |
| 154 | + process_group = first_group if rank < 2 else second_group |
| 155 | + |
| 156 | + # Only rank 0 has local work. Its DP sibling must continue, while the |
| 157 | + # independent DP subgroup must be able to stop. |
| 158 | + local_work_remaining = torch.tensor(rank == 0, dtype=torch.int32) |
| 159 | + global_work_remaining = GlobalWorkRemaining(process_group)(local_work_remaining) |
| 160 | + |
| 161 | + assert global_work_remaining is (rank < 2) |
| 162 | + |
| 163 | + |
| 164 | +class TestZero3SynchronizedEarlyStop(DistributedTest): |
| 165 | + world_size = 2 |
| 166 | + |
| 167 | + def test(self): |
| 168 | + """Different local EOS steps stop at the latest EOS, not max length.""" |
| 169 | + from transformers import Qwen2Config, Qwen2ForCausalLM |
| 170 | + |
| 171 | + rank = dist.get_rank() |
| 172 | + device = torch.device(f"cuda:{rank}") |
| 173 | + model = Qwen2ForCausalLM( |
| 174 | + Qwen2Config( |
| 175 | + vocab_size=32, |
| 176 | + max_position_embeddings=16, |
| 177 | + hidden_size=32, |
| 178 | + intermediate_size=64, |
| 179 | + num_hidden_layers=2, |
| 180 | + num_attention_heads=4, |
| 181 | + num_key_value_heads=4, |
| 182 | + bos_token_id=1, |
| 183 | + eos_token_id=2, |
| 184 | + pad_token_id=0, |
| 185 | + )) |
| 186 | + ds_config = { |
| 187 | + "train_micro_batch_size_per_gpu": 1, |
| 188 | + "optimizer": { |
| 189 | + "type": "AdamW", |
| 190 | + "params": { |
| 191 | + "lr": 1e-5, |
| 192 | + "torch_adam": True |
| 193 | + }, |
| 194 | + }, |
| 195 | + "zero_optimization": { |
| 196 | + "stage": 3 |
| 197 | + }, |
| 198 | + "hybrid_engine": { |
| 199 | + "enabled": True, |
| 200 | + "inference_tp_size": 1, |
| 201 | + "pin_parameters": True, |
| 202 | + }, |
| 203 | + } |
| 204 | + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=ds_config) |
| 205 | + engine.eval() |
| 206 | + |
| 207 | + target_eos_step = 2 if rank == 0 else 5 |
| 208 | + forward_count = 0 |
| 209 | + original_forward = engine.module.forward |
| 210 | + |
| 211 | + def scripted_forward(_module, *args, **kwargs): |
| 212 | + nonlocal forward_count |
| 213 | + output = original_forward(*args, **kwargs) |
| 214 | + forward_count += 1 |
| 215 | + scripted_token = 2 if forward_count >= target_eos_step else 3 |
| 216 | + logits = torch.full_like(output.logits, torch.finfo(output.logits.dtype).min) |
| 217 | + logits[:, -1, scripted_token] = 0 |
| 218 | + output.logits = logits |
| 219 | + return output |
| 220 | + |
| 221 | + engine.module.forward = MethodType(scripted_forward, engine.module) |
| 222 | + tokenizer = SimpleNamespace(pad_token_id=0, eos_token_id=2) |
| 223 | + rollout = HybridEngineRollout(engine, tokenizer) |
| 224 | + request = RolloutRequest( |
| 225 | + prompt_ids=torch.tensor([[5, 6]], device=device), |
| 226 | + prompt_attention_mask=torch.ones(1, 2, dtype=torch.long, device=device), |
| 227 | + ) |
| 228 | + result = rollout.generate(request, SamplingConfig(max_new_tokens=8, temperature=0)) |
| 229 | + |
| 230 | + counts = [torch.zeros(1, dtype=torch.int32, device=device) for _ in range(2)] |
| 231 | + dist.all_gather(counts, torch.tensor([forward_count], dtype=torch.int32, device=device)) |
| 232 | + assert [count.item() for count in counts] == [5, 5] |
| 233 | + assert int(result.attention_mask[:, 2:].sum()) == target_eos_step |
| 234 | + assert result.input_ids[0, 1 + target_eos_step].item() == 2 |
| 235 | + |
| 236 | + |
83 | 237 | @patch("deepspeed.runtime.rollout.hybrid_engine_rollout.time.perf_counter") |
84 | 238 | @patch("deepspeed.runtime.rollout.hybrid_engine_rollout.get_accelerator") |
85 | 239 | def test_generate_records_profile_when_enabled(mock_get_accelerator, mock_perf_counter): |
|
0 commit comments