|
8 | 8 | """ |
9 | 9 |
|
10 | 10 | from types import SimpleNamespace |
| 11 | +from types import MethodType |
11 | 12 | from unittest.mock import MagicMock, patch |
12 | 13 |
|
13 | 14 | import pytest |
14 | 15 | import torch |
15 | 16 | from transformers import PretrainedConfig |
16 | 17 |
|
| 18 | +import deepspeed |
| 19 | +import deepspeed.comm as dist |
17 | 20 | from deepspeed.ops.transformer.inference.op_binding.workspace import WorkspaceOp |
18 | 21 | from deepspeed.runtime.hybrid_engine import DeepSpeedHybridEngine |
19 | 22 | from deepspeed.runtime.rollout.base import RolloutRequest, SamplingConfig |
20 | 23 | from deepspeed.runtime.rollout.hybrid_engine_rollout import ( |
| 24 | + GlobalWorkRemaining, |
21 | 25 | HybridEngineRollout, |
22 | 26 | HybridEngineRolloutConfig, |
| 27 | + _GlobalEosStoppingCriteria, |
23 | 28 | ) |
24 | 29 | from deepspeed.utils.static_cache import DeepSpeedStaticCache |
| 30 | +from unit.common import DistributedTest |
25 | 31 |
|
26 | 32 |
|
27 | 33 | def _make_engine(): |
@@ -550,6 +556,242 @@ def test_continuous_generation_treats_none_repetition_penalty_as_one(): |
550 | 556 |
|
551 | 557 | assert next_tokens.tolist() == [[0]] |
552 | 558 |
|
| 559 | +def test_global_work_remaining_uses_requested_group(monkeypatch): |
| 560 | + process_group = object() |
| 561 | + all_reduce = MagicMock(side_effect=lambda flag, **kwargs: flag.fill_(1)) |
| 562 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True) |
| 563 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.get_world_size", lambda group: 2) |
| 564 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.all_reduce", all_reduce) |
| 565 | + |
| 566 | + assert GlobalWorkRemaining(process_group)(torch.tensor(0)) is True |
| 567 | + assert all_reduce.call_args.kwargs["group"] is process_group |
| 568 | + |
| 569 | + |
| 570 | +def test_global_eos_stops_only_when_no_global_work_remains(): |
| 571 | + rendezvous = MagicMock(side_effect=[True, False]) |
| 572 | + criterion = _GlobalEosStoppingCriteria(response_start=2, eos_token_id=2, work_remaining=rendezvous) |
| 573 | + |
| 574 | + keep_going = criterion(torch.tensor([[10, 11, 2], [10, 11, 5]]), scores=None) |
| 575 | + stop = criterion(torch.tensor([[10, 11, 2], [10, 11, 2]]), scores=None) |
| 576 | + |
| 577 | + assert keep_going is False |
| 578 | + assert stop is True |
| 579 | + # Transformers 4.32 applies Python any() to criterion results. |
| 580 | + assert any([keep_going]) is False |
| 581 | + assert any([stop]) is True |
| 582 | + assert [call.args[0].item() for call in rendezvous.call_args_list] == [1, 0] |
| 583 | + |
| 584 | + |
| 585 | +def test_zero3_rendezvous_uses_optimizer_dp_group(monkeypatch): |
| 586 | + engine = _make_engine() |
| 587 | + process_group = object() |
| 588 | + engine.zero_optimization_partition_weights.return_value = True |
| 589 | + engine.optimizer.dp_process_group = process_group |
| 590 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True) |
| 591 | + |
| 592 | + rendezvous = HybridEngineRollout(engine, _make_tokenizer())._zero3_work_remaining() |
| 593 | + |
| 594 | + assert rendezvous.process_group is process_group |
| 595 | + |
| 596 | + |
| 597 | +def test_non_zero3_generation_uses_native_eos(): |
| 598 | + engine = _make_engine() |
| 599 | + engine.zero_optimization_partition_weights.return_value = False |
| 600 | + engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]]) |
| 601 | + rollout = HybridEngineRollout(engine, _make_tokenizer()) |
| 602 | + |
| 603 | + rollout.generate(_make_request(), _make_sampling()) |
| 604 | + |
| 605 | + assert engine.module.generate.call_args.kwargs["eos_token_id"] == 2 |
| 606 | + assert engine.module.generate.call_args.kwargs["stopping_criteria"] is None |
| 607 | + # Non-ZeRO-3 generation must keep the Transformers default. |
| 608 | + assert engine.module.generate.call_args.kwargs["synced_gpus"] is None |
| 609 | + |
| 610 | + |
| 611 | +def test_zero3_generation_disables_native_synced_gpus(monkeypatch): |
| 612 | + engine = _make_engine() |
| 613 | + engine.zero_optimization_partition_weights.return_value = True |
| 614 | + engine.optimizer.dp_process_group = object() |
| 615 | + engine.module.generate.return_value = torch.tensor([[0, 1, 2, 5, 6], [0, 3, 4, 7, 8]]) |
| 616 | + monkeypatch.setattr("deepspeed.runtime.rollout.hybrid_engine_rollout.dist.is_initialized", lambda: True) |
| 617 | + |
| 618 | + HybridEngineRollout(engine, _make_tokenizer()).generate(_make_request(), _make_sampling()) |
| 619 | + |
| 620 | + kwargs = engine.module.generate.call_args.kwargs |
| 621 | + # Transformers would otherwise derive synced_gpus from the default world |
| 622 | + # size and issue an ungrouped world all-reduce inside the decode loop. |
| 623 | + assert kwargs["synced_gpus"] is False |
| 624 | + assert kwargs["eos_token_id"] is None |
| 625 | + assert kwargs["stopping_criteria"] is not None |
| 626 | + |
| 627 | + |
| 628 | +class TestGlobalWorkRemainingSubgroups(DistributedTest): |
| 629 | + world_size = 4 |
| 630 | + backend = "gloo" |
| 631 | + requires_cuda_env = False |
| 632 | + init_distributed = False |
| 633 | + |
| 634 | + def test(self): |
| 635 | + # This is a CPU/Gloo topology test. Avoid binding the Gloo default |
| 636 | + # group to the host's visible accelerator when running on a GPU node. |
| 637 | + with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator: |
| 638 | + mock_accelerator.return_value.device_name.return_value = "cpu" |
| 639 | + deepspeed.init_distributed(dist_backend="gloo") |
| 640 | + first_group = dist.new_group(ranks=[0, 1]) |
| 641 | + second_group = dist.new_group(ranks=[2, 3]) |
| 642 | + rank = dist.get_rank() |
| 643 | + process_group = first_group if rank < 2 else second_group |
| 644 | + |
| 645 | + # Only rank 0 has local work. Its DP sibling must continue, while the |
| 646 | + # independent DP subgroup must be able to stop. |
| 647 | + local_work_remaining = torch.tensor(rank == 0, dtype=torch.int32) |
| 648 | + global_work_remaining = GlobalWorkRemaining(process_group)(local_work_remaining) |
| 649 | + |
| 650 | + assert global_work_remaining is (rank < 2) |
| 651 | + |
| 652 | + |
| 653 | +class TestZero3SubgroupGeneration(DistributedTest): |
| 654 | + """ZeRO-3 rollout inside one data-parallel subgroup must not touch WORLD. |
| 655 | +
|
| 656 | + When Transformers can observe ZeRO-3 it derives ``synced_gpus`` from the |
| 657 | + default world size, and the decode loop then issues an ungrouped world |
| 658 | + all-reduce. A subgroup that enters rollout while the remaining WORLD ranks |
| 659 | + stay out would block forever on that collective. Here ranks 0-1 generate |
| 660 | + inside one subgroup and ranks 2-3 never enter rollout at all. |
| 661 | + """ |
| 662 | + world_size = 4 |
| 663 | + backend = "gloo" |
| 664 | + requires_cuda_env = False |
| 665 | + init_distributed = False |
| 666 | + |
| 667 | + def test(self): |
| 668 | + from transformers import Qwen2Config, Qwen2ForCausalLM |
| 669 | + from transformers.integrations.deepspeed import HfDeepSpeedConfig, is_deepspeed_zero3_enabled |
| 670 | + |
| 671 | + # This is a CPU/Gloo topology test. Avoid binding the Gloo default |
| 672 | + # group to the host's visible accelerator when running on a GPU node. |
| 673 | + with patch("deepspeed.comm.torch.get_accelerator") as mock_accelerator: |
| 674 | + mock_accelerator.return_value.device_name.return_value = "cpu" |
| 675 | + deepspeed.init_distributed(dist_backend="gloo") |
| 676 | + first_group = dist.new_group(ranks=[0, 1]) |
| 677 | + dist.new_group(ranks=[2, 3]) |
| 678 | + rank = dist.get_rank() |
| 679 | + dist.barrier() |
| 680 | + |
| 681 | + # Make ZeRO-3 visible to Transformers so the native default for |
| 682 | + # synced_gpus would be True. Keep the config alive for the process. |
| 683 | + deepspeed_config = HfDeepSpeedConfig({"zero_optimization": {"stage": 3}}) |
| 684 | + assert is_deepspeed_zero3_enabled() |
| 685 | + |
| 686 | + if rank >= 2: |
| 687 | + # Non-participating WORLD ranks must not be required by rollout. |
| 688 | + del deepspeed_config |
| 689 | + return |
| 690 | + |
| 691 | + model = Qwen2ForCausalLM( |
| 692 | + Qwen2Config( |
| 693 | + vocab_size=32, |
| 694 | + max_position_embeddings=16, |
| 695 | + hidden_size=32, |
| 696 | + intermediate_size=64, |
| 697 | + num_hidden_layers=2, |
| 698 | + num_attention_heads=4, |
| 699 | + num_key_value_heads=4, |
| 700 | + bos_token_id=1, |
| 701 | + eos_token_id=2, |
| 702 | + pad_token_id=0, |
| 703 | + )) |
| 704 | + engine = SimpleNamespace( |
| 705 | + module=model, |
| 706 | + zero_optimization_partition_weights=lambda: True, |
| 707 | + optimizer=SimpleNamespace(dp_process_group=first_group), |
| 708 | + ) |
| 709 | + rollout = HybridEngineRollout(engine, SimpleNamespace(pad_token_id=0, eos_token_id=2)) |
| 710 | + request = RolloutRequest( |
| 711 | + prompt_ids=torch.tensor([[5, 6]]), |
| 712 | + prompt_attention_mask=torch.ones(1, 2, dtype=torch.long), |
| 713 | + ) |
| 714 | + |
| 715 | + result = rollout.generate(request, SamplingConfig(max_new_tokens=3, temperature=0)) |
| 716 | + |
| 717 | + assert result.input_ids.shape == (1, 5) |
| 718 | + assert result.attention_mask.shape == (1, 5) |
| 719 | + assert result.input_ids[0, :2].tolist() == [5, 6] |
| 720 | + del deepspeed_config |
| 721 | + |
| 722 | + |
| 723 | +class TestZero3SynchronizedEarlyStop(DistributedTest): |
| 724 | + world_size = 2 |
| 725 | + |
| 726 | + def test(self): |
| 727 | + """Different local EOS steps stop at the latest EOS, not max length.""" |
| 728 | + from transformers import Qwen2Config, Qwen2ForCausalLM |
| 729 | + |
| 730 | + rank = dist.get_rank() |
| 731 | + device = torch.device(f"cuda:{rank}") |
| 732 | + model = Qwen2ForCausalLM( |
| 733 | + Qwen2Config( |
| 734 | + vocab_size=32, |
| 735 | + max_position_embeddings=16, |
| 736 | + hidden_size=32, |
| 737 | + intermediate_size=64, |
| 738 | + num_hidden_layers=2, |
| 739 | + num_attention_heads=4, |
| 740 | + num_key_value_heads=4, |
| 741 | + bos_token_id=1, |
| 742 | + eos_token_id=2, |
| 743 | + pad_token_id=0, |
| 744 | + )) |
| 745 | + ds_config = { |
| 746 | + "train_micro_batch_size_per_gpu": 1, |
| 747 | + "optimizer": { |
| 748 | + "type": "AdamW", |
| 749 | + "params": { |
| 750 | + "lr": 1e-5, |
| 751 | + "torch_adam": True |
| 752 | + }, |
| 753 | + }, |
| 754 | + "zero_optimization": { |
| 755 | + "stage": 3 |
| 756 | + }, |
| 757 | + "hybrid_engine": { |
| 758 | + "enabled": True, |
| 759 | + "inference_tp_size": 1, |
| 760 | + "pin_parameters": True, |
| 761 | + }, |
| 762 | + } |
| 763 | + engine, _, _, _ = deepspeed.initialize(model=model, model_parameters=model.parameters(), config=ds_config) |
| 764 | + engine.eval() |
| 765 | + |
| 766 | + target_eos_step = 2 if rank == 0 else 5 |
| 767 | + forward_count = 0 |
| 768 | + original_forward = engine.module.forward |
| 769 | + |
| 770 | + def scripted_forward(_module, *args, **kwargs): |
| 771 | + nonlocal forward_count |
| 772 | + output = original_forward(*args, **kwargs) |
| 773 | + forward_count += 1 |
| 774 | + scripted_token = 2 if forward_count >= target_eos_step else 3 |
| 775 | + logits = torch.full_like(output.logits, torch.finfo(output.logits.dtype).min) |
| 776 | + logits[:, -1, scripted_token] = 0 |
| 777 | + output.logits = logits |
| 778 | + return output |
| 779 | + |
| 780 | + engine.module.forward = MethodType(scripted_forward, engine.module) |
| 781 | + tokenizer = SimpleNamespace(pad_token_id=0, eos_token_id=2) |
| 782 | + rollout = HybridEngineRollout(engine, tokenizer) |
| 783 | + request = RolloutRequest( |
| 784 | + prompt_ids=torch.tensor([[5, 6]], device=device), |
| 785 | + prompt_attention_mask=torch.ones(1, 2, dtype=torch.long, device=device), |
| 786 | + ) |
| 787 | + result = rollout.generate(request, SamplingConfig(max_new_tokens=8, temperature=0)) |
| 788 | + |
| 789 | + counts = [torch.zeros(1, dtype=torch.int32, device=device) for _ in range(2)] |
| 790 | + dist.all_gather(counts, torch.tensor([forward_count], dtype=torch.int32, device=device)) |
| 791 | + assert [count.item() for count in counts] == [5, 5] |
| 792 | + assert int(result.attention_mask[:, 2:].sum()) == target_eos_step |
| 793 | + assert result.input_ids[0, 1 + target_eos_step].item() == 2 |
| 794 | + |
553 | 795 |
|
554 | 796 | @patch("deepspeed.runtime.rollout.hybrid_engine_rollout.time.perf_counter") |
555 | 797 | @patch("deepspeed.runtime.rollout.hybrid_engine_rollout.get_accelerator") |
|
0 commit comments