Skip to content

Commit 0c31794

Browse files
committed
Persist ZeRO-3 HybridMegatron QKV layout changes
Signed-off-by: LiRunGuo <li19107254665@gmail.com>
1 parent dc046b1 commit 0c31794

2 files changed

Lines changed: 58 additions & 2 deletions

File tree

‎deepspeed/module_inject/containers/features/hybrid_megatron.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ def transform_for_inference(self) -> None:
4343
param_list = [self.qkvw, self.qkvb]
4444
non_active_params = [param for param in param_list if (hasattr(param, 'ds_id') and \
4545
param.ds_status == ZeroParamStatus.NOT_AVAILABLE)]
46-
with GatheredParameters(non_active_params):
46+
# Persist the QKV layout mutation when ZeRO-3 repartitions the
47+
# gathered parameters on context exit.
48+
with GatheredParameters(non_active_params, modifier_rank=0):
4749
self._align_qkv(self.qkvw)
4850
self._align_qkv(self.qkvb)
4951
else:
@@ -79,7 +81,8 @@ def transform_for_training(self):
7981
param_list = [self.qkvw, self.qkvb]
8082
non_active_params = [param for param in param_list if (hasattr(param, 'ds_id') and \
8183
param.ds_status == ZeroParamStatus.NOT_AVAILABLE)]
82-
with GatheredParameters(non_active_params):
84+
# Persist the inverse mutation before returning to training.
85+
with GatheredParameters(non_active_params, modifier_rank=0):
8386
self._partition_qkv(self.qkvw)
8487
self._partition_qkv(self.qkvb)
8588
else:
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# DeepSpeed Team
5+
6+
import torch
7+
8+
from deepspeed.module_inject.containers.gptneox import DS_GPTNEOXContainer
9+
from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
10+
11+
12+
class _RecordedGather:
13+
14+
calls = []
15+
16+
def __init__(self, params, modifier_rank=None):
17+
self.params = params
18+
self.modifier_rank = modifier_rank
19+
20+
def __enter__(self):
21+
self.calls.append(self.modifier_rank)
22+
23+
def __exit__(self, exc_type, exc_value, traceback):
24+
return False
25+
26+
27+
def _partitioned_parameter(values):
28+
parameter = torch.nn.Parameter(values)
29+
parameter.ds_id = 0
30+
parameter.ds_status = ZeroParamStatus.NOT_AVAILABLE
31+
return parameter
32+
33+
34+
def test_zero3_qkv_layout_round_trip_persists_mutation(monkeypatch):
35+
monkeypatch.setattr("deepspeed.runtime.zero.GatheredParameters", _RecordedGather)
36+
_RecordedGather.calls.clear()
37+
38+
container = object.__new__(DS_GPTNEOXContainer)
39+
container.num_attention_heads = 2
40+
container.qkvw = _partitioned_parameter(torch.arange(48, dtype=torch.float32).reshape(12, 4))
41+
container.qkvb = _partitioned_parameter(torch.arange(12, dtype=torch.float32))
42+
43+
initial_weight = container.qkvw.detach().clone()
44+
initial_bias = container.qkvb.detach().clone()
45+
46+
container.transform_for_inference()
47+
assert not torch.equal(container.qkvw, initial_weight)
48+
assert not torch.equal(container.qkvb, initial_bias)
49+
50+
container.transform_for_training()
51+
assert torch.equal(container.qkvw, initial_weight)
52+
assert torch.equal(container.qkvb, initial_bias)
53+
assert _RecordedGather.calls == [0, 0]

0 commit comments

Comments
 (0)