|
| 1 | +# Copyright (c) Microsoft Corporation. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +# DeepSpeed Team |
| 5 | + |
| 6 | +import torch |
| 7 | + |
| 8 | +from deepspeed.module_inject.containers.gptneox import DS_GPTNEOXContainer |
| 9 | +from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus |
| 10 | + |
| 11 | + |
| 12 | +class _RecordedGather: |
| 13 | + |
| 14 | + calls = [] |
| 15 | + |
| 16 | + def __init__(self, params, modifier_rank=None): |
| 17 | + self.params = params |
| 18 | + self.modifier_rank = modifier_rank |
| 19 | + |
| 20 | + def __enter__(self): |
| 21 | + self.calls.append(self.modifier_rank) |
| 22 | + |
| 23 | + def __exit__(self, exc_type, exc_value, traceback): |
| 24 | + return False |
| 25 | + |
| 26 | + |
| 27 | +def _partitioned_parameter(values): |
| 28 | + parameter = torch.nn.Parameter(values) |
| 29 | + parameter.ds_id = 0 |
| 30 | + parameter.ds_status = ZeroParamStatus.NOT_AVAILABLE |
| 31 | + return parameter |
| 32 | + |
| 33 | + |
| 34 | +def test_zero3_qkv_layout_round_trip_persists_mutation(monkeypatch): |
| 35 | + monkeypatch.setattr("deepspeed.runtime.zero.GatheredParameters", _RecordedGather) |
| 36 | + _RecordedGather.calls.clear() |
| 37 | + |
| 38 | + container = object.__new__(DS_GPTNEOXContainer) |
| 39 | + container.num_attention_heads = 2 |
| 40 | + container.qkvw = _partitioned_parameter(torch.arange(48, dtype=torch.float32).reshape(12, 4)) |
| 41 | + container.qkvb = _partitioned_parameter(torch.arange(12, dtype=torch.float32)) |
| 42 | + |
| 43 | + initial_weight = container.qkvw.detach().clone() |
| 44 | + initial_bias = container.qkvb.detach().clone() |
| 45 | + |
| 46 | + container.transform_for_inference() |
| 47 | + assert not torch.equal(container.qkvw, initial_weight) |
| 48 | + assert not torch.equal(container.qkvb, initial_bias) |
| 49 | + |
| 50 | + container.transform_for_training() |
| 51 | + assert torch.equal(container.qkvw, initial_weight) |
| 52 | + assert torch.equal(container.qkvb, initial_bias) |
| 53 | + assert _RecordedGather.calls == [0, 0] |
0 commit comments