|
20 | 20 | from tests.utility import find_free_port |
21 | 21 |
|
22 | 22 |
|
23 | | -def patch_config_file(original_config_path: Path, activation_type: str, tmp_dir: Path) -> Path: |
| 23 | +def patch_config_file(original_config_path: Path, activation_type: str, tmp_dir: Path, file_tag: str = "") -> Path: |
24 | 24 | """Patches the original configuration file to set a custom activation type.""" |
25 | 25 | with original_config_path.open("r", encoding="utf-8") as f: |
26 | 26 | config_dict = yaml.safe_load(f) |
27 | 27 |
|
28 | 28 | config_dict["model_raw"]["config"]["activation_type"] = activation_type |
29 | 29 |
|
30 | | - tmp_file_path = tmp_dir / original_config_path.name |
| 30 | + file_suffix = f"_{file_tag}" if file_tag else "" |
| 31 | + tmp_file_path = tmp_dir / f"{original_config_path.stem}{file_suffix}{original_config_path.suffix}" |
31 | 32 | with tmp_file_path.open("w", encoding="utf-8") as f: |
32 | 33 | yaml.safe_dump(config_dict, f) |
33 | 34 |
|
@@ -103,12 +104,16 @@ def _test_tp_sharding_impl( |
103 | 104 | ): |
104 | 105 | # Seed before FSDP2 instantiation |
105 | 106 | torch.manual_seed(42) |
106 | | - fsdp2_path = patch_config_file(fsdp2_config_path, activation_type, tmp_config_dir) |
| 107 | + fsdp2_path = patch_config_file( |
| 108 | + fsdp2_config_path, activation_type, tmp_config_dir, file_tag=f"{activation_type}_rank{process_id}_fsdp2" |
| 109 | + ) |
107 | 110 | fsdp2_model, fsdp2_mesh = self._get_components(fsdp2_path, tmp_path) |
108 | 111 |
|
109 | 112 | # Seed again before TP instantiation to match |
110 | 113 | torch.manual_seed(42) |
111 | | - tp_path = patch_config_file(tp_config_path, activation_type, tmp_config_dir) |
| 114 | + tp_path = patch_config_file( |
| 115 | + tp_config_path, activation_type, tmp_config_dir, file_tag=f"{activation_type}_rank{process_id}_tp" |
| 116 | + ) |
112 | 117 | tp_model, tp_mesh = self._get_components(tp_path, tmp_path) |
113 | 118 |
|
114 | 119 | # Ensure models use the correct MLP |
|
0 commit comments