diff --git a/CHANGELOG_DEV.md b/CHANGELOG_DEV.md index 43d0c6e2d..82a99c22d 100644 --- a/CHANGELOG_DEV.md +++ b/CHANGELOG_DEV.md @@ -217,4 +217,17 @@ This PR improves training monitoring and logging across runs besides some other * Add tutorials on Einsum Transformer (Example model integration) and profiling **Breaking Changes** -* experiments_root_path is now exposed on an API level \ No newline at end of file +* experiments_root_path is now exposed on an API level + +## PR #XXX Torch-free dataloader utilities for CPU-only tooling + +**`TokenizerInstantiationModel`.** Lets a tool reuse a packing config for its tokenizer without +also having to satisfy that config's settings, which name a specific source file the tool has no +interest in. + +**`get_logger` no longer requires torch.** The `import torch` in `utils/logger_utils.py` is only +there for the rank prefix, but it is the reason `dataloader/create_index.py` -- and therefore +`IndexGenerator` -- could not be imported without torch. Since modalities declares torch in its +`cpu`/`cu12x` extras rather than as a base dependency, that made CPU-only data-preprocessing +tooling install a multi-gigabyte wheel it never called. The import is now guarded and the rank +prefix is simply omitted when torch is absent; behaviour with torch installed is unchanged. diff --git a/src/modalities/config/instantiation_models.py b/src/modalities/config/instantiation_models.py index fd7fd3b78..5e0172e50 100644 --- a/src/modalities/config/instantiation_models.py +++ b/src/modalities/config/instantiation_models.py @@ -223,6 +223,18 @@ class PackedDatasetSettings(BaseModel): settings: PackedDatasetSettings +class TokenizerInstantiationModel(BaseModel): + """Builds only the tokenizer of a config. + + Lets a tool reuse a packing config for its tokenizer without also having to satisfy + that config's settings, which name a specific source file the tool has no interest + in. Building from the packing config is what guarantees the same tokenizer is used + for measuring token estimates and for the packing those estimates predict. + """ + + tokenizer: PydanticTokenizerIFType + + class TextGenerationInstantiationModel(BaseModel): class TextGenerationSettings(BaseModel): model_path: FilePath diff --git a/src/modalities/utils/logger_utils.py b/src/modalities/utils/logger_utils.py index 095e998d1..549df3806 100644 --- a/src/modalities/utils/logger_utils.py +++ b/src/modalities/utils/logger_utils.py @@ -1,12 +1,19 @@ import logging -import torch +try: + # Only needed for the rank prefix. Guarded so that CPU-only tooling -- data + # preprocessing in particular -- can import the dataloader utilities without + # pulling in torch, which modalities declares in its cpu/cu12x extras rather + # than as a base dependency. + import torch +except ImportError: # pragma: no cover - exercised only in torch-free installs + torch = None def get_logger(name: str = "main") -> logging.Logger: rank_info = "" - if torch.distributed.is_initialized(): + if torch is not None and torch.distributed.is_initialized(): rank_info = f"[RANK {torch.distributed.get_rank()}] " logger = logging.getLogger(name)