Merge pull request #465 from Modalities/feat/weighted-combined-dataset

feat: torch-free dataloader utilities for CPU-only tooling

Author: rrutmann

CHANGELOG_DEV.md

@@ -217,4 +217,17 @@ This PR improves training monitoring and logging across runs besides some other
 * Add tutorials on Einsum Transformer (Example model integration) and profiling
 
 **Breaking Changes**
-* experiments_root_path is now exposed on an API level
+* experiments_root_path is now exposed on an API level
+
+## PR #XXX Torch-free dataloader utilities for CPU-only tooling
+
+**`TokenizerInstantiationModel`.** Lets a tool reuse a packing config for its tokenizer without
+also having to satisfy that config's settings, which name a specific source file the tool has no
+interest in.
+
+**`get_logger` no longer requires torch.** The `import torch` in `utils/logger_utils.py` is only
+there for the rank prefix, but it is the reason `dataloader/create_index.py` -- and therefore
+`IndexGenerator` -- could not be imported without torch. Since modalities declares torch in its
+`cpu`/`cu12x` extras rather than as a base dependency, that made CPU-only data-preprocessing
+tooling install a multi-gigabyte wheel it never called. The import is now guarded and the rank
+prefix is simply omitted when torch is absent; behaviour with torch installed is unchanged.

src/modalities/config/instantiation_models.py

@@ -223,6 +223,18 @@ class PackedDatasetSettings(BaseModel):
     settings: PackedDatasetSettings
 
 
+class TokenizerInstantiationModel(BaseModel):
+    """Builds only the tokenizer of a config.
+
+    Lets a tool reuse a packing config for its tokenizer without also having to satisfy
+    that config's settings, which name a specific source file the tool has no interest
+    in. Building from the packing config is what guarantees the same tokenizer is used
+    for measuring token estimates and for the packing those estimates predict.
+    """
+
+    tokenizer: PydanticTokenizerIFType
+
+
 class TextGenerationInstantiationModel(BaseModel):
     class TextGenerationSettings(BaseModel):
         model_path: FilePath

src/modalities/utils/logger_utils.py

@@ -1,12 +1,19 @@
 import logging
 
-import torch
+try:
+    # Only needed for the rank prefix. Guarded so that CPU-only tooling -- data
+    # preprocessing in particular -- can import the dataloader utilities without
+    # pulling in torch, which modalities declares in its cpu/cu12x extras rather
+    # than as a base dependency.
+    import torch
+except ImportError:  # pragma: no cover - exercised only in torch-free installs
+    torch = None
 
 
 def get_logger(name: str = "main") -> logging.Logger:
     rank_info = ""
 
-    if torch.distributed.is_initialized():
+    if torch is not None and torch.distributed.is_initialized():
         rank_info = f"[RANK {torch.distributed.get_rank()}] "
 
     logger = logging.getLogger(name)