diff --git a/gptqmodel/utils/hf.py b/gptqmodel/utils/hf.py index 49c57adb6..a24eac705 100644 --- a/gptqmodel/utils/hf.py +++ b/gptqmodel/utils/hf.py @@ -138,9 +138,14 @@ def _sync_local_remote_code_cache(model_id_or_path: Optional[str]) -> None: def get_hf_config_dtype(config: Any) -> Optional[torch.dtype]: - dtype = getattr(config, "dtype", None) - if dtype is None: + dtype = getattr(config, "dtype", _MISSING) + if dtype is _MISSING: dtype = getattr(config, "torch_dtype", None) + elif dtype is None: + # Transformers 5.14 exposes `torch_dtype` as a deprecated property that + # warns even when only read. Preserve genuinely stored legacy metadata + # without touching that alias on modern configs. + dtype = getattr(config, "__dict__", {}).get("torch_dtype") if dtype is None: return None @@ -759,9 +764,28 @@ def inner(self, *args, **kwargs): cache_base_cls = getattr(cache_utils, "Cache", None) if cache_utils is not None else None if cache_base_cls is not None and not hasattr(cache_base_cls, "get_max_length") and hasattr(cache_base_cls, "get_max_cache_shape"): # Older remote decoders expect `get_max_length()`, while newer - # transformers renamed that API to `get_max_cache_shape()`. + # transformers renamed that API to `get_max_cache_shape()`. In + # transformers 5.14, the deprecated cache-level method delegates + # back to `get_max_length()`, so prefer the layer API to avoid a + # compatibility-alias recursion. def get_max_length(self, layer_idx: int = 0) -> Optional[int]: - max_length = self.get_max_cache_shape(layer_idx) + missing = object() + max_length = missing + layers = getattr(self, "layers", None) + if layers is not None: + if 0 <= layer_idx < len(layers): + layer = layers[layer_idx] + layer_get_max_length = getattr(layer, "get_max_length", None) + layer_get_max_cache_shape = getattr(layer, "get_max_cache_shape", None) + if callable(layer_get_max_length): + max_length = layer_get_max_length() + elif callable(layer_get_max_cache_shape): + max_length = layer_get_max_cache_shape() + else: + max_length = -1 + + if max_length is missing: + max_length = self.get_max_cache_shape(layer_idx) return None if max_length is None or max_length < 0 else max_length cache_base_cls.get_max_length = get_max_length diff --git a/tests/test_hf_config_compat.py b/tests/test_hf_config_compat.py index 604fc3b0f..a12bf34bd 100644 --- a/tests/test_hf_config_compat.py +++ b/tests/test_hf_config_compat.py @@ -13,6 +13,7 @@ from gptqmodel.utils import internal_gguf from gptqmodel.utils.hf import ( INTERNAL_HF_GGUF_FILE_KWARG, + get_hf_config_dtype, get_hf_gguf_load_kwargs, normalize_hf_config_compat, normalize_model_id_or_path_for_hf_gguf, @@ -22,6 +23,23 @@ ) +def test_get_hf_config_dtype_avoids_modern_deprecated_alias(): + class ModernConfig: + dtype = None + + @property + def torch_dtype(self): + raise AssertionError("deprecated torch_dtype alias should not be read") + + assert get_hf_config_dtype(ModernConfig()) is None + + +def test_get_hf_config_dtype_preserves_stored_legacy_value(): + config = SimpleNamespace(dtype=None, torch_dtype="float16") + + assert get_hf_config_dtype(config) is torch.float16 + + def test_normalize_hf_config_compat_backfills_llama_rope_parameters(): config = LlamaConfig(rope_parameters=None, rope_theta=12345.0) @@ -238,6 +256,9 @@ def get_mask_sizes(self, query_length): def get_seq_length(self): return self._seq_length + def get_max_length(self): + return self._max_cache_shape + def get_max_cache_shape(self): return self._max_cache_shape @@ -245,6 +266,10 @@ class DummyCache(cache_utils.Cache): def __init__(self, seq_length, max_cache_shape): self.layers = [DummyLayer(seq_length, max_cache_shape)] + class EmptyCache(cache_utils.Cache): + def __init__(self): + self.layers = [] + normalize_hf_config_compat(SimpleNamespace(), trust_remote_code=True) limited_cache = DummyCache(seq_length=8, max_cache_shape=10) @@ -254,6 +279,7 @@ def __init__(self, seq_length, max_cache_shape): assert limited_cache.get_usable_length(4) == 6 assert dynamic_cache.get_max_length() is None assert dynamic_cache.get_usable_length(4) == 8 + assert EmptyCache().get_max_length() is None def test_normalize_hf_config_compat_restores_legacy_dynamic_cache_converters(monkeypatch):