Skip to content

Commit bbd8964

Browse files
committed
only warn if memory is not trival
1 parent 03108d9 commit bbd8964

1 file changed

Lines changed: 34 additions & 9 deletions

File tree

‎TPTBox/segmentation/VibeSeg/inference_nnunet.py‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -267,13 +267,6 @@ def run_inference_on_file(
267267
if "memory_factor" not in ds_info:
268268
missing_mem_keys.append("memory_factor")
269269
memory_factor = float(ds_info.get("memory_factor", 160))
270-
if missing_mem_keys:
271-
_suggest_memory_estimation_script(
272-
idx,
273-
model_path,
274-
f"Memory parameter(s) {missing_mem_keys} not set in the model's dataset.json; falling back to defaults. {memory_base=}, {memory_factor=}",
275-
logger=logger,
276-
)
277270

278271
use_folds_arg = tuple(folds) if len(folds) != 5 else None
279272
# Include every setting that changes the loaded predictor so a cache hit is always equivalent
@@ -362,11 +355,43 @@ def run_inference_on_file(
362355
if padd != 0:
363356
p = (padd, padd)
364357
input_nii = [i.apply_pad([p, p, p], mode="reflect") for i in input_nii]
358+
359+
def _defaults_fit_gpu(memory: float, gpu: int | None, safety_factor: float = 2.0) -> bool:
360+
"""Whether the fallback ``memory_base`` reservation is a small fraction of GPU memory.
361+
362+
Returns ``True`` when ``memory_base * safety_factor <= total_gpu_memory_mb`` —
363+
i.e. the fallback reservation leaves ample headroom for tile scheduling and
364+
the missing ``memory_base``/``memory_factor`` defaults are safe to use
365+
without warning. Returns ``False`` on CUDA-unavailable systems or when GPU
366+
memory can't be probed, so the load-time warning still fires on
367+
memory-constrained setups (CPU-only, small GPUs, or driver errors).
368+
"""
369+
try:
370+
import torch
371+
372+
if not torch.cuda.is_available():
373+
return False
374+
device = torch.device(f"cuda:{gpu}") if gpu is not None else torch.device("cuda:0")
375+
_, total_bytes = torch.cuda.mem_get_info(device)
376+
total_mb = total_bytes / (1024**2)
377+
except Exception: # noqa: BLE001
378+
return False
379+
return memory * safety_factor <= total_mb
380+
381+
num_classes = int(nnunet.label_manager.num_segmentation_heads)
382+
est_full_mb = estimate_peak_ram_mb(input_nii[0].shape, num_classes, len(input_nii))
383+
# Only warn if memory requirement is non-trivial.
384+
if missing_mem_keys and not _defaults_fit_gpu(est_full_mb, gpu):
385+
_suggest_memory_estimation_script(
386+
idx,
387+
model_path,
388+
f"Memory parameter(s) {missing_mem_keys} not set in the model's dataset.json; falling back to defaults. {memory_base=}, {memory_factor=}",
389+
logger=logger,
390+
)
391+
365392
if _cpu_chunks is None or _cpu_chunks <= 1:
366-
num_classes = int(nnunet.label_manager.num_segmentation_heads)
367393
total_ram_mb = _get_total_ram_mb()
368394
target_ram_mb = total_ram_mb * 0.5
369-
est_full_mb = estimate_peak_ram_mb(input_nii[0].shape, num_classes, len(input_nii))
370395
if est_full_mb > target_ram_mb:
371396
shape = input_nii[0].shape
372397
split_axis = int(np.argmax(shape))

0 commit comments

Comments
 (0)