Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/lighteval/utils/parallelism.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import gc
import inspect
import logging
from collections.abc import Callable

import torch

Expand Down Expand Up @@ -54,14 +55,14 @@ def should_reduce_batch_size(exception: Exception) -> bool:
return False


def find_executable_batch_size(function: callable = None, starting_batch_size: int = 128):
def find_executable_batch_size(function: Callable[..., object] | None = None, starting_batch_size: int = 128):
"""A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or
CUDNN, the batch size is cut in half and passed to `function`

`function` must take in a `batch_size` parameter as its first argument.

Args:
function (`callable`, *optional*):
function (`Callable`, *optional*):
A function to wrap
starting_batch_size (`int`, *optional*):
The batch size to try and fit into memory
Expand Down