From 249a5c9e9fa017dc2fc07868c51e56302bdc0c95 Mon Sep 17 00:00:00 2001 From: Daniel Zautner Date: Wed, 25 Mar 2026 13:20:56 +0200 Subject: [PATCH] Add --load-tasks-multilingual flag and fix --custom-tasks for inspect backend The inspect backend hardcoded load_multilingual=False and custom_tasks=None, making it impossible to use multilingual tasks or custom task files. Wire both CLI parameters through to the Registry, consistent with all other backends (vllm, accelerate, sglang, endpoint, etc). --- src/lighteval/main_inspect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py index 4aa5cbf87..1b8fb8db4 100644 --- a/src/lighteval/main_inspect.py +++ b/src/lighteval/main_inspect.py @@ -37,6 +37,7 @@ from typer import Argument, Option from typing_extensions import Annotated +from lighteval.cli_args import load_tasks_multilingual as load_tasks_multilingual_arg from lighteval.models.abstract_model import InspectAIModelConfig from lighteval.tasks.lighteval_task import LightevalTaskConfig @@ -432,10 +433,11 @@ def eval( # noqa C901 rich_help_panel=HELP_PANEL_NAME_4, ), ] = False, + load_tasks_multilingual: load_tasks_multilingual_arg.type = load_tasks_multilingual_arg.default, ): from lighteval.tasks.registry import Registry - registry = Registry(tasks=tasks, custom_tasks=None, load_multilingual=False) + registry = Registry(tasks=tasks, custom_tasks=custom_tasks, load_multilingual=load_tasks_multilingual) task_configs = registry.task_to_configs inspect_ai_tasks = []