diff --git a/garak/harnesses/base.py b/garak/harnesses/base.py index aac2c98c9..5dbae02be 100644 --- a/garak/harnesses/base.py +++ b/garak/harnesses/base.py @@ -191,11 +191,15 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None: attempt_results, (list, types.GeneratorType) ), "probing should always return an ordered iterable" + probe_display_name = probe.probename.replace("garak.", "") for d in detectors: logging.debug("harness: run detector %s", d.detectorname) attempt_iterator = tqdm.tqdm(attempt_results, leave=False) detector_probe_name = d.detectorname.replace("garak.detectors.", "") - attempt_iterator.set_description("detectors." + detector_probe_name) + # issue #324: include probe name in the detector progress bar + attempt_iterator.set_description( + f"{probe_display_name}/detectors.{detector_probe_name}" + ) for attempt in attempt_iterator: if d.skip: continue diff --git a/tests/harnesses/test_harnesses.py b/tests/harnesses/test_harnesses.py index fee10ecac..faaece570 100644 --- a/tests/harnesses/test_harnesses.py +++ b/tests/harnesses/test_harnesses.py @@ -1,11 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import pytest import importlib +import tempfile + +import pytest -from garak import _plugins +from garak import _plugins, _config +import garak.buffs.base import garak.harnesses.base HARNESSES = [ @@ -13,6 +16,22 @@ ] +@pytest.fixture(autouse=True) +def _config_loaded(): + """Load base config + a temp report file so harness.run() can write.""" + _config.load_base_config() + _config.plugins.probes["test"]["generations"] = 1 + temp_report_file = tempfile.NamedTemporaryFile( + mode="w+", suffix=".report.jsonl", delete=False + ) + _config.transient.report_filename = temp_report_file.name + _config.transient.reportfile = open( + _config.transient.report_filename, "w", buffering=1, encoding="utf-8" + ) + + yield + + @pytest.mark.parametrize("classname", HARNESSES) def test_harness_structure(classname): @@ -47,3 +66,42 @@ def test_harness_modality_match(): assert garak.harnesses.base._modality_match(t, tvi, False) is True assert garak.harnesses.base._modality_match(ti, tvi, False) is True assert garak.harnesses.base._modality_match(t, ti, False) is True + + +def test_harness_detector_progress_shows_probe_name(mocker, monkeypatch): + """Detector progress bar must show which probe is being processed (#324).""" + mocker.patch("garak.harnesses.base._initialize_runtime_services") + + captured_descriptions = [] + real_tqdm = garak.harnesses.base.tqdm.tqdm + + def capturing_tqdm(*args, **kwargs): + bar = real_tqdm(*args, **kwargs) + original_set_description = bar.set_description + + def set_description(desc, *a, **kw): + captured_descriptions.append(desc) + return original_set_description(desc, *a, **kw) + + bar.set_description = set_description + return bar + + monkeypatch.setattr(garak.harnesses.base.tqdm, "tqdm", capturing_tqdm) + + harness = garak.harnesses.base.Harness() + model = _plugins.load_plugin("generators.test.Blank") + probe = _plugins.load_plugin("probes.test.Blank") + detector = _plugins.load_plugin("detectors.always.Pass") + monkeypatch.setattr( + _config.buffmanager, + "buffs", + [garak.buffs.base.Buff()], + ) + + harness.run(model, [probe], [detector], mocker.Mock()) + + assert captured_descriptions, "detector progress bar should have a description" + assert any( + "probes.test.Blank" in desc and "detectors.always.Pass" in desc + for desc in captured_descriptions + ), "detector progress description should include probe and detector names"