Skip to content

fix: print the failing output text in verbose evaluator results #1975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkzung wants to merge 2 commits into
base: main
Choose a base branch
from
+79 −4
Open

fix: print the failing output text in verbose evaluator results #1975

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ffd8c17
fix: print the failing output text in verbose evaluator results
mkzung Jul 18, 2026
7e4d431
fix: escape control characters in verbose evaluator output
mkzung Jul 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions garak/evaluators/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,11 @@ def print_results_wide(

if _config.system.verbose > 0 and messages:
for m in messages:
if m is None or m.text is None:
continue
try:
print("❌", m.strip().replace("\n", " "))
except:
print("❌", m.text.strip().replace("\n", " "))
except Exception:
pass

def print_results_narrow(
Expand Down Expand Up @@ -380,9 +382,11 @@ def print_results_narrow(

if _config.system.verbose > 0 and messages:
for m in messages:
if m is None or m.text is None:
continue
try:
print("❌", m.strip().replace("\n", " "))
except:
print("❌", m.text.strip().replace("\n", " "))
except Exception:
pass


Expand Down
16 changes: 16 additions & 0 deletions tests/evaluators/test_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@

import pytest

import garak._config
import garak.analyze
import garak.attempt
import garak.evaluators.base


Expand Down Expand Up @@ -39,3 +41,17 @@ def test_get_z_rating_none_zscore():
returned_z, symbol = ev.get_z_rating("probe.Probe", "detector.Detector", 50)
assert returned_z is None
assert symbol == ""


@pytest.mark.parametrize(
"print_func_name", ["print_results_wide", "print_results_narrow"]
)
def test_print_results_prints_failing_output(capsys, monkeypatch, print_func_name):
# messages hold Message objects, so the text has to come from .text
monkeypatch.setattr(garak._config.system, "verbose", 1, raising=False)
monkeypatch.setattr(garak._config.system, "show_z", False, raising=False)
ev = garak.evaluators.base.Evaluator.__new__(garak.evaluators.base.Evaluator)
ev.probename = "probe.Probe"
messages = [garak.attempt.Message(text="unsafe\noutput"), None]
getattr(ev, print_func_name)("detector.Detector", 0, 1, messages)
assert "unsafe output" in capsys.readouterr().out
Loading