Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 1 addition & 41 deletions ui/litellm-dashboard/eslint-suppressions.json
Original file line number Diff line number Diff line change
Expand Up @@ -336,9 +336,6 @@
}
},
"src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx": {
"no-restricted-imports": {
"count": 1
},
"react-hooks/set-state-in-effect": {
"count": 1
}
Expand All @@ -351,9 +348,6 @@
"src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx": {
"no-nested-ternary": {
"count": 3
},
"no-restricted-imports": {
"count": 1
}
},
"src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx": {
Expand Down Expand Up @@ -1486,16 +1480,10 @@
},
"src/app/(dashboard)/projects/_components/ProjectDetailsPage.tsx": {
"no-nested-ternary": {
"count": 3
},
"no-restricted-imports": {
"count": 1
"count": 2
}
},
"src/app/(dashboard)/projects/_components/ProjectKeysSection.tsx": {
"no-restricted-imports": {
"count": 1
},
"react-hooks/set-state-in-effect": {
"count": 1
}
Expand Down Expand Up @@ -1526,11 +1514,6 @@
"count": 2
}
},
"src/app/(dashboard)/projects/_components/ProjectsPage.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/app/(dashboard)/prompts/_components/add_prompt_form.tsx": {
"local/filename-pascal-case": {
"count": 1
Expand Down Expand Up @@ -2109,16 +2092,6 @@
"count": 1
}
},
"src/components/DeletedKeysPage/DeletedKeysPage.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/DeletedTeamsPage/DeletedTeamsPage.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/DeprecationBanner.tsx": {
"no-restricted-imports": {
"count": 1
Expand Down Expand Up @@ -3827,11 +3800,6 @@
"count": 1
}
},
"src/components/ui/AntDLoadingSpinner.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/ui/alert-dialog.tsx": {
"local/filename-pascal-case": {
"count": 1
Expand Down Expand Up @@ -4040,11 +4008,6 @@
"count": 1
}
},
"src/components/view_logs/AuditLogDrawer/AuditLogDrawer.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/view_logs/CostBreakdownViewer.tsx": {
"no-restricted-imports": {
"count": 1
Expand Down Expand Up @@ -4207,9 +4170,6 @@
"src/components/view_logs/index.tsx": {
"local/filename-pascal-case": {
"count": 1
},
"no-restricted-imports": {
"count": 1
}
},
"src/components/view_logs/log_filter_logic.tsx": {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import userEvent from "@testing-library/user-event";
import { render, screen, waitFor } from "@testing-library/react";
import { EvaluationSettingsModal } from "./EvaluationSettingsModal";

const mockFetchAvailableModels = vi.fn();
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: (...args: unknown[]) => mockFetchAvailableModels(...args),
}));

const modelGroups = [{ model_group: "gpt-5.2" }, { model_group: "claude-sonnet-5" }];

const defaultProps = {
open: true,
onClose: vi.fn(),
guardrailName: "pii-detector",
accessToken: "test-token",
onRunEvaluation: vi.fn(),
};

async function selectModel(user: ReturnType<typeof userEvent.setup>, label: string) {
await user.click(screen.getByRole("combobox"));
const options = await screen.findAllByText(label);
await user.click(options[options.length - 1]);
}

describe("EvaluationSettingsModal", () => {
beforeEach(() => {
vi.clearAllMocks();
mockFetchAvailableModels.mockResolvedValue(modelGroups);
});

it("should render nothing while closed", () => {
render(<EvaluationSettingsModal {...defaultProps} open={false} />);
expect(screen.queryByText("Evaluation Settings")).not.toBeInTheDocument();
});

it("should show the title and the guardrail-specific description when open", () => {
render(<EvaluationSettingsModal {...defaultProps} />);
expect(screen.getByText("Evaluation Settings")).toBeInTheDocument();
expect(screen.getByText("Configure AI evaluation for pii-detector")).toBeInTheDocument();
});

it("should fall back to a generic description when no guardrail name is given", () => {
render(<EvaluationSettingsModal {...defaultProps} guardrailName={undefined} />);
expect(screen.getByText("Configure AI evaluation for re-running on logs")).toBeInTheDocument();
});

it("should prefill the prompt and the response schema with their defaults", () => {
render(<EvaluationSettingsModal {...defaultProps} />);
expect(screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/)).toBeInTheDocument();
expect(
screen.getByDisplayValue(/"verdict": "correct" \| "false_positive" \| "false_negative"/),
).toBeInTheDocument();
});

it("should restore the default prompt when 'Reset to default' is clicked", async () => {
const user = userEvent.setup();
render(<EvaluationSettingsModal {...defaultProps} />);

const promptBox = screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/);
await user.clear(promptBox);
await user.type(promptBox, "custom prompt");
expect(screen.getByDisplayValue("custom prompt")).toBeInTheDocument();

await user.click(screen.getByText("Reset to default"));
expect(screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/)).toBeInTheDocument();
});

it("should load the available models with the access token when opened", async () => {
render(<EvaluationSettingsModal {...defaultProps} />);
await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalledWith("test-token"));
});

it("should not load models when there is no access token", () => {
render(<EvaluationSettingsModal {...defaultProps} accessToken={null} />);
expect(mockFetchAvailableModels).not.toHaveBeenCalled();
});

it("should not run an evaluation while no model is selected", async () => {
const user = userEvent.setup();
const onRunEvaluation = vi.fn();
const onClose = vi.fn();
render(<EvaluationSettingsModal {...defaultProps} onRunEvaluation={onRunEvaluation} onClose={onClose} />);

await user.click(screen.getByRole("button", { name: /run evaluation/i }));

expect(onRunEvaluation).not.toHaveBeenCalled();
expect(onClose).not.toHaveBeenCalled();
});

it("should run the evaluation with the selected model and the current prompt and schema", async () => {
const user = userEvent.setup();
const onRunEvaluation = vi.fn();
const onClose = vi.fn();
render(<EvaluationSettingsModal {...defaultProps} onRunEvaluation={onRunEvaluation} onClose={onClose} />);

await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalled());
await selectModel(user, "claude-sonnet-5");
await user.click(screen.getByRole("button", { name: /run evaluation/i }));

expect(onRunEvaluation).toHaveBeenCalledWith({
model: "claude-sonnet-5",
prompt: expect.stringContaining("Evaluate whether this guardrail's decision was correct"),
schema: expect.stringContaining('"verdict"'),
});
expect(onClose).toHaveBeenCalled();
});

it("should close without running when 'Cancel' is clicked", async () => {
const user = userEvent.setup();
const onRunEvaluation = vi.fn();
const onClose = vi.fn();
render(<EvaluationSettingsModal {...defaultProps} onRunEvaluation={onRunEvaluation} onClose={onClose} />);

await user.click(screen.getByRole("button", { name: /cancel/i }));

expect(onClose).toHaveBeenCalled();
expect(onRunEvaluation).not.toHaveBeenCalled();
});
});
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
import { CloseOutlined, PlayCircleOutlined } from "@ant-design/icons";
import { Button, Modal, Select, Input } from "antd";
import React, { useEffect, useState } from "react";
import { Play } from "lucide-react";
import React, { useEffect, useMemo, useState } from "react";
import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models";
import { SearchSelect } from "@/components/shared/SearchSelect";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Textarea } from "@/components/ui/textarea";

const DEFAULT_PROMPT = `Evaluate whether this guardrail's decision was correct.
Analyze the user input, the guardrail action taken, and determine if it was appropriate.
Expand Down Expand Up @@ -73,79 +83,81 @@ export function EvaluationSettingsModal({
}
};

const modelSelectOptions = modelOptions.map((m) => ({
value: m.model_group,
label: m.model_group,
}));
const modelSelectOptions = useMemo(
() => modelOptions.map((m) => ({ value: m.model_group, label: m.model_group })),
[modelOptions],
);

return (
<Modal
title="Evaluation Settings"
open={open}
onCancel={onClose}
width={640}
footer={null}
closeIcon={<CloseOutlined />}
destroyOnClose
>
<p className="text-sm text-gray-500 mb-4">
{guardrailName
? `Configure AI evaluation for ${guardrailName}`
: "Configure AI evaluation for re-running on logs"}
</p>
<Dialog open={open} onOpenChange={(nextOpen) => !nextOpen && onClose()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[640px]">
<DialogHeader>
<DialogTitle>Evaluation Settings</DialogTitle>
<DialogDescription>
{guardrailName
? `Configure AI evaluation for ${guardrailName}`
: "Configure AI evaluation for re-running on logs"}
</DialogDescription>
</DialogHeader>

<div className="space-y-4">
<div>
<div className="flex items-center justify-between mb-1.5">
<label className="text-sm font-medium text-gray-700">Evaluation Prompt</label>
<button type="button" onClick={handleResetPrompt} className="text-xs text-indigo-600 hover:text-indigo-700">
Reset to default
</button>
<div className="space-y-4">
<div>
<div className="mb-1.5 flex items-center justify-between">
<label htmlFor="evaluation-prompt" className="text-sm font-medium text-foreground">
Evaluation Prompt
</label>
<Button variant="link" size="xs" onClick={handleResetPrompt}>
Reset to default
</Button>
</div>
<Textarea
id="evaluation-prompt"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
rows={6}
className="field-sizing-fixed font-mono text-sm"
/>
<p className="mt-1 text-xs text-muted-foreground">
System prompt sent to the evaluation model. Output is structured via response_format.
</p>
</div>
<Input.TextArea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
rows={6}
className="font-mono text-sm"
/>
<p className="text-xs text-gray-400 mt-1">
System prompt sent to the evaluation model. Output is structured via response_format.
</p>
</div>

<div>
<label className="block text-sm font-medium text-gray-700 mb-1.5">Response Schema</label>
<p className="text-xs text-gray-400 mb-1">response_format: json_schema</p>
<Input.TextArea
value={schema}
onChange={(e) => setSchema(e.target.value)}
rows={6}
className="font-mono text-sm"
/>
</div>
<div>
<label htmlFor="evaluation-schema" className="mb-1.5 block text-sm font-medium text-foreground">
Response Schema
</label>
<p className="mb-1 text-xs text-muted-foreground">response_format: json_schema</p>
<Textarea
id="evaluation-schema"
value={schema}
onChange={(e) => setSchema(e.target.value)}
rows={6}
className="field-sizing-fixed font-mono text-sm"
/>
</div>

<div>
<label className="block text-sm font-medium text-gray-700 mb-1.5">Model</label>
<Select
placeholder={loadingModels ? "Loading models…" : "Select a model"}
value={model ?? undefined}
onChange={setModel}
options={modelSelectOptions}
style={{ width: "100%" }}
showSearch
optionFilterProp="label"
loading={loadingModels}
notFoundContent={!accessToken ? "Sign in to see models" : "No models available"}
/>
<div>
<p className="mb-1.5 text-sm font-medium text-foreground">Model</p>
<SearchSelect
options={modelSelectOptions}
value={model ?? undefined}
onValueChange={(value) => setModel(value || null)}
placeholder={loadingModels ? "Loading models…" : "Select a model"}
emptyText={!accessToken ? "Sign in to see models" : "No models available"}
/>
</div>
</div>
</div>

<div className="flex items-center justify-end gap-2 mt-6 pt-4 border-t border-gray-100">
<Button onClick={onClose}>Cancel</Button>
<Button type="primary" icon={<PlayCircleOutlined />} onClick={handleRun} disabled={!model}>
Run Evaluation
</Button>
</div>
</Modal>
<DialogFooter className="border-t border-border pt-4">
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
<Button onClick={handleRun} disabled={!model}>
<Play className="size-4" />
Run Evaluation
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
Loading
Loading