From 3b949cfc12092686ea30eb01f2fad599331e2375 Mon Sep 17 00:00:00 2001 From: Zipson Date: Thu, 5 Nov 2026 12:30:35 +0800 Subject: [PATCH] feat: fix inspect defect check level at L3 and drop --check-level flag The defect check depth is no longer user-configurable. Every inspect run now executes at the deep L3 level: - Remove the --check-level option and its L1/L2/L3 validation from the inspect command - Add a DEFAULT_CHECK_LEVEL constant (L3) used by both the defect check and the saved markdown report - Change DefectChecker.check and save_markdown_report defaults from L2 to L3 - Update README: remove check-level examples and the Check Levels table - Add regression tests covering the removed flag and the fixed L3 level - Record the breaking change in CHANGELOG --- CHANGELOG.md | 2 + README.md | 18 +---- src/sanityops_cli/commands/inspect.py | 16 ++--- src/sanityops_cli/defect_checker/checker.py | 2 +- .../defect_checker/markdown_reporter.py | 2 +- tests/unit/commands/test_inspect.py | 67 +++++++++++++++++++ tests/unit/defect_checker/test_checker.py | 4 +- .../defect_checker/test_markdown_reporter.py | 2 +- 8 files changed, 81 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cdbae1..408eeaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking**: `inspect` no longer accepts `--check-level`; the defect check + always runs at the deep L3 level - **Breaking**: tool schema uploads now emit the provider-neutral flat format `{name, description, parameters}` in both upload paths (push and inspect). The OpenAI `{type, function}` wrapper is stripped during packing; the diff --git a/README.md b/README.md index a66bfa8..e12e0a8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A command-line tool for static defect inspection of AI logical artifacts (System ## Features -- **Multi-level inspection depth** — Choose between L1 (fast), L2 (standard), or L3 (deep) check levels +- **Deep inspection** — Runs a comprehensive L3 defect check on every artifact by default - **Multiple artifact types** — Inspect System Prompts, Skills, and Tool Schemas - **CI/CD friendly** — Non-interactive runs, configuration via environment variables, dated markdown reports for pipeline artifacts - **Configurable rules** — Customize inspection behavior via YAML configuration @@ -100,15 +100,9 @@ skills: ### 3. Run Inspection ```bash -# Standard inspection (L2 depth) +# Standard inspection (deep L3 defect check) sanityops-cli inspect -# Fast inspection (L1 depth) -sanityops-cli inspect --check-level L1 - -# Deep inspection (L3 depth) -sanityops-cli inspect --check-level L3 - # Skip defect check, only analyze artifacts sanityops-cli inspect --skip-defect-check @@ -203,14 +197,6 @@ CLI flags (`--timeout`, `--token-budget`) override these values when explicitly All CLI operations are logged to `~/.sanityops/logs/sanityops-cli-.log`. Logs include timestamps, log levels, and redacted API keys for security. -### Check Levels - -| Level | Description | Use Case | -|-------|-------------|----------| -| L1 | Fast | Quick validation during development | -| L2 | Standard | Default, balanced depth for regular use | -| L3 | Deep | Comprehensive analysis for production releases | - ## Documentation Full documentation is available at [sanityops.org](https://sanityops.org). diff --git a/src/sanityops_cli/commands/inspect.py b/src/sanityops_cli/commands/inspect.py index c1fb5f8..437019d 100644 --- a/src/sanityops_cli/commands/inspect.py +++ b/src/sanityops_cli/commands/inspect.py @@ -50,6 +50,9 @@ #: URL users attach log files to when reporting issues. ISSUE_URL = "https://github.com/sanityops-org/sanityops-cli/issues" +#: Defect check depth used for every inspection run (fixed, not user-configurable). +DEFAULT_CHECK_LEVEL = "L3" + def _load_agent_config(config_path: Path) -> dict: """Read the optional `agent:` section from inspect_config.yaml. @@ -479,11 +482,6 @@ def inspect( help="Path to inspect_config.yaml. If omitted, looks for .sanityops/inspect_config.yaml in cwd.", ), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), - check_level: str = typer.Option( - "L2", - "--check-level", - help="Defect check level: L1 (fast), L2 (standard), L3 (deep).", - ), skip_defect_check: bool = typer.Option( False, "--skip-defect-check", @@ -516,10 +514,6 @@ def inspect( if ctx.invoked_subcommand is not None: return - if check_level not in {"L1", "L2", "L3"}: - console.print(f"[red]✗ Invalid check level: {escape(check_level)} (must be L1/L2/L3)[/red]") - raise typer.Exit(code=EXIT_FAILURE) - logger = Logger() tracker = ProgressTracker(console, logger, verbose=verbose) @@ -674,7 +668,7 @@ def inspect( async def run_check(): checker = DefectChecker(llm_config) - return await checker.check(result, check_level=check_level) + return await checker.check(result, check_level=DEFAULT_CHECK_LEVEL) try: with tracker.step("Running defect check..."): @@ -693,7 +687,7 @@ async def run_check(): response, report_dir, project_id=project_id, - check_level=check_level, + check_level=DEFAULT_CHECK_LEVEL, qc_result=qc_result, ) except OSError as e: diff --git a/src/sanityops_cli/defect_checker/checker.py b/src/sanityops_cli/defect_checker/checker.py index aab5dde..1a26535 100644 --- a/src/sanityops_cli/defect_checker/checker.py +++ b/src/sanityops_cli/defect_checker/checker.py @@ -42,7 +42,7 @@ def __init__(self, llm_config: dict[str, str]): async def check( self, findings: FindingsResult, - check_level: str = "L2", + check_level: str = "L3", ) -> dict: """Run defect check and return the raw SDK response dict.""" tools = self._convert_tools(findings.tools) diff --git a/src/sanityops_cli/defect_checker/markdown_reporter.py b/src/sanityops_cli/defect_checker/markdown_reporter.py index 0921336..923bd60 100644 --- a/src/sanityops_cli/defect_checker/markdown_reporter.py +++ b/src/sanityops_cli/defect_checker/markdown_reporter.py @@ -380,7 +380,7 @@ def save_markdown_report( output_dir: Path, *, project_id: str | None = None, - check_level: str = "L2", + check_level: str = "L3", qc_result: QCInspectionResult | None = None, ) -> Path: """Persist a full markdown report and return the written file path. diff --git a/tests/unit/commands/test_inspect.py b/tests/unit/commands/test_inspect.py index 83d1237..d2692ed 100644 --- a/tests/unit/commands/test_inspect.py +++ b/tests/unit/commands/test_inspect.py @@ -335,3 +335,70 @@ def analyze_files_sync(self, prompts, tools, skills): i for i, line in enumerate(lines) if f"sanityops-cli v{__version__}" in line ) assert "Project ID: not set" in lines[banner_line_idx + 1] + + +def test_inspect_check_level_flag_removed(monkeypatch, tmp_path): + """--check-level is no longer an accepted option (fixed at L3).""" + monkeypatch.setenv("HOME", str(tmp_path)) + skill_file = tmp_path / "skill.md" + skill_file.write_text("---\nname: s\ndescription: d\n---\n# X\n") + cfg = tmp_path / "inspect_config.yaml" + cfg.write_text( + "project:\n id: 00000000-0000-0000-0000-000000000000\n" + f"skills:\n - file: {skill_file}\n" + ) + + class FakeAgent: + def analyze_files_sync(self, prompts, tools, skills): + return _findings() + + monkeypatch.setattr( + "sanityops_cli.commands.inspect.ScannerAgent", + lambda *a, **k: FakeAgent(), + ) + + result = runner.invoke(app, ["inspect", "--config", str(cfg), "--check-level", "L1"]) + assert result.exit_code != 0 + combined = result.stdout + (result.stderr or "") + assert "no such option" in combined.lower() + + +def test_inspect_runs_defect_check_at_l3(monkeypatch, tmp_path): + """The defect check always runs at the fixed L3 level and reports it.""" + monkeypatch.setenv("HOME", str(tmp_path)) + skill_file = tmp_path / "skill.md" + skill_file.write_text("---\nname: s\ndescription: d\n---\n# X\n") + cfg = tmp_path / "inspect_config.yaml" + cfg.write_text( + "project:\n id: 00000000-0000-0000-0000-000000000000\n" + f"skills:\n - file: {skill_file}\n" + ) + + class FakeAgent: + def analyze_files_sync(self, prompts, tools, skills): + return _findings() + + monkeypatch.setattr( + "sanityops_cli.commands.inspect.ScannerAgent", + lambda *a, **k: FakeAgent(), + ) + + captured = {} + + async def fake_check(self, findings, check_level="L2"): + captured["check_level"] = check_level + return { + "status": "completed", + "summary": {"total_defects": 0, "p0_count": 0, "p1_count": 0, "p2_count": 0, "gate_result": "PASS"}, + "results": [], + "errors": [], + "metadata": {}, + } + + monkeypatch.setattr( + "sanityops_cli.defect_checker.checker.DefectChecker.check", fake_check + ) + + result = runner.invoke(app, ["inspect", "--config", str(cfg)]) + assert result.exit_code == 0 + assert captured["check_level"] == "L3" diff --git a/tests/unit/defect_checker/test_checker.py b/tests/unit/defect_checker/test_checker.py index 8711c54..d4006d0 100644 --- a/tests/unit/defect_checker/test_checker.py +++ b/tests/unit/defect_checker/test_checker.py @@ -132,12 +132,12 @@ async def fake_check(tools, prompts, skills, **kwargs): "llm_base_url": "", } ) - await checker.check(result, check_level="L2") + await checker.check(result, check_level="L3") assert captured["skills"] == [{"id": str(skill_dir), "name": "s", "content": ""}] assert captured["tools"] == [] assert captured["prompts"] == [] - assert captured["kwargs"]["check_level"] == "L2" + assert captured["kwargs"]["check_level"] == "L3" assert captured["kwargs"]["llm_provider"] == "anthropic" assert captured["kwargs"]["llm_api_key"] == "k" assert captured["kwargs"]["llm_model_id"] == "m" diff --git a/tests/unit/defect_checker/test_markdown_reporter.py b/tests/unit/defect_checker/test_markdown_reporter.py index a0f5985..3297efc 100644 --- a/tests/unit/defect_checker/test_markdown_reporter.py +++ b/tests/unit/defect_checker/test_markdown_reporter.py @@ -94,7 +94,7 @@ def test_report_contains_summary_table(self, tmp_path): "gate_result": "FAIL", }, ) - path = save_markdown_report(response, tmp_path, check_level="L2") + path = save_markdown_report(response, tmp_path, check_level="L3") content = path.read_text(encoding="utf-8") assert "## Summary" in content assert "| Total defects | 5 |" in content