diff --git a/README.md b/README.md index 4268d8d..a66bfa8 100644 --- a/README.md +++ b/README.md @@ -120,23 +120,6 @@ sanityops-cli inspect --config path/to/config.yaml `inspect` exits `0` when the run completes — including when defects are found — and non-zero on failure. CI pipelines should therefore gate on the generated report (`.sanityops/results/inspect-*.md`), not on the exit code. -### 4. Generate Repairs (Optional) - -After inspection, generate repaired artifacts from the report: - -```bash -# Use the latest inspection report -sanityops-cli inspect repair - -# Use a specific report -sanityops-cli inspect repair --report .sanityops/results/inspect-20250115-120000.md - -# Specify timeout and token budget for large projects -sanityops-cli inspect repair --timeout 3600 --token-budget 500000 -``` - -Repairs are written to `.sanityops/repairs/repair-.md` without modifying source files. - ## Data Handling What leaves your machine, and when: diff --git a/src/sanityops_cli/agents/scanner_agent/models/finding.py b/src/sanityops_cli/agents/scanner_agent/models/finding.py index 454148a..43a4043 100644 --- a/src/sanityops_cli/agents/scanner_agent/models/finding.py +++ b/src/sanityops_cli/agents/scanner_agent/models/finding.py @@ -30,6 +30,7 @@ class FindingType(StrEnum): class Section(BaseModel): """A markdown section within a skill file.""" + title: str = Field(description="Section heading text") content: str = Field(description="Section body content") @@ -37,7 +38,9 @@ class Section(BaseModel): class ToolSchema(BaseModel): name: str description: str = "" - parameters: dict = Field(default_factory=dict,description="Tool parameters, JSON Schema (Draft-07) format") + parameters: dict = Field( + default_factory=dict, description="Tool parameters, JSON Schema (Draft-07) format" + ) class SkillContent(BaseModel): @@ -54,7 +57,11 @@ class SkillContent(BaseModel): #: Frontmatter fields that must be positive integers (QDS-0.4). LLM-based #: extraction can stringify numeric values, so coerce them back to int. _POSITIVE_INT_FIELDS: tuple[str, ...] = ( - "max_items", "max_chars", "timeout_seconds", "max_tool_calls", "max_retries", + "max_items", + "max_chars", + "timeout_seconds", + "max_tool_calls", + "max_retries", ) @model_validator(mode="after") @@ -100,23 +107,45 @@ def to_markdown(self) -> str: if not parts: return "" parts.append( - "\n\n".join( - f"## {section.title}\n\n{section.content}" - for section in self.sections - ) + "\n\n".join(f"## {section.title}\n\n{section.content}" for section in self.sections) ) return "\n".join(parts) class ToolContent(BaseModel): """Extracted content from a tool file.""" + name: str = Field(description="Tool name from definition") description: str = Field(description="Tool description") - parameters: dict[str, Any] = Field(default_factory=dict, description="JSON Schema of parameters") + parameters: dict[str, Any] = Field( + default_factory=dict, description="JSON Schema of parameters" + ) + #: pre-inspect-extract alignment: extraction fidelity labels. Default empty + #: when the analyzer does not provide them (backward compatible). + completeness: str = Field( + default="", + description=( + "Completeness level per pre-inspect-extract: full original / " + "assembled / incomplete / summary / not inspectable" + ), + ) + evidence: str = Field( + default="", description="Short verbatim excerpt from the source proving the extraction" + ) class PromptContent(BaseModel): content: str + #: pre-inspect-extract alignment: extraction fidelity labels. Default empty + #: when the analyzer does not provide them (backward compatible). + completeness: str = Field( + default="", + description=( + "Completeness level per pre-inspect-extract: full original / " + "original · variable unexpanded / assembled / incomplete / summary / not inspectable" + ), + ) + evidence: str = Field(default="", description="Short verbatim excerpt from the source") class Finding(BaseModel): diff --git a/src/sanityops_cli/agents/scanner_agent/prompts.py b/src/sanityops_cli/agents/scanner_agent/prompts.py index 4f742a6..acc13c7 100644 --- a/src/sanityops_cli/agents/scanner_agent/prompts.py +++ b/src/sanityops_cli/agents/scanner_agent/prompts.py @@ -21,15 +21,27 @@ - `SKILL.md` file (primary indicator), OR - `README.md` with skill-defining frontmatter/headings (`# Skill`, `# Workflow`) -## 2. Exclusion Rules (STRICT) +A skill may also be **carried inside code** (e.g. a `@skill` decorator, a skill class/definition embedded in a `.py`/`.ts`/`.json` file). In that case treat it like a tool/prompt carrier: strip the code shell and extract the skill definition verbatim; classify the file by what it actually contains. + +A skill is only inspectable when its definition carries the structural header — `name`/`description` frontmatter. A bare `## Skills` list or a bare function call that only *describes* a capability is documentation, not an artifact; skip it, do not inspect it. + +**Four-way classification** (before extracting, decide what the candidate is): +- **Artifact** — the Skill definition itself (with structural header). Send to inspection. +- **Carrier** — code/markup that wraps a skill definition (decorator, class, embedded dict). Strip the shell, extract verbatim. +- **Scaffolding** — metadata, `Evaluation Dimensions`, `Scoring Criteria`, `Test Suggestions`, `Notes`. Discard. +- **Reference file** — pure code or pure data that contains no extractable skill definition. Skip it — do not report it as a skill. + +## 2. Exclusion Rules (STRICT) - HIGHEST PRIORITY +These directories MUST be excluded from all search operations: - Hidden paths starting with `.` (e.g., `.git`, `.github`, `.vscode`, `.idea`, `.venv`, `.env`, `.cache`) - Common dependency & build output folders (e.g., `node_modules`, `vendor`, `dist`, `build`, `out`, `__pycache__`, `.egg-info`, `site-packages`) - Test and fixture directories (e.g., `tests/`, `__tests__/`, `fixtures/`, `mocks/`) - Irrelevant non-skill docs (e.g., root `README.md`, `CONTRIBUTING.md`, `LICENSE`, `CHANGELOG.md`, component/API docs) +- Scaffolding content: `Evaluation Dimensions`, `Scoring Criteria`, `Test Suggestions`, `Notes` (metadata describing a skill, not the skill itself) -## 3. Search Strategy -1. Use `glob` to find `**/SKILL.md` within max depth `{depth}` -2. Verify README.md for skill markers +## 3. Search Strategy (Exclusions applied at each step) +1. Use `glob` to find `**/SKILL.md` within max depth `{depth}` - automatically exclude paths matching Section 2 rules +2. Verify README.md for skill markers - skip excluded directories 3. Confirm directory structure ## 4. Reporting via store_findings @@ -53,6 +65,15 @@ TOOL_FINDER_RULES = """\ You are an expert tool discovery agent. Find all **tools** in `{directory}` and report each one immediately via `store_findings`. +## 0. Artifact vs Carrier vs Scaffolding vs Reference File (STRICT) +Before extracting a tool, classify what you see: +- **Artifact** — the Tool Schema itself (JSON Schema / function-calling definition with `name` + `description` + `parameters`/`inputSchema`). Send to inspection. +- **Carrier** — code or markup that wraps a tool definition (e.g. `def f(...): return ...`, a `@tool` decorator, a `Tool(...)` constructor call, backtick strings). A carrier is NOT a tool; strip the shell and extract the schema inside. +- **Scaffolding** — metadata, Evaluation Dimensions, Scoring Criteria, Test Suggestions, Notes, example usage, registration lists. Discard — never report scaffolding as a tool. +- **Reference file** — pure code or pure data that contains no extractable tool schema (e.g. a utility module, a data file). Skip it — do not report it as a tool. + +Do NOT enumerate syntax case by case: `Tool(...)`, `@tool`, `BaseTool`, backticks and `"..."` differ only in which shell to strip, not in whether a tool is present. + ## 1. Framework-Aware Search Patterns Identify tools by recognizing framework-specific patterns: @@ -73,16 +94,45 @@ - OpenAPI specs: `"type": "function"` with `operationId` - JSON/YAML tool configs: Anthropic `"input_schema"`, OpenAI `"parameters"` -## 2. Exclusion Rules (STRICT) +## 1.5 Extraction Tiers (pre-inspect-extract alignment) +For each candidate tool, determine which tier it falls into and state the basis: + +- **Tier 1 — directly readable**: the schema exists in loadable form — a complete + JSON Schema inside ``` fences, or a `Tool(name=..., description=..., inputSchema=)` + where `inputSchema` is already a JSON Schema dict, or a pure JSON Schema. + → **verbatim copy**: take the schema as-is, drop only the surrounding shell. +- **Tier 2 — wrapped in code**: the schema is complete but carried inside code + (a dict literal in a function, a `.parameters`/`input_schema` object property). + → **verbatim copy**: extract the literal, unescape, keep variables as-is. +- **Tier 3 — only code, no ready-made schema**: constraints are written in + chained calls (Zod `z.object(...)`), decorators with type annotations, + or builder patterns (`mcp.NewTool(...)` + `WithString/...`). + → **assembly**: copy field-by-field everything already literal in the source + (`name`, `description`, each parameter key + description, `enum` values, + the `required` list, `default` values). **Add no field absent from the source.** + If a constraint cannot be converted (e.g. `.refine()`), do not fabricate an + equivalent — note it as not representable. + **The `description` field must not be lost**: description / `.describe()` / + docstring is the core basis of Tool Schema inspection and is the easiest thing + to lose during assembly. If the source has a description but the result does + not, label the whole thing `assembled` and record "description field at risk + of loss". If the source has no description, the result must not have one either + — do not write one. +- **Tier X — unrecognizable**: no structural signature, dynamic registration + with no literal schema, or only a registration list without `inputSchema`/`parameters`. + → label **not inspectable**, record the reason, and do not force a verdict. + +## 2. Exclusion Rules (STRICT) - HIGHEST PRIORITY +These directories MUST be excluded from all search operations: - Hidden paths starting with `.` (e.g., `.git`, `.github`, `.vscode`, `.idea`, `.venv`, `.env`, `.cache`) - Common dependency & build output folders (e.g., `node_modules`, `vendor`, `dist`, `build`, `out`, `__pycache__`, `.egg-info`, `site-packages`) - Test and fixture directories (e.g., `tests/`, `__tests__/`, `fixtures/`, `mocks/`) - Irrelevant non-tool docs (e.g., root `README.md`, `CONTRIBUTING.md`, `LICENSE`, `CHANGELOG.md`, component/API docs) -## 3. Search Strategy -1. **Pattern-based discovery**: Use `glob` for `*tool*`, `*function*`, `*schema*` -2. **Content search**: Use `grep`/`rg` for framework markers (e.g., `@tool`, `BaseTool`, `input_schema`) -3. **Extract schema**: Read file, extract tool name, description, and parameter structure +## 3. Search Strategy (Exclusions applied at each step) +1. **Pattern-based discovery**: Use `glob` for `*tool*`, `*function*`, `*schema*` - automatically exclude paths matching Section 2 rules +2. **Content search**: Use `grep`/`rg` for framework markers (e.g., `@tool`, `BaseTool`, `input_schema`) - skip excluded directories +3. **Extract schema**: Read file, classify the tool (Section 1.5), extract tool name, description, and parameter structure, and record the completeness level ## 4. Reporting via store_findings For EACH tool found, immediately call: @@ -95,21 +145,45 @@ "content": {{ "name": "", "description": "", - "parameters": {{}} + "parameters": {{}}, + "completeness": "", + "evidence": "" }} }} }}) **CRITICAL**: Report immediately, do NOT wait to collect all tools. Each finding = one tool call. +- **Completeness labels**: `full original` = verbatim schema; `assembled` = rebuilt from code; + `incomplete` = missing base fields (e.g. no schema, only name/description); + `summary` = cannot be matched to the original in full; `not inspectable` = cannot be extracted. +- **Evidence**: quote a short verbatim excerpt of the source (head/tail fragment for verbatim + copies, per-field source fragments for assembled schemas). A reader must be able to find + every point in the source using only the evidence. + ## 5. Output Format -Return summary when done: "Completed: found N tools in {directory}" +Return summary when done as JSON: {{"directory": "{directory}", "tools_found": , "not_inspectable": , "status": "completed"}} """ PROMPT_FINDER_RULES = """\ You are a prompt discovery agent. Your objective is to find all **LLM prompts** within `{directory}` by analyzing standalone prompt files and reverse-tracing LLM call sites in the codebase. Report each finding immediately via `store_findings`. -## 1. Search Strategy (Execute in Sequence) +## 0. Artifact vs Carrier vs Scaffolding vs Reference File (STRICT) +Before extracting a prompt, classify what you see: +- **Artifact** — the System Prompt text itself (prose/markdown instructions for the LLM). Send to inspection. +- **Carrier** — code or markup that wraps a prompt (a string constant, an f-string triple-quoted block, backtick template strings, a `return "..."` in a function). A carrier is NOT a prompt; strip the shell and copy the string content verbatim. +- **Scaffolding** — non-prompt content: Metadata, `Evaluation Dimensions`, `Scoring Criteria`, `Test Suggestions`, `Notes`, `Source Profile` sections, or registration lists. Discard — never report scaffolding as a prompt. +- **Reference file** — pure code or pure data that contains no extractable prompt (e.g. a utility module, a data file). Skip it — do not report it as a prompt. + +**Verbatim copy rule**: copy the prompt word-for-word; do not rewrite, polish, or complete it. Drop only the shell (assignment statement, function definition line, `return`, fence markers, backticks). Unescape (`\\\\n` → newline); keep variables as-is, do not substitute them. + +## 0.5 Variable examination (STRICT) +For every prompt with variables, examine each placeholder: +- **Runtime data** (e.g. `{{local_tz}}`, `{{date.today()}}`, `{{context}}`) → keep as-is, label **full original**. +- **A variable that expands into a rule sentence** (e.g. `{{reference_prompt}}`, `{{tone_prompt}}`) → label **original · variable unexpanded**, and state which variable and how many instructions it expands into. +A rule written inside a variable is absent from the literal string. If labeled "full original", checks like "reference specification not declared" would wrongly fail even though the source did declare it. + +## 1. Search Strategy (Execute in Sequence, Exclusions applied at each step) ### Track A: Reverse Call-Site Tracing (Primary) 1. **Locate LLM Invocation Sites**: `grep` for common LLM call methods across the codebase: @@ -127,11 +201,13 @@ - Variable names: `SYSTEM_PROMPT`, `PROMPT_TEMPLATE`, `USER_PROMPT_FMT`, `DEFAULT_PROMPT` - String markers: `You are a`, `Act as a`, `System Instructions:`, `<>` -## 2. Exclusion Rules (STRICT) +## 2. Exclusion Rules (STRICT) - HIGHEST PRIORITY +These directories MUST be excluded from all search operations: - Hidden paths starting with `.` (e.g., `.git`, `.github`, `.vscode`, `.idea`, `.venv`, `.env`, `.cache`) - Common dependency & build output folders (e.g., `node_modules`, `vendor`, `dist`, `build`, `out`, `__pycache__`, `.egg-info`, `site-packages`) - Test and fixture directories (e.g., `tests/`, `__tests__/`, `fixtures/`, `mocks/`) - Irrelevant non-prompt docs (e.g., root `LICENSE`) +- Scaffolding sections: `Evaluation Dimensions`, `Scoring Criteria`, `Test Suggestions`, `Notes`, `Source Profile` (metadata, not prompts) ## 3. Reporting via store_findings For EACH prompt found (whether standalone or traced from a call site), call `store_findings` IMMEDIATELY: @@ -142,18 +218,27 @@ "type": "prompt", "relative": "", "content": {{ - "content": "" + "content": "", + "completeness": "", + "evidence": "" }} }} }}) **CRITICAL RULES**: - For traced prompts, extract the actual prompt content/template defined before the call, NOT just the `client.create()` line itself. +- **Completeness labels**: `full original` = verbatim copy, complete; `original · variable unexpanded` = verbatim copy with unexpanded rule variables (list them); `incomplete` = missing base fields / assembly logic not provided; `summary` = summarized or transcribed, cannot be matched to the original in full; `not inspectable` = cannot be extracted. +- **Evidence**: quote a short verbatim excerpt (head/tail fragment) of the source. A reader must be able to find every point in the original using only the evidence. - Report immediately upon discovery. ## 4. Output Format -When all searches and tracings are complete, return: -"Completed: found N prompts in {directory}" +When all searches and tracings are complete, return a JSON summary with this exact structure: +{{ + "directory": "{directory}", + "prompts_found": , + "not_inspectable": , + "status": "completed" +}} """ INSPECT_PARENT_PROMPT = """\ @@ -213,6 +298,8 @@ }} }} }}) +``` +- A skill is only inspectable when its definition carries the `name`/`description` structural header. A markdown file that only *describes* skills (e.g. a `## Skills` list) is scaffolding, not a skill artifact — skip it. **CRITICAL**: Report each file immediately after analysis. Do not batch. @@ -250,13 +337,17 @@ "content": {{ "name": "", "description": "", - "parameters": {{}} + "parameters": {{}}, + "completeness": "", + "evidence": "" }} }} }}) **CRITICAL**: Report each file immediately after analysis. Do not batch. +- **Completeness labels**: `full original` = verbatim schema; `assembled` = rebuilt from code (Zod/decorator/builder); `incomplete` = missing base fields (e.g. only name/description, no schema); `summary` = cannot be matched to the original in full; `not inspectable` = cannot be extracted. When a schema is assembled, attach the source fragment for each field. + ## Completion Return: "Completed: analyzed N tools" """ @@ -281,13 +372,18 @@ "type": "prompt", "relative": "", "content": {{ - "content": "" + "content": "", + "completeness": "", + "evidence": "" }} }} }}) **CRITICAL**: Report each file immediately after analysis. Do not batch. The prompt `content` you report is the artifact that gets inspected — a truncated or summarized copy produces false defects. +- **Completeness labels**: `full original` = verbatim copy, complete; `original · variable unexpanded` = verbatim copy with unexpanded rule variables (list them); `assembled` = rebuilt from code fragments; `incomplete` = missing base fields; `summary` = summarized or transcribed; `not inspectable` = cannot be extracted. +- **Variable examination**: for every placeholder, distinguish runtime data (`{{question}}`) from a variable that expands into rules (`{{reference_prompt}}`). The latter must be labeled `original · variable unexpanded`. + ## Completion Return: "Completed: analyzed N prompts" """ diff --git a/src/sanityops_cli/commands/inspect.py b/src/sanityops_cli/commands/inspect.py index 6c97ddf..c1fb5f8 100644 --- a/src/sanityops_cli/commands/inspect.py +++ b/src/sanityops_cli/commands/inspect.py @@ -277,7 +277,7 @@ def _write_repairs_markdown( # ============================================================================ -@inspect_app.command("repair") +# @inspect_app.command("repair") # Disabled: repair command is closed def inspect_repair( ctx: typer.Context, # B008: Typer requires function calls in argument defaults for its CLI option diff --git a/src/sanityops_cli/defect_checker/markdown_reporter.py b/src/sanityops_cli/defect_checker/markdown_reporter.py index f459812..0921336 100644 --- a/src/sanityops_cli/defect_checker/markdown_reporter.py +++ b/src/sanityops_cli/defect_checker/markdown_reporter.py @@ -51,21 +51,6 @@ def _esc_md_cell(text: str | None) -> str: return text.strip() -def _format_score(score: dict[str, Any] | None) -> str: - """Format score/grade/gate for a section header, or empty string.""" - if score is None: - return "" - parts = [] - if "total_score" in score: - max_score = score.get("max_score", 100.0) - parts.append(f"score {score['total_score']}/{max_score}") - if score.get("grade"): - parts.append(f"grade {score['grade']}") - if score.get("gate_result"): - parts.append(f"gate {score['gate_result']}") - return ", ".join(parts) - - def _aggregate_cross_defects(cross_results: list[dict]) -> list[dict]: """Aggregate defects from all CROSS sub-results (PS/PT/ST) into one list. @@ -335,14 +320,10 @@ def _build_report( module = result.get("module", "") label = _MODULE_LABELS.get(module, module) result_status = result.get("status", "unknown") - score = result.get("score") defects = _dedup_defects(result.get("defects", [])) error = result.get("error") - score_str = _format_score(score) header = f"## {label} ({module}) — {result_status}" - if score_str: - header += f", {score_str}" lines.append(header) lines.append("") @@ -369,20 +350,15 @@ def _build_report( continue # Standard defect table for other modules - lines.append("| ID | Name | Severity | Description | Location | Impact | Fix |") - lines.append("|---|---|---|---|---|---|---|") + lines.append("| ID | Name | Severity | Description | Location |") + lines.append("|---|---|---|---|---|") for defect in defects: defect_id = _esc_md_cell(defect.get("id")) name = _esc_md_cell(defect.get("name")) severity = _esc_md_cell(defect.get("severity")) description = _esc_md_cell(defect.get("description")) location = _esc_md_cell(defect.get("location")) - impact = _esc_md_cell(defect.get("impact")) - fix = _esc_md_cell(defect.get("fix_suggestion")) - lines.append( - f"| {defect_id} | {name} | {severity} | {description} |" - f" {location} | {impact} | {fix} |" - ) + lines.append(f"| {defect_id} | {name} | {severity} | {description} | {location} |") lines.append("") # Errors section diff --git a/src/sanityops_cli/defect_checker/renderer.py b/src/sanityops_cli/defect_checker/renderer.py index 2b6c4df..5e1dca0 100644 --- a/src/sanityops_cli/defect_checker/renderer.py +++ b/src/sanityops_cli/defect_checker/renderer.py @@ -197,9 +197,7 @@ def _render_artifact_groups(self, results: list[dict[str, Any]]) -> None: self._render_group(title, defects, module=module) self.console.print() - def _render_group( - self, label: str, defects: list[dict[str, Any]], *, module: str = "" - ) -> None: + def _render_group(self, label: str, defects: list[dict[str, Any]], *, module: str = "") -> None: lines: list[str] = [] shown = defects[:MAX_DEFECTS_PER_GROUP] for defect in shown: @@ -236,10 +234,7 @@ def _format_defect(self, defect: dict[str, Any], *, module: str = "") -> str: lines = [f"[bold red]✗ [{severity}] {defect_id}[/]"] if defect.get("location"): lines.append(f" [dim]Location[/] : {defect['location']}") - if defect.get("impact"): - lines.append(f" [dim]Impact[/] : {defect['impact']}") - if defect.get("fix_suggestion"): - lines.append(f" [dim]Fix[/] : {defect['fix_suggestion']}") + # Removed: impact, fix_suggestion return "\n".join(lines) def _format_permission_defect(self, defect: dict[str, Any]) -> str: @@ -265,8 +260,6 @@ def _format_permission_defect(self, defect: dict[str, Any]) -> str: # Standard fields if defect.get("location"): lines.append(f" [dim]Location[/] : {defect['location']}") - if defect.get("impact"): - lines.append(f" [dim]Impact[/] : {defect['impact']}") # Permission-specific fields from details dict details = defect.get("details", {}) @@ -281,10 +274,7 @@ def _format_permission_defect(self, defect: dict[str, Any]) -> str: if duty_side: lines.append(f" [dim]Duty[/] : {duty_side}") - # Fix suggestion - if defect.get("fix_suggestion"): - lines.append(f" [dim]Fix[/] : {defect['fix_suggestion']}") - + # Removed: impact, fix_suggestion return "\n".join(lines) # ------------------------------------------------------------------ diff --git a/src/sanityops_cli/help_panel.py b/src/sanityops_cli/help_panel.py index d1b83bb..ece1b79 100644 --- a/src/sanityops_cli/help_panel.py +++ b/src/sanityops_cli/help_panel.py @@ -6,8 +6,7 @@ GETTING_STARTED_TEXT = """\ 1. Run 'sanityops-cli init' to create the default configuration 2. Edit .sanityops/inspect_config.yaml to configure your project - 3. Run 'sanityops-cli inspect' to start the inspection - 4. Run 'sanityops-cli inspect repair' to generate fixes (optional) + 3. Run 'sanityops-cli inspect' to start the inspection\ """ ADVANCED_USAGE_TEXT = """\ @@ -25,7 +24,7 @@ def get_getting_started_panel() -> Panel: """Create the Getting Started panel for help output. Returns: - A Rich Panel containing the 4-step Getting Started workflow. + A Rich Panel containing the 3-step Getting Started workflow. """ return Panel( Text(GETTING_STARTED_TEXT, justify="left"), diff --git a/tests/commands/test_inspect_repair.py b/tests/commands/test_inspect_repair.py index de6065c..5ecf2b3 100644 --- a/tests/commands/test_inspect_repair.py +++ b/tests/commands/test_inspect_repair.py @@ -27,12 +27,14 @@ class TestFindLatestReport: def test_returns_none_when_dir_not_exists(self, tmp_path: Path): """Returns None when results directory does not exist.""" from sanityops_cli.commands.inspect import _find_latest_report + result = _find_latest_report(tmp_path / "nonexistent") assert result is None def test_returns_none_when_no_reports(self, tmp_path: Path): """Returns None when no inspect reports exist.""" from sanityops_cli.commands.inspect import _find_latest_report + (tmp_path / "results").mkdir() result = _find_latest_report(tmp_path / "results") assert result is None @@ -127,18 +129,15 @@ def test_creates_output_directory(self, tmp_path: Path): class TestInspectRepairCommand: - """Integration tests for inspect repair command.""" + """The repair command is disabled (closed); its helpers remain for reference.""" - def test_repair_command_shows_help(self): - """Repair command shows help without error.""" + def test_repair_command_not_registered(self): + """The disabled repair command is no longer registered on the CLI.""" result = runner.invoke(inspect_app, ["repair", "--help"]) - assert result.exit_code == 0 - assert "repair" in result.output.lower() + assert result.exit_code != 0 - def test_repair_command_requires_config_or_report(self, tmp_path: Path, monkeypatch): - """Repair command fails gracefully when no config exists.""" - # Change to temp directory with no config + def test_repair_command_rejected_without_help(self, tmp_path: Path, monkeypatch): + """Invoking the disabled repair command fails with a usage error.""" monkeypatch.chdir(tmp_path) result = runner.invoke(inspect_app, ["repair"]) - # Should fail because no config exists assert result.exit_code != 0 diff --git a/tests/test_help_panel.py b/tests/test_help_panel.py index d1c41b3..37af59c 100644 --- a/tests/test_help_panel.py +++ b/tests/test_help_panel.py @@ -39,12 +39,10 @@ def _run_entry_point(*args: str) -> tuple[int, str]: def test_getting_started_text_content(): - """Verify the Getting Started text contains all 4 steps.""" + """Verify the Getting Started text contains the inspection steps.""" assert "sanityops-cli init" in GETTING_STARTED_TEXT assert "inspect_config.yaml" in GETTING_STARTED_TEXT assert "sanityops-cli inspect" in GETTING_STARTED_TEXT - assert "sanityops-cli inspect repair" in GETTING_STARTED_TEXT - assert "(optional)" in GETTING_STARTED_TEXT def test_getting_started_panel_returns_panel(): diff --git a/tests/unit/defect_checker/test_markdown_reporter.py b/tests/unit/defect_checker/test_markdown_reporter.py index feb0cf7..a0f5985 100644 --- a/tests/unit/defect_checker/test_markdown_reporter.py +++ b/tests/unit/defect_checker/test_markdown_reporter.py @@ -29,7 +29,9 @@ def _response(**overrides): "results": [], "summary": { "total_defects": 0, - "p0_count": 0, "p1_count": 0, "p2_count": 0, + "p0_count": 0, + "p1_count": 0, + "p2_count": 0, "gate_result": "PASS", }, "errors": [], @@ -39,11 +41,24 @@ def _response(**overrides): return result -def _defect(defect_id="d1", name="n", severity="P0", description="desc", location="loc", impact="imp", fix="fix"): +def _defect( + defect_id="d1", + name="n", + severity="P0", + description="desc", + location="loc", + impact="imp", + fix="fix", +): return { - "id": defect_id, "name": name, "severity": severity, - "category": "cat", "description": description, - "location": location, "impact": impact, "fix_suggestion": fix, + "id": defect_id, + "name": name, + "severity": severity, + "category": "cat", + "description": description, + "location": location, + "impact": impact, + "fix_suggestion": fix, "artifact_refs": ["/proj/skills/s.md"], } @@ -57,6 +72,7 @@ def test_save_creates_file_in_output_dir(self, tmp_path): assert path.suffix == ".md" # Filename matches inspect-YYYYMMDD-HHMMSS-microseconds.md pattern import re + assert re.match(r"inspect-\d{8}-\d{6}-\d{6}\.md", path.name) def test_save_creates_parent_dirs(self, tmp_path): @@ -71,7 +87,10 @@ class TestReportContent: def test_report_contains_summary_table(self, tmp_path): response = _response( summary={ - "total_defects": 5, "p0_count": 1, "p1_count": 2, "p2_count": 2, + "total_defects": 5, + "p0_count": 1, + "p1_count": 2, + "p2_count": 2, "gate_result": "FAIL", }, ) @@ -86,24 +105,30 @@ def test_report_contains_module_header_and_defect_table(self, tmp_path): response = _response( results=[ { - "module": "QDS", "status": "completed", "defects": [_defect(defect_id="d1", severity="P0")], + "module": "QDS", + "status": "completed", + "defects": [_defect(defect_id="d1", severity="P0")], } ] ) path = save_markdown_report(response, tmp_path) content = path.read_text(encoding="utf-8") assert "## Skills (QDS) — completed" in content - assert "| ID | Name | Severity | Description | Location | Impact | Fix |" in content - assert "| d1 | n | P0 | desc | loc | imp | fix |" in content + assert "| ID | Name | Severity | Description | Location |" in content + assert "| d1 | n | P0 | desc | loc |" in content def test_report_escapes_pipes_and_newlines(self, tmp_path): response = _response( results=[ { - "module": "QDS", "status": "completed", - "defects": [_defect( - name="a|b", description="line1\nline2", - )], + "module": "QDS", + "status": "completed", + "defects": [ + _defect( + name="a|b", + description="line1\nline2", + ) + ], } ] ) @@ -164,11 +189,13 @@ class TestAggregateCrossDefects: def test_empty_input(self): """Returns empty list for empty input.""" from sanityops_cli.defect_checker.markdown_reporter import _aggregate_cross_defects + assert _aggregate_cross_defects([]) == [] def test_single_cross_result(self): """Aggregates defects from a single CROSS result.""" from sanityops_cli.defect_checker.markdown_reporter import _aggregate_cross_defects + cross_results = [ { "defects": [ @@ -187,6 +214,7 @@ def test_single_cross_result(self): def test_multiple_cross_results(self): """Aggregates defects from multiple CROSS results (PS/PT/ST).""" from sanityops_cli.defect_checker.markdown_reporter import _aggregate_cross_defects + cross_results = [ { "defects": [ @@ -207,6 +235,7 @@ def test_multiple_cross_results(self): def test_skips_non_dict_defects(self): """Skips defects that are not dicts.""" from sanityops_cli.defect_checker.markdown_reporter import _aggregate_cross_defects + cross_results = [ { "defects": [ @@ -224,6 +253,7 @@ class TestCalculateCrossScore: def test_empty_input(self): """Returns None for empty input.""" from sanityops_cli.defect_checker.markdown_reporter import _calculate_cross_score + assert _calculate_cross_score([], "L2") is None def test_valid_defects(self): @@ -233,6 +263,7 @@ def test_valid_defects(self): rather than mocking, to verify the integration works end-to-end. """ from sanityops_cli.defect_checker.markdown_reporter import _calculate_cross_score + defects = [ {"defect_id": "QD-PT-1", "defect_level": "P1", "relation": "QD-PT"}, ] @@ -276,9 +307,16 @@ def test_cross_module_merged_in_report(self, tmp_path): "module": "CROSS", "status": "completed", "defects": [ - {"id": "QD-PT-1", "name": "cross1", "severity": "P0", - "description": "d", "location": "l", "impact": "i", - "fix_suggestion": "f", "category": "QD-PT"}, + { + "id": "QD-PT-1", + "name": "cross1", + "severity": "P0", + "description": "d", + "location": "l", + "impact": "i", + "fix_suggestion": "f", + "category": "QD-PT", + }, ], }, ] @@ -297,18 +335,32 @@ def test_multiple_cross_subresults_merged(self, tmp_path): "module": "CROSS", "status": "completed", "defects": [ - {"id": "QD-PS-1", "name": "ps1", "severity": "P1", - "description": "d", "location": "l", "impact": "i", - "fix_suggestion": "f", "category": "QD-PS"}, + { + "id": "QD-PS-1", + "name": "ps1", + "severity": "P1", + "description": "d", + "location": "l", + "impact": "i", + "fix_suggestion": "f", + "category": "QD-PS", + }, ], }, { "module": "CROSS", "status": "completed", "defects": [ - {"id": "QD-PT-1", "name": "pt1", "severity": "P0", - "description": "d", "location": "l", "impact": "i", - "fix_suggestion": "f", "category": "QD-PT"}, + { + "id": "QD-PT-1", + "name": "pt1", + "severity": "P0", + "description": "d", + "location": "l", + "impact": "i", + "fix_suggestion": "f", + "category": "QD-PT", + }, ], }, ] @@ -413,7 +465,10 @@ def test_permission_module_uses_extended_table(self, tmp_path): content = path.read_text(encoding="utf-8") assert "## Permissions (QD-PM)" in content # Extended table header - assert "| ID | Name | Severity | Description | Action | Permission Side | Duty Side | Fix |" in content + assert ( + "| ID | Name | Severity | Description | Action | Permission Side | Duty Side | Fix |" + in content + ) # Data row with permission fields assert "QD-PM-1.1" in content assert "read" in content @@ -525,7 +580,7 @@ def test_permission_alongside_other_modules(self, tmp_path): assert "## Skills (QDS)" in content assert "## Permissions (QD-PM)" in content # QDS should use standard table - assert "| Description | Location | Impact |" in content + assert "| Description | Location |" in content # QD-PM should use extended table assert "| Action | Permission Side | Duty Side |" in content