From 5832525a6f1b46376239a808ace55e221c77fdeb Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Thu, 28 May 2026 21:44:25 +0100 Subject: [PATCH 01/18] fix: tests tests were breaking if the target had a `.` somewhere in global path higher than project directory --- graphify/extract.py | 7 ++++--- tests/test_extract.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index bc10f7d7d..b285a7bbc 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -5077,9 +5077,10 @@ def _ignored(p: Path) -> bool: return bool(patterns and _is_ignored(p, ignore_root, patterns, _cache=ignore_cache)) if not follow_symlinks: - # The old rglob filter rejected paths with a noise component anywhere, - # including components of target itself — preserve that. - if any(_is_noise_dir(part) for part in target.parts): + # Only reject if the target directory *itself* is a noise dir (e.g. + # node_modules passed directly). Do NOT check ancestor path components + # — that would incorrectly exclude projects living inside .worktrees/. + if _is_noise_dir(target.name): return [] # When negation (!) patterns exist, skip directory-level ignore pruning # so negated files inside ignored dirs can still be reached (same diff --git a/tests/test_extract.py b/tests/test_extract.py index 98b73edb9..f9f7a5ce7 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -382,7 +382,7 @@ def test_collect_files_from_dir(): def test_collect_files_skips_hidden(): files = collect_files(FIXTURES) for f in files: - assert not any(part.startswith(".") for part in f.parts) + assert not any(part.startswith(".") for part in f.relative_to(FIXTURES).parts) def test_collect_files_follows_symlinked_directory(tmp_path): @@ -445,7 +445,7 @@ def _legacy_collect_files(target, *, root=None): results.extend( p for p in target.rglob(f"*{ext}") if p.suffix == ext - and not any(_is_noise_dir(part) for part in p.parts) + and not any(_is_noise_dir(part) for part in p.relative_to(target).parts) and not (patterns and _is_ignored(p, ignore_root, patterns)) ) return sorted(results) From ed8be13c4f0ee8a27a9558c6fac0c4b1fc36c405 Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Thu, 28 May 2026 23:02:49 +0100 Subject: [PATCH 02/18] pytest: Skip tests that need network when running in sandbox --- tests/test_security.py | 9 +++++++++ tests/test_utils.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/test_utils.py diff --git a/tests/test_security.py b/tests/test_security.py index 74669f938..113f4dd11 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -27,14 +27,18 @@ _sanitize_metadata_value, ) +from .test_utils import skip_in_sandbox + # --------------------------------------------------------------------------- # validate_url # --------------------------------------------------------------------------- +@skip_in_sandbox() def test_validate_url_accepts_http(): assert validate_url("http://example.com/page") == "http://example.com/page" +@skip_in_sandbox() def test_validate_url_accepts_https(): assert validate_url("https://arxiv.org/abs/1706.03762") == "https://arxiv.org/abs/1706.03762" @@ -78,6 +82,7 @@ def test_safe_fetch_rejects_ftp_url(): with pytest.raises(ValueError, match="ftp"): safe_fetch("ftp://example.com/file.zip") +@skip_in_sandbox() def test_safe_fetch_returns_bytes(tmp_path): mock_resp = _make_mock_response(b"hello world") with patch("graphify.security._build_opener") as mock_opener_fn: @@ -87,6 +92,7 @@ def test_safe_fetch_returns_bytes(tmp_path): result = safe_fetch("https://example.com/") assert result == b"hello world" +@skip_in_sandbox() def test_safe_fetch_raises_on_non_2xx(): mock_resp = _make_mock_response(b"Not Found", status=404) with patch("graphify.security._build_opener") as mock_opener_fn: @@ -96,6 +102,7 @@ def test_safe_fetch_raises_on_non_2xx(): with pytest.raises(urllib.error.HTTPError): safe_fetch("https://example.com/missing") +@skip_in_sandbox() def test_safe_fetch_raises_on_size_exceeded(): # Build a response larger than max_bytes big_chunk = b"x" * 65_537 @@ -119,6 +126,7 @@ def test_safe_fetch_raises_on_size_exceeded(): # safe_fetch_text # --------------------------------------------------------------------------- +@skip_in_sandbox() def test_safe_fetch_text_decodes_utf8(): content = "héllo wörld".encode("utf-8") mock_resp = _make_mock_response(content) @@ -129,6 +137,7 @@ def test_safe_fetch_text_decodes_utf8(): result = safe_fetch_text("https://example.com/") assert result == "héllo wörld" +@skip_in_sandbox() def test_safe_fetch_text_replaces_bad_bytes(): bad = b"hello \xff world" mock_resp = _make_mock_response(bad) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..26f21ce6f --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,14 @@ +"""Shared test utilities and markers.""" + +import os + +import pytest + + +def skip_in_sandbox(): + """Skip tests that need access to external resources (git, network) in a sandbox.""" + sandbox_variables = ["NIX_ENFORCE_PURITY"] + return pytest.mark.skipif( + any(var in os.environ for var in sandbox_variables), + reason="Sandboxed environment with limited external access" + ) From 7a82cf86bccbaec0ceb5245c4d9b805a50f1d747 Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Tue, 26 May 2026 01:03:16 +0100 Subject: [PATCH 03/18] add nix flake --- .gitignore | 1 + flake.lock | 120 +++++++++++++++++++++++++++++ flake.nix | 216 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 337 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.gitignore b/.gitignore index 0a6775b2a..640aef9b3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ docs/superpowers/ .vscode/ .kilo openspec/ +result # Local benchmark scripts — never commit scripts/run_k2_*.py scripts/llm.py diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..b5a47d707 --- /dev/null +++ b/flake.lock @@ -0,0 +1,120 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778716662, + "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1779560665, + "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ], + "uv2nix": [ + "uv2nix" + ] + }, + "locked": { + "lastModified": 1779676664, + "narHash": "sha256-MbXylBTkWqVm8/VYjoULtMoVRgWBN1gSHbeRKsOsPlU=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "7bff980f37fc24e09dbc986643719900c139bf12", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778901413, + "narHash": "sha256-GSKXTAnFqRAMlZkJrIPcQMYf+lpMr66K3i60mB9STvc=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "a228447c3e179d477c1b6246ef3efa8cfe3c469a", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "uv2nix": "uv2nix" + } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1779411315, + "narHash": "sha256-IMFlxeyClau51KplhhSRGhdGTvD/knShHdybP1UOTuk=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "fdf2a76275d7a9c27deb5d2f2ab33526ac9052ff", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..73b73e0d5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,216 @@ +{ + description = "flake for graphify using uv2nix"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + + pyproject-nix = { + url = "github:pyproject-nix/pyproject.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + uv2nix = { + url = "github:pyproject-nix/uv2nix"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + pyproject-build-systems = { + url = "github:pyproject-nix/build-system-pkgs"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.uv2nix.follows = "uv2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = inputs @ { + flake-parts, + pyproject-nix, + uv2nix, + pyproject-build-systems, + ... + }: + flake-parts.lib.mkFlake {inherit inputs;} { + systems = ["x86_64-linux" "aarch64-linux" "aarch64-darwin"]; + + perSystem = { + pkgs, + lib, + ... + }: let + pyproject = lib.importTOML ./pyproject.toml; + projectMeta = pyproject.project; + + workspace = uv2nix.lib.workspace.loadWorkspace {workspaceRoot = ./.;}; + + overlay = workspace.mkPyprojectOverlay { + sourcePreference = "wheel"; + }; + + editableOverlay = workspace.mkEditablePyprojectOverlay { + root = "$REPO_ROOT"; + }; + + python = pkgs.python312; + + baseSet = + (pkgs.callPackage pyproject-nix.build.packages { + inherit python; + }) + .overrideScope + ( + lib.composeManyExtensions [ + pyproject-build-systems.overlays.wheel + overlay + ] + ); + + pythonSet = baseSet.overrideScope (final: prev: { + # numba manylinux wheel dlopens libtbb.so at runtime; expose it so + # autoPatchelfHook (from pyproject-build-systems' wheel overlay) can + # resolve it on the rpath. + numba = prev.numba.overrideAttrs (old: { + buildInputs = (old.buildInputs or []) ++ [pkgs.tbb]; + }); + + # nuitka's sdist doesn't declare setuptools as a build dep. + nuitka = prev.nuitka.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # jieba's sdist doesn't declare setuptools as a build dep. + jieba = prev.jieba.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # tree-sitter-dm's sdist doesn't declare setuptools as a build dep. + tree-sitter-dm = prev.tree-sitter-dm.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # Expose tests via passthru.tests so they can be wired into flake + # checks (mirrors the uv2nix testing pattern). + graphifyy = prev.graphifyy.overrideAttrs (old: { + passthru = + (old.passthru or {}) + // { + tests = let + # Virtualenv containing graphify plus the dev dependency + # group (which carries pytest and friends). + testVenv = final.mkVirtualEnv "graphify-test-env" (workspace.deps.default + // { + graphifyy = ["dev"]; + }); + in + (old.passthru.tests or {}) + // { + pytest = pkgs.stdenv.mkDerivation { + name = "${final.graphifyy.name}-pytest"; + inherit (final.graphifyy) src; + nativeBuildInputs = [testVenv pkgs.git]; + dontConfigure = true; + + buildPhase = '' + runHook preBuild + # The Nix build sandbox sets HOME=/homeless-shelter + # which is unwritable; several tests (e.g. the Gemini + # install ones) call helpers that resolve paths via + # Path.home() when not project-scoped. Point HOME at a + # writable temp dir so those tests pass under + # `nix flake check`. + export HOME=''${PWD}/home + pytest + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + touch $out + runHook postInstall + ''; + }; + }; + }; + }); + }); + + editablePythonSet = pythonSet.overrideScope editableOverlay; + virtualenv = editablePythonSet.mkVirtualEnv "graphify-dev-env" workspace.deps.all; + + graphifyEnv = pythonSet.mkVirtualEnv "graphify-env" workspace.deps.default; + + # Wrap the virtualenv so the default package exposes the `graphify` + # entry point directly while still carrying metadata from pyproject.toml. + graphifyPackage = pkgs.stdenv.mkDerivation { + pname = projectMeta.name; + version = projectMeta.version; + + dontUnpack = true; + dontBuild = true; + dontConfigure = true; + + nativeBuildInputs = [pkgs.makeWrapper]; + + installPhase = '' + mkdir -p $out/bin + makeWrapper ${graphifyEnv}/bin/graphify $out/bin/graphify + ''; + + passthru = { + inherit graphifyEnv; + }; + + meta = { + description = projectMeta.description; + homepage = projectMeta.urls.Homepage; + license = lib.licenses.mit; + mainProgram = "graphify"; + platforms = lib.platforms.unix; + }; + }; + in { + devShells.default = pkgs.mkShell { + packages = [ + virtualenv + pkgs.uv + pkgs.python3Packages.pytest + ]; + env = { + UV_NO_SYNC = "1"; + UV_PYTHON = editablePythonSet.python.interpreter; + UV_PYTHON_DOWNLOADS = "never"; + UV_PROJECT_ENVIRONMENT = virtualenv.outPath; + VIRTUAL_ENV = virtualenv.outPath; + }; + + shellHook = '' + unset PYTHONPATH + export REPO_ROOT=$(git rev-parse --show-toplevel) + ''; + }; + + packages.default = graphifyPackage; + + checks = { + inherit (pythonSet.graphifyy.passthru.tests) pytest; + }; + + apps.default = { + type = "app"; + program = "${graphifyPackage}/bin/graphify"; + meta = graphifyPackage.meta; + }; + }; + }; +} From 768ed8c7a66fe8f67ff163d95a6a8514cb2be524 Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Tue, 26 May 2026 02:12:05 +0100 Subject: [PATCH 04/18] added nix flake install option to README --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index e73db3aa0..0cbde15a4 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ Every system ran on the same harness with the same model and budgets, scored by | Python | 3.10+ | `python --version` | [python.org](https://www.python.org/downloads/) | | uv *(recommended)* | any | `uv --version` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | | pipx *(alternative)* | any | `pipx --version` | `pip install pipx` | +| Nix *(alternative)* | 2.4+ (flakes enabled) | `nix --version` | [nixos.org](https://nixos.org/download/) | **macOS quick install (Homebrew):** ```bash @@ -163,6 +164,36 @@ pipx install graphifyy pip install graphifyy # may need PATH setup — see note below ``` +**Nix (flake):** + +Run directly without installing: +```bash +nix run github:safishamsi/graphify +``` + +To expose `graphify` as a package inside another flake, add it as an input and reference its default package: +```nix +{ + inputs = { + graphify.url = "github:safishamsi/graphify"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + }; + + outputs = { nixpkgs, graphify, ... }: { + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + in { + devShells.${system}.default = pkgs.mkShell { + buildInputs = [ graphify.packages.${system}.default ]; + }; + }; + }; +} +``` + +--- + **Step 2 — register the skill with your AI assistant:** ```bash From 67ea00d27af8c7a51655ae6d50ca2081ecc50c0b Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Thu, 28 May 2026 14:46:57 +0100 Subject: [PATCH 05/18] add nix ci job --- .github/workflows/ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..7dcaf7cf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,3 +104,22 @@ jobs: - name: pip-audit (dependency vulnerabilities) continue-on-error: true run: uv run --frozen pip-audit --strict + nix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: Check flake + run: nix flake check --all-systems + + - name: Build default package + run: nix build .#default + + - name: Verify built binary runs + run: nix run .#default -- --help From b4e09f7035b19ea0c89ab9f64ade41be7207e7a1 Mon Sep 17 00:00:00 2001 From: Vitali Karasenko Date: Thu, 4 Jun 2026 17:56:54 +0100 Subject: [PATCH 06/18] pytest: skip tests that need .git in sandbox .git is not available in a nix sandbox --- tests/test_skillgen.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 0c09e601e..7d323d147 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -13,6 +13,8 @@ import pytest +from .test_utils import skip_in_sandbox + # tests/ -> repo root is one parent up; put it on the path so tools.skillgen # imports regardless of pytest's import mode. REPO_ROOT = Path(__file__).resolve().parent.parent @@ -22,6 +24,7 @@ from tools.skillgen import gen # noqa: E402 +@skip_in_sandbox() def test_audit_coverage_passes(): """Every v8 heading lands in the lean core or exactly one reference.""" platforms = gen.load_platforms() @@ -248,6 +251,7 @@ def test_check_passes_for_codex_and_windows(): assert problems == [], f"[{key}]\n" + "\n".join(problems) +@skip_in_sandbox() def test_audit_coverage_passes_for_codex_and_windows(): """Every v8 heading single-homes for the cli-inline split hosts too.""" platforms = gen.load_platforms() @@ -388,6 +392,7 @@ def test_schema_singleton_catches_legacy_enums(): ) +@skip_in_sandbox() def test_all_progressive_hosts_check_and_audit_clean(): """check + audit-coverage pass for every rendered progressive host.""" platforms = gen.load_platforms() @@ -473,6 +478,7 @@ def test_monoliths_render_inline_single_file_no_references(): assert "references/" not in arts[0].content or "see `references/" not in arts[0].content.lower() +@skip_in_sandbox() def test_monolith_roundtrip_passes_for_aider_and_devin(): """Each monolith is diff-clean vs v8 except the file_type enum unification.""" platforms = gen.load_platforms() @@ -481,6 +487,7 @@ def test_monolith_roundtrip_passes_for_aider_and_devin(): assert problems == [], f"[{key}]\n" + "\n".join(problems) +@skip_in_sandbox() def test_monoliths_change_only_sanctioned_lines(): """Every line that differs from pristine v8 is a sanctioned change-class. @@ -607,6 +614,7 @@ def test_always_on_included_in_full_render_not_per_platform(): assert "graphify/always_on/claude-md.md" not in claude_only +@skip_in_sandbox() def test_always_on_roundtrip_is_byte_faithful(): """Each always_on/*.md reproduces its former __main__.py constant byte for byte. @@ -685,6 +693,7 @@ def test_always_on_files_are_guarded_by_check(tmp_path): # --- the per-host coverage audit (the systemic guard) -------------------------- +@skip_in_sandbox() def test_audit_coverage_passes_for_every_split_host(): """Every split host's render single-homes its own v8 body's headings.""" platforms = gen.load_platforms() @@ -705,6 +714,7 @@ def test_audit_reads_each_host_against_its_own_v8_body(): assert gen._v8_baseline_ref("vscode") == "47042beb05d1f6dd2186c0c499ae2840ce604ead:graphify/skill-vscode.md" +@skip_in_sandbox() def test_audit_catches_an_induced_per_host_drop(): """Re-inducing the trae regression (claude-flavored hooks) fails the audit. @@ -722,6 +732,7 @@ def test_audit_catches_an_induced_per_host_drop(): assert any("native AGENTS.md integration (Trae)" in p for p in problems), problems +@skip_in_sandbox() def test_audit_catches_a_dropped_non_allowlisted_heading(): """A core fragment that drops a real v8 heading fails the audit. @@ -882,6 +893,7 @@ def test_amp_has_no_pretooluse_caveat_anywhere(): assert "Trae" not in b2 +@skip_in_sandbox() def test_amp_audit_coverage_passes_against_its_own_v8(): """The per-host audit (the guard amp is the exact case for) passes for amp. From beab28f3331bce1faf9faebe00c9bd3cd315e179 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 11 Jul 2026 11:19:13 +0200 Subject: [PATCH 07/18] adopt simit-maintained Nix CI --- .github/workflows/ci.yaml | 35 +++++++++++ .github/workflows/ci.yml | 125 -------------------------------------- README.md | 19 +++++- nix/pre-commit.nix | 61 +++++++++++++++++++ simit.toml | 10 +++ 5 files changed, 123 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/ci.yml create mode 100644 nix/pre-commit.nix create mode 100644 simit.toml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..caff8dcfd --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,35 @@ +# Generated by simit. Manual edits will be reported as ci=drift. +name: CI + +on: + push: + branches: ["**"] + tags-ignore: ["**"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + env: + NIX_CONFIG: "experimental-features = nix-command flakes" + XDG_CACHE_HOME: "/tmp/.cache" + CARGO_HOME: "/tmp/.cargo" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Check generated flake wiring + run: nix run git+https://codeberg.org/caniko/simit.git -- init flake --check --diff + + - name: Check flake evaluation + run: nix flake check --no-build + + - name: Build check pytest + run: nix build .#checks.x86_64-linux.pytest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 7dcaf7cf3..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: CI - -on: - push: - branches: ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "main"] - pull_request: - branches: ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "main"] - workflow_dispatch: - -jobs: - skillgen-check: - # Fast lint-style guard: the skill files under graphify/ are generated from - # the fragments in tools/skillgen/. This fails if someone hand-edited a - # generated file or forgot to re-run the generator and bless expected/, and it - # runs the build-time validators that guard per-host coverage, the file_type - # enum, the monolith round-trips, and the always-on round-trips. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - # The audit-coverage, monolith-roundtrip, and always-on-roundtrip - # validators read blobs from origin/v8. A shallow checkout omits that - # ref, so fetch the full history here and the validators run for real - # (rather than skipping). The other jobs stay shallow. - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: "3.12" - - # --frozen keeps uv from re-resolving and rewriting uv.lock as a side - # effect of `uv run`; the lock is committed and must not churn in CI. - - name: Check generated skill artifacts are up to date - run: uv run --frozen python -m tools.skillgen --check - - - name: Audit per-host v8 coverage - run: uv run --frozen python -m tools.skillgen --audit-coverage - - - name: Check the file_type enum is a singleton - run: uv run --frozen python -m tools.skillgen --schema-singleton - - - name: Round-trip the monoliths against v8 - run: uv run --frozen python -m tools.skillgen --monolith-roundtrip - - - name: Round-trip the always-on blocks against v8 - run: uv run --frozen python -m tools.skillgen --always-on-roundtrip - - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.12"] - - steps: - - uses: actions/checkout@v6 - with: - # test_skillgen.py reads pre-split skill bodies from the immutable - # baseline commit via `git show`; a shallow checkout omits that history - # and the baseline tests fail. Full history mirrors the skillgen-check job. - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: ${{ matrix.python-version }} - - # --frozen installs straight from the committed uv.lock without re-resolving - # or rewriting it, so CI never churns the lock. - - name: Install dependencies - run: uv sync --all-extras --frozen - - - name: Run tests - run: uv run --frozen pytest tests/ -q --tb=short - - - name: Verify install works end-to-end - run: | - uv run --frozen graphify --help - uv run --frozen graphify install - - security-scan: - # The dev deps include bandit and pip-audit. Run them in CI so a new - # HIGH-severity finding or vulnerable dependency is caught on the PR that - # introduces it, rather than at the next manual audit. - # Non-blocking for now (continue-on-error) to avoid breaking CI on - # pre-existing findings; remove continue-on-error after the initial - # cleanup pass. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: "3.12" - - - name: Install dependencies - run: uv sync --frozen - - - name: bandit (static security analysis) - continue-on-error: true - run: uv run --frozen bandit -r graphify -ll - - - name: pip-audit (dependency vulnerabilities) - continue-on-error: true - run: uv run --frozen pip-audit --strict - nix: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - name: Check flake - run: nix flake check --all-systems - - - name: Build default package - run: nix build .#default - - - name: Verify built binary runs - run: nix run .#default -- --help diff --git a/README.md b/README.md index 0cbde15a4..e352d5c33 100644 --- a/README.md +++ b/README.md @@ -168,14 +168,14 @@ pip install graphifyy # may need PATH setup — see note below Run directly without installing: ```bash -nix run github:safishamsi/graphify +nix run github:caniko/graphify ``` To expose `graphify` as a package inside another flake, add it as an input and reference its default package: ```nix { inputs = { - graphify.url = "github:safishamsi/graphify"; + graphify.url = "github:caniko/graphify"; nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; }; @@ -192,6 +192,21 @@ To expose `graphify` as a package inside another flake, add it as an input and r } ``` +The flake and GitHub Actions workflow are generated and checked by +[simit](https://codeberg.org/caniko/simit). After changing the flake outputs, +refresh the generated wiring and verify the same checks CI runs: + +```bash +simit init flake --check --diff +simit init ci --platform github --runtime nix --check --diff +nix flake check --no-build +nix build .#checks.x86_64-linux.pytest +``` + +Keep `simit.toml`, `nix/pre-commit.nix`, and `.github/workflows/ci.yaml` in +sync; the workflow intentionally builds the named `pytest` check instead of +duplicating a second Python dependency installation path. + --- **Step 2 — register the skill with your AI assistant:** diff --git a/nix/pre-commit.nix b/nix/pre-commit.nix new file mode 100644 index 000000000..8f841d55c --- /dev/null +++ b/nix/pre-commit.nix @@ -0,0 +1,61 @@ +{ + pkgs, + treefmtWrapper, + rustToolchain ? null, +}: { + treefmt = { + enable = true; + name = "treefmt"; + entry = "${treefmtWrapper}/bin/treefmt --fail-on-change"; + pass_filenames = false; + }; + + cargo-fmt = { + enable = true; + name = "cargo fmt"; + entry = "cargo fmt --all -- --check"; + extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain; + pass_filenames = false; + }; + + cargo-clippy = { + enable = true; + name = "cargo clippy"; + entry = "cargo clippy --all-targets --all-features -- --deny warnings"; + extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain; + pass_filenames = false; + }; + + cargo-audit = { + enable = true; + name = "cargo audit"; + entry = "cargo audit"; + extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain ++ [pkgs.cargo-audit]; + pass_filenames = false; + }; + + nix-flake-check = { + enable = true; + name = "nix flake check"; + entry = "nix --extra-experimental-features 'nix-command flakes' flake check --cores 0 --max-jobs auto --no-update-lock-file"; + extraPackages = [pkgs.nix]; + pass_filenames = false; + stages = ["manual"]; + }; + + uv-ruff-format = { + enable = true; + name = "uv ruff format"; + entry = "uv run ruff format --check ."; + extraPackages = [pkgs.uv]; + pass_filenames = false; + }; + + uv-mypy = { + enable = true; + name = "uv mypy"; + entry = "uv run mypy ."; + extraPackages = [pkgs.uv]; + pass_filenames = false; + }; +} diff --git a/simit.toml b/simit.toml new file mode 100644 index 000000000..66542e84a --- /dev/null +++ b/simit.toml @@ -0,0 +1,10 @@ +[flake] +mode = "custom" +scope = "hooks-only" + +[flake.expected_outputs] +checks = ["pytest"] + +[ci] +runtime = "nix" +runner = "ubuntu-latest" From c521b8426355e6d9ae8e98e2058138e2edb0550d Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 11 Jul 2026 11:25:24 +0200 Subject: [PATCH 08/18] make the Nix test check hermetic --- flake.nix | 10 ++++++++-- graphify/extract.py | 9 ++++++++- tests/test_extraction_spec_ids.py | 3 ++- tests/test_skillgen.py | 1 + 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 73b73e0d5..8e40a0caa 100644 --- a/flake.nix +++ b/flake.nix @@ -110,14 +110,20 @@ # group (which carries pytest and friends). testVenv = final.mkVirtualEnv "graphify-test-env" (workspace.deps.default // { - graphifyy = ["dev"]; + # The retry-cap tests exercise the OpenAI-compatible Ollama + # path, so include that optional extra in the test-only + # environment without pulling every runtime extra. + graphifyy = ["dev" "ollama"]; }); in (old.passthru.tests or {}) // { pytest = pkgs.stdenv.mkDerivation { name = "${final.graphifyy.name}-pytest"; - inherit (final.graphifyy) src; + # Test the repository tree rather than the wheel source: + # skillgen's fixtures and extraction-spec fragments are + # intentionally repository assets, not package payload. + src = ./.; nativeBuildInputs = [testVenv pkgs.git]; dontConfigure = true; diff --git a/graphify/extract.py b/graphify/extract.py index b285a7bbc..a030c6ee0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3486,7 +3486,14 @@ def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: except OSError: return classes for cs_path in cs_files: - if any(_is_noise_dir(part) for part in cs_path.parts): + # Check only project-relative components. Absolute build sandboxes + # commonly mount sources below ``/build``; treating that ancestor as a + # project noise directory would hide every ViewModel from XAML linking. + try: + relative_parts = cs_path.relative_to(root).parts + except ValueError: + relative_parts = cs_path.parts + if any(_is_noise_dir(part) for part in relative_parts[:-1]): continue if patterns and _is_ignored(cs_path, root, patterns, _cache=ignore_cache): continue diff --git a/tests/test_extraction_spec_ids.py b/tests/test_extraction_spec_ids.py index 46fabc3ba..8a931262f 100644 --- a/tests/test_extraction_spec_ids.py +++ b/tests/test_extraction_spec_ids.py @@ -38,7 +38,8 @@ def _spec_files() -> list[Path]: for p in root.rglob("extraction-spec.md"): # build/ is a packaging artifact; expected/ is skillgen's own golden # output and is already covered by `skillgen --check`. - if "/build/" in p.as_posix() or "/expected/" in p.as_posix(): + relative = p.relative_to(REPO_ROOT).parts + if "build" in relative or "expected" in relative: continue files.append(p) return sorted(files) diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 7d323d147..483d0f6ff 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -955,6 +955,7 @@ def test_agents_body_matches_amp_modulo_hooks_wording(): assert amp["hooks.md"] != agents["hooks.md"] +@skip_in_sandbox() def test_agents_audit_baseline_is_amps_v8_body(): """`agents` is a post-v8 platform, so its audit baseline is amp's v8 body.""" platforms = gen.load_platforms() From bac5c26621469e1af0d798d3572fdf6e242d5606 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 11 Jul 2026 11:54:29 +0200 Subject: [PATCH 09/18] scope simit generation to Python and Nix --- .github/workflows/ci.yaml | 1 - nix/pre-commit.nix | 33 --------------------------------- simit.toml | 2 ++ 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index caff8dcfd..8de65289e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,6 @@ jobs: env: NIX_CONFIG: "experimental-features = nix-command flakes" XDG_CACHE_HOME: "/tmp/.cache" - CARGO_HOME: "/tmp/.cargo" steps: - name: Checkout uses: actions/checkout@v4 diff --git a/nix/pre-commit.nix b/nix/pre-commit.nix index 8f841d55c..c6d6d3b4a 100644 --- a/nix/pre-commit.nix +++ b/nix/pre-commit.nix @@ -1,7 +1,6 @@ { pkgs, treefmtWrapper, - rustToolchain ? null, }: { treefmt = { enable = true; @@ -10,30 +9,6 @@ pass_filenames = false; }; - cargo-fmt = { - enable = true; - name = "cargo fmt"; - entry = "cargo fmt --all -- --check"; - extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain; - pass_filenames = false; - }; - - cargo-clippy = { - enable = true; - name = "cargo clippy"; - entry = "cargo clippy --all-targets --all-features -- --deny warnings"; - extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain; - pass_filenames = false; - }; - - cargo-audit = { - enable = true; - name = "cargo audit"; - entry = "cargo audit"; - extraPackages = pkgs.lib.optional (rustToolchain != null) rustToolchain ++ [pkgs.cargo-audit]; - pass_filenames = false; - }; - nix-flake-check = { enable = true; name = "nix flake check"; @@ -50,12 +25,4 @@ extraPackages = [pkgs.uv]; pass_filenames = false; }; - - uv-mypy = { - enable = true; - name = "uv mypy"; - entry = "uv run mypy ."; - extraPackages = [pkgs.uv]; - pass_filenames = false; - }; } diff --git a/simit.toml b/simit.toml index 66542e84a..98687d863 100644 --- a/simit.toml +++ b/simit.toml @@ -1,6 +1,7 @@ [flake] mode = "custom" scope = "hooks-only" +components = ["treefmt", "nix-flake-check", "uv-ruff-format"] [flake.expected_outputs] checks = ["pytest"] @@ -8,3 +9,4 @@ checks = ["pytest"] [ci] runtime = "nix" runner = "ubuntu-latest" +components = ["flake-wiring", "flake-evaluation", "checks"] From 37fb1b13a90980c7aa1f65b158477123150775a0 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Mon, 13 Jul 2026 10:36:03 +0200 Subject: [PATCH 10/18] feat: add Graphify NixOS module and PostgreSQL extraction --- README.md | 36 ++ flake.nix | 201 ++++++++- graphify/cli.py | 6 + nix/nixos-module.nix | 761 +++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_backend_extras.py | 6 + tests/test_extract_cli.py | 787 ++++------------------------------- uv.lock | 4 +- 8 files changed, 1063 insertions(+), 740 deletions(-) create mode 100644 nix/nixos-module.nix diff --git a/README.md b/README.md index e352d5c33..8129ab168 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,42 @@ Keep `simit.toml`, `nix/pre-commit.nix`, and `.github/workflows/ci.yaml` in sync; the workflow intentionally builds the named `pytest` check instead of duplicating a second Python dependency installation path. +For a long-running NixOS deployment, import `graphify.nixosModules.default`. +The module uses the full runtime package (including MCP, PostgreSQL, graph +database exports, and every built-in LLM provider) while `packages.default` +stays lean for interactive AST-only use. Each named instance owns independent +state, source selection, extraction schedule, optional file watcher, HTTP MCP +listener, and Neo4j/FalkorDB sinks: + +```nix +{ + imports = [inputs.graphify.nixosModules.default]; + + services.graphify = { + enable = true; + instances.database = { + source.postgresql = { + enable = true; + host = "/run/postgresql"; + database = "app"; + user = "graphify"; + systemdService = "postgresql.service"; + }; + extraction.onCalendar = "daily"; + server.enable = true; # loopback HTTP MCP on port 8080 + }; + }; +} +``` + +PostgreSQL is a read-only schema input, not Graphify's persistence backend; +the resulting graph remains +`/var/lib/graphify//graphify-out/graph.json`. The reusable module +does not create database roles or grants. Host policy must provision a role +that can see the selected schema. Passwords, pgpass files, provider keys, MCP +keys, and graph-database passwords have file-backed options and are loaded as +systemd credentials instead of appearing on process command lines. + --- **Step 2 — register the skill with your AI assistant:** diff --git a/flake.nix b/flake.nix index 8e40a0caa..8ac4b675e 100644 --- a/flake.nix +++ b/flake.nix @@ -38,6 +38,15 @@ flake-parts.lib.mkFlake {inherit inputs;} { systems = ["x86_64-linux" "aarch64-linux" "aarch64-darwin"]; + flake.nixosModules.default = { + lib, + pkgs, + ... + }: { + imports = [./nix/nixos-module.nix]; + services.graphify.package = lib.mkDefault inputs.self.packages.${pkgs.stdenv.hostPlatform.system}.full; + }; + perSystem = { pkgs, lib, @@ -155,37 +164,161 @@ virtualenv = editablePythonSet.mkVirtualEnv "graphify-dev-env" workspace.deps.all; graphifyEnv = pythonSet.mkVirtualEnv "graphify-env" workspace.deps.default; + graphifyFullEnv = pythonSet.mkVirtualEnv "graphify-full-env" (workspace.deps.default + // { + graphifyy = ["all"]; + }); - # Wrap the virtualenv so the default package exposes the `graphify` - # entry point directly while still carrying metadata from pyproject.toml. - graphifyPackage = pkgs.stdenv.mkDerivation { - pname = projectMeta.name; - version = projectMeta.version; + # Wrap a virtualenv so consumers receive stable public entry points while + # the environment remains available for smoke checks and composition. + mkGraphifyPackage = { + environment, + suffix ? "", + }: + pkgs.stdenv.mkDerivation { + pname = projectMeta.name + suffix; + version = projectMeta.version; - dontUnpack = true; - dontBuild = true; - dontConfigure = true; + dontUnpack = true; + dontBuild = true; + dontConfigure = true; - nativeBuildInputs = [pkgs.makeWrapper]; + nativeBuildInputs = [pkgs.makeWrapper]; - installPhase = '' - mkdir -p $out/bin - makeWrapper ${graphifyEnv}/bin/graphify $out/bin/graphify - ''; + installPhase = '' + mkdir -p $out/bin + makeWrapper ${environment}/bin/graphify $out/bin/graphify + if [ -x ${environment}/bin/graphify-mcp ]; then + makeWrapper ${environment}/bin/graphify-mcp $out/bin/graphify-mcp + fi + ''; - passthru = { - inherit graphifyEnv; - }; + passthru = { + graphifyEnv = environment; + }; - meta = { - description = projectMeta.description; - homepage = projectMeta.urls.Homepage; - license = lib.licenses.mit; - mainProgram = "graphify"; - platforms = lib.platforms.unix; + meta = { + description = projectMeta.description; + homepage = projectMeta.urls.Homepage; + license = lib.licenses.mit; + mainProgram = "graphify"; + platforms = lib.platforms.unix; + }; }; + + graphifyPackage = mkGraphifyPackage {environment = graphifyEnv;}; + graphifyFullPackage = mkGraphifyPackage { + environment = graphifyFullEnv; + suffix = "-full"; + }; + + moduleSample = inputs.nixpkgs.lib.nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + specialArgs.graphifyPackage = graphifyFullPackage; + modules = [ + ./nix/nixos-module.nix + { + system.stateVersion = "24.11"; + services.graphify = { + enable = true; + instances.postgres-only = { + source.postgresql = { + enable = true; + database = "catalog"; + }; + extraction.noCluster = true; + }; + instances.matrix = { + source = { + path = "/srv/source"; + cargo = true; + postgresql = { + enable = true; + host = "/run/postgresql"; + database = "app"; + user = "graphify"; + sslMode = "disable"; + pgpassFile = "/run/keys/pgpass"; + systemdService = "postgresql.service"; + }; + }; + extraction = { + mode = "deep"; + codeOnly = true; + noCluster = true; + dedup = true; + googleWorkspace = true; + global = true; + tag = "matrix"; + maxWorkers = 2; + tokenBudget = 4096; + maxConcurrency = 2; + apiTimeout = 30; + resolution = 1.25; + excludeHubs = 0.95; + excludes = ["vendor" "target"]; + timing = true; + onCalendar = "hourly"; + }; + llm = { + backend = "openai"; + model = "test-model"; + baseUrl = "http://127.0.0.1:8081/v1"; + apiKeyFile = "/run/keys/openai"; + }; + watch = { + enable = true; + debounce = 1.5; + }; + server = { + enable = true; + host = "0.0.0.0"; + port = 8080; + path = "/mcp"; + jsonResponse = true; + stateless = true; + sessionTimeout = 0; + apiKeyFile = "/run/keys/mcp"; + openFirewall = true; + }; + exports = { + neo4j = { + enable = true; + uri = "bolt://127.0.0.1:7687"; + passwordFile = "/run/keys/neo4j"; + }; + falkordb = { + enable = true; + uri = "falkordb://127.0.0.1:6379"; + onCalendar = "daily"; + }; + }; + environment.GRAPHIFY_MAX_RETRIES = "3"; + environmentFiles = ["/run/keys/graphify.env"]; + }; + }; + } + ]; }; in { + formatter = pkgs.writeShellApplication { + name = "graphify-nix-format"; + runtimeInputs = [pkgs.alejandra]; + text = '' + has_path=0 + for argument in "$@"; do + case "$argument" in + -*) ;; + *) has_path=1 ;; + esac + done + if [ "$has_path" -eq 0 ]; then + set -- "$@" . + fi + exec alejandra "$@" + ''; + }; + devShells.default = pkgs.mkShell { packages = [ virtualenv @@ -206,10 +339,32 @@ ''; }; - packages.default = graphifyPackage; + packages = { + default = graphifyPackage; + full = graphifyFullPackage; + }; checks = { inherit (pythonSet.graphifyy.passthru.tests) pytest; + full-package = pkgs.runCommand "graphify-full-package-check" {} '' + test -x ${graphifyFullPackage}/bin/graphify + test -x ${graphifyFullPackage}/bin/graphify-mcp + ${graphifyFullEnv}/bin/python -c 'import anthropic, boto3, falkordb, mcp, neo4j, openai, psycopg' + touch $out + ''; + nixos-module = pkgs.runCommand "graphify-nixos-module-check" {} '' + test '${moduleSample.config.services.graphify.instances.matrix.source.postgresql.database}' = app + test '${toString moduleSample.config.services.graphify.instances.matrix.server.port}' = 8080 + test '${toString moduleSample.config.networking.firewall.allowedTCPPorts}' = 8080 + case ${lib.escapeShellArg (toString moduleSample.config.systemd.services.graphify-matrix-extract.serviceConfig.ExecStart)} in + *graphify-matrix-extract*) ;; + *) echo 'missing Graphify extract service' >&2; exit 1 ;; + esac + test '${moduleSample.config.services.graphify.instances.matrix.exports.neo4j.uri}' = 'bolt://127.0.0.1:7687' + test -n '${toString moduleSample.config.systemd.services.graphify-matrix-neo4j.serviceConfig.ExecStart}' + test '${toString moduleSample.config.systemd.timers.graphify-matrix-falkordb.timerConfig.OnCalendar}' = daily + touch $out + ''; }; apps.default = { diff --git a/graphify/cli.py b/graphify/cli.py index 774034220..7ca95199f 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2550,6 +2550,12 @@ def _parse_float(name: str, raw: str) -> float: ) if not has_path: + # PostgreSQL-only extraction has no filesystem corpus to detect, but + # the common reporting/manifest path below still consumes the + # detection result. Keep the same shape as detect() returns so + # ``graphify extract --postgres DSN`` can continue through database + # introspection without reading an unbound local. + detection = {"files": {}, "unclassified": []} code_files = [] doc_files = [] paper_files = [] diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix new file mode 100644 index 000000000..6c35274fe --- /dev/null +++ b/nix/nixos-module.nix @@ -0,0 +1,761 @@ +{ + config, + graphifyPackage ? null, + lib, + pkgs, + ... +}: let + inherit (lib) concatLists escapeShellArgs filterAttrs flatten mapAttrs' mapAttrsToList mkEnableOption mkIf mkMerge mkOption nameValuePair optional optionalAttrs optionalString types unique; + cfg = config.services.graphify; + + builtInBackends = [ + "azure" + "bedrock" + "claude" + "claude-cli" + "deepseek" + "gemini" + "kimi" + "ollama" + "openai" + ]; + + backendApiKeyVariables = { + azure = "AZURE_OPENAI_API_KEY"; + claude = "ANTHROPIC_API_KEY"; + deepseek = "DEEPSEEK_API_KEY"; + gemini = "GEMINI_API_KEY"; + kimi = "MOONSHOT_API_KEY"; + ollama = "OLLAMA_API_KEY"; + openai = "OPENAI_API_KEY"; + }; + + backendBaseUrlVariables = { + azure = "AZURE_OPENAI_ENDPOINT"; + claude = "ANTHROPIC_BASE_URL"; + deepseek = "DEEPSEEK_BASE_URL"; + gemini = "GEMINI_BASE_URL"; + kimi = "KIMI_BASE_URL"; + ollama = "OLLAMA_BASE_URL"; + openai = "OPENAI_BASE_URL"; + }; + + positiveInt = types.ints.positive; + nullablePositiveInt = types.nullOr positiveInt; + nullablePositiveNumber = types.nullOr (types.addCheck types.number (value: value > 0)); + + postgresOptions = {name, ...}: { + options = { + enable = mkEnableOption "PostgreSQL schema introspection for ${name}"; + + host = mkOption { + type = types.str; + default = "/run/postgresql"; + description = "libpq host name, address, or Unix socket directory."; + }; + + port = mkOption { + type = types.port; + default = 5432; + description = "PostgreSQL port."; + }; + + database = mkOption { + type = types.nullOr types.str; + default = null; + description = "Database whose schema Graphify introspects."; + }; + + user = mkOption { + type = types.str; + default = cfg.user; + defaultText = lib.literalExpression "config.services.graphify.user"; + description = "PostgreSQL role. Local peer authentication can use the Graphify system user."; + }; + + sslMode = mkOption { + type = types.enum ["disable" "allow" "prefer" "require" "verify-ca" "verify-full"]; + default = "prefer"; + description = "libpq SSL mode."; + }; + + pgpassFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Optional pgpass file loaded as a systemd credential; it never appears on argv."; + }; + + systemdService = mkOption { + type = types.nullOr types.str; + default = null; + example = "postgresql.service"; + description = "Optional local PostgreSQL unit required before extraction."; + }; + }; + }; + + extractionOptions = {name, ...}: { + options = { + enable = mkEnableOption "headless extraction for ${name}" // {default = true;}; + startAtBoot = mkOption { + type = types.bool; + default = true; + description = "Run extraction during multi-user startup."; + }; + onCalendar = mkOption { + type = types.nullOr types.str; + default = null; + example = "daily"; + description = "Optional systemd calendar schedule."; + }; + mode = mkOption { + type = types.enum ["normal" "deep"]; + default = "normal"; + description = "Semantic extraction depth."; + }; + codeOnly = mkOption { + type = types.bool; + default = false; + }; + noCluster = mkOption { + type = types.bool; + default = false; + }; + dedup = mkOption { + type = types.bool; + default = false; + }; + googleWorkspace = mkOption { + type = types.bool; + default = false; + }; + global = mkOption { + type = types.bool; + default = false; + }; + tag = mkOption { + type = types.nullOr types.str; + default = null; + }; + maxWorkers = mkOption { + type = nullablePositiveInt; + default = null; + }; + tokenBudget = mkOption { + type = nullablePositiveInt; + default = null; + }; + maxConcurrency = mkOption { + type = nullablePositiveInt; + default = null; + }; + apiTimeout = mkOption { + type = nullablePositiveNumber; + default = null; + }; + resolution = mkOption { + type = types.addCheck types.number (value: value > 0); + default = 1.0; + }; + excludeHubs = mkOption { + type = types.nullOr types.number; + default = null; + }; + excludes = mkOption { + type = types.listOf types.str; + default = []; + }; + timing = mkOption { + type = types.bool; + default = false; + }; + }; + }; + + llmOptions = {name, ...}: { + options = { + backend = mkOption { + type = types.nullOr types.str; + default = null; + example = "openai"; + description = "Built-in or custom Graphify backend name."; + }; + customProvider = mkOption { + type = types.bool; + default = false; + description = "Whether backend names an entry from providersFile instead of a built-in backend."; + }; + model = mkOption { + type = types.nullOr types.str; + default = null; + }; + baseUrl = mkOption { + type = types.nullOr types.str; + default = null; + description = "Built-in provider endpoint override."; + }; + apiKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Provider API key loaded through a systemd credential."; + }; + apiKeyEnvironmentVariable = mkOption { + type = types.nullOr (types.addCheck types.str (value: builtins.match "[A-Z_][A-Z0-9_]*" value != null)); + default = null; + example = "OPENAI_API_KEY"; + description = "Explicit key variable for a custom provider or non-default alias."; + }; + providersFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Custom providers.json mounted read-only at HOME/.graphify/providers.json."; + }; + }; + }; + + serverOptions = {name, ...}: { + options = { + enable = mkEnableOption "HTTP MCP serving for ${name}"; + host = mkOption { + type = types.str; + default = "127.0.0.1"; + }; + port = mkOption { + type = types.port; + default = 8080; + }; + path = mkOption { + type = types.str; + default = "/mcp"; + }; + jsonResponse = mkOption { + type = types.bool; + default = false; + }; + stateless = mkOption { + type = types.bool; + default = false; + }; + sessionTimeout = mkOption { + type = types.addCheck types.number (value: value >= 0); + default = 3600; + }; + apiKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "MCP bearer key loaded through a systemd credential."; + }; + unsafeAllowUnauthenticated = mkOption { + type = types.bool; + default = false; + description = "Explicitly allow an unauthenticated non-loopback listener."; + }; + openFirewall = mkOption { + type = types.bool; + default = false; + }; + }; + }; + + sinkOptions = sink: {name, ...}: { + options = { + enable = mkEnableOption "${sink} export for ${name}"; + uri = mkOption { + type = types.nullOr types.str; + default = null; + example = + if sink == "neo4j" + then "bolt://127.0.0.1:7687" + else "falkordb://127.0.0.1:6379"; + description = "Graph database URI passed to Graphify's push exporter."; + }; + user = mkOption { + type = types.nullOr types.str; + default = + if sink == "neo4j" + then "neo4j" + else null; + description = "Optional graph database user."; + }; + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Password loaded through a systemd credential and provider-specific environment variable."; + }; + onExtraction = mkOption { + type = types.bool; + default = true; + description = "Push after each successful extraction."; + }; + onCalendar = mkOption { + type = types.nullOr types.str; + default = null; + description = "Optional independent systemd calendar schedule."; + }; + }; + }; + + instanceOptions = {name, ...}: { + options = { + stateDirectory = mkOption { + type = types.str; + default = "/var/lib/graphify/${name}"; + description = "Mutable extraction state; graph.json is stored below graphify-out/."; + }; + + source = { + path = mkOption { + type = types.nullOr types.str; + default = null; + description = "Optional absolute runtime corpus path."; + }; + cargo = mkOption { + type = types.bool; + default = false; + }; + postgresql = mkOption { + type = types.submodule postgresOptions; + default = {}; + }; + }; + + extraction = mkOption { + type = types.submodule extractionOptions; + default = {}; + }; + llm = mkOption { + type = types.submodule llmOptions; + default = {}; + }; + + watch = { + enable = mkEnableOption "filesystem watching for ${name}"; + debounce = mkOption { + type = types.addCheck types.number (value: value > 0); + default = 3.0; + }; + }; + + server = mkOption { + type = types.submodule serverOptions; + default = {}; + }; + + exports = { + neo4j = mkOption { + type = types.submodule (sinkOptions "neo4j"); + default = {}; + }; + falkordb = mkOption { + type = types.submodule (sinkOptions "falkordb"); + default = {}; + }; + }; + + environment = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Additional non-secret Graphify/provider environment variables."; + }; + + environmentFiles = mkOption { + type = types.listOf types.path; + default = []; + description = "Systemd environment files for provider/AWS/runtime settings and secrets."; + }; + + runtimePackages = mkOption { + type = types.listOf types.package; + default = []; + example = lib.literalExpression "[ pkgs.claude-code pkgs.gws ]"; + description = "External executables added to service PATH, such as claude for claude-cli or gws for Google Workspace export."; + }; + }; + }; + + enabledInstances = cfg.instances; + + graphOut = instance: "${instance.stateDirectory}/graphify-out"; + graphPath = instance: "${graphOut instance}/graph.json"; + + llmKeyVariable = instance: + if instance.llm.apiKeyEnvironmentVariable != null + then instance.llm.apiKeyEnvironmentVariable + else if instance.llm.backend != null + then backendApiKeyVariables.${instance.llm.backend} or null + else null; + + baseEnvironment = instance: let + pg = instance.source.postgresql; + backend = instance.llm.backend; + baseUrlVariable = + if backend != null + then backendBaseUrlVariables.${backend} or null + else null; + in + { + HOME = instance.stateDirectory; + PYTHONDONTWRITEBYTECODE = "1"; + } + // optionalAttrs pg.enable { + PGHOST = pg.host; + PGPORT = toString pg.port; + PGDATABASE = pg.database; + PGUSER = pg.user; + PGSSLMODE = pg.sslMode; + } + // optionalAttrs (instance.llm.baseUrl != null && baseUrlVariable != null) { + ${baseUrlVariable} = instance.llm.baseUrl; + } + // instance.environment; + + extractionArguments = name: instance: + ["extract"] + ++ optional (instance.source.path != null) instance.source.path + ++ optional (instance.llm.backend != null) "--backend=${instance.llm.backend}" + ++ optional (instance.llm.model != null) "--model=${instance.llm.model}" + ++ optional (instance.extraction.mode == "deep") "--mode=deep" + ++ ["--out=${instance.stateDirectory}"] + ++ optional instance.extraction.noCluster "--no-cluster" + ++ optional instance.extraction.dedup "--dedup-llm" + ++ optional instance.extraction.codeOnly "--code-only" + ++ optional instance.extraction.googleWorkspace "--google-workspace" + ++ optional instance.extraction.global "--global" + ++ optional (instance.extraction.tag != null) "--as=${instance.extraction.tag}" + ++ optional (instance.extraction.maxWorkers != null) "--max-workers=${toString instance.extraction.maxWorkers}" + ++ optional (instance.extraction.tokenBudget != null) "--token-budget=${toString instance.extraction.tokenBudget}" + ++ optional (instance.extraction.maxConcurrency != null) "--max-concurrency=${toString instance.extraction.maxConcurrency}" + ++ optional (instance.extraction.apiTimeout != null) "--api-timeout=${toString instance.extraction.apiTimeout}" + ++ ["--resolution=${toString instance.extraction.resolution}"] + ++ optional (instance.extraction.excludeHubs != null) "--exclude-hubs=${toString instance.extraction.excludeHubs}" + ++ flatten (map (value: ["--exclude" value]) instance.extraction.excludes) + ++ optional instance.source.postgresql.enable "--postgres=" + ++ optional instance.source.cargo "--cargo" + ++ optional instance.extraction.timing "--timing"; + + credentialSetup = instance: let + keyVariable = llmKeyVariable instance; + in '' + ${optionalString (instance.source.postgresql.pgpassFile != null) '' + export PGPASSFILE="$CREDENTIALS_DIRECTORY/postgresql-pgpass" + ''} + ${optionalString (instance.llm.apiKeyFile != null && keyVariable != null) '' + export ${keyVariable}="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/llm-api-key")" + ''} + ''; + + extractionScript = name: instance: + pkgs.writeShellScript "graphify-${name}-extract" '' + set -eu + ${credentialSetup instance} + exec ${lib.getExe cfg.package} ${escapeShellArgs (extractionArguments name instance)} + ''; + + serverScript = name: instance: + pkgs.writeShellScript "graphify-${name}-mcp" '' + set -eu + ${optionalString (instance.server.apiKeyFile != null) '' + export GRAPHIFY_API_KEY="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/mcp-api-key")" + ''} + exec ${cfg.package}/bin/graphify-mcp ${escapeShellArgs ([ + (graphPath instance) + "--transport=http" + "--host=${instance.server.host}" + "--port=${toString instance.server.port}" + "--path=${instance.server.path}" + "--session-timeout=${toString instance.server.sessionTimeout}" + ] + ++ optional instance.server.jsonResponse "--json-response" + ++ optional instance.server.stateless "--stateless")} + ''; + + exportScript = name: instance: sink: sinkCfg: let + passwordVariable = + if sink == "neo4j" + then "NEO4J_PASSWORD" + else "FALKORDB_PASSWORD"; + in + pkgs.writeShellScript "graphify-${name}-${sink}-export" '' + set -eu + ${optionalString (sinkCfg.passwordFile != null) '' + export ${passwordVariable}="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/graph-database-password")" + ''} + exec ${lib.getExe cfg.package} ${escapeShellArgs ([ + "export" + sink + "--graph" + (graphPath instance) + "--push" + sinkCfg.uri + ] + ++ optional (sinkCfg.user != null) "--user=${sinkCfg.user}")} + ''; + + commonServiceConfig = instance: { + User = cfg.user; + Group = cfg.group; + UMask = "0077"; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = "read-only"; + ReadWritePaths = [instance.stateDirectory]; + ReadOnlyPaths = optional (instance.source.path != null) instance.source.path; + EnvironmentFile = map toString instance.environmentFiles; + }; + + instanceAssertions = concatLists (mapAttrsToList (name: instance: let + pg = instance.source.postgresql; + server = instance.server; + loopback = builtins.elem server.host ["127.0.0.1" "::1" "localhost"]; + keyVariable = llmKeyVariable instance; + sinkAssertions = concatLists (mapAttrsToList (sink: sinkCfg: [ + { + assertion = !sinkCfg.enable || sinkCfg.uri != null; + message = "services.graphify.instances.${name}.exports.${sink}.uri is required when the sink is enabled."; + } + { + assertion = !sinkCfg.enable || !sinkCfg.onExtraction || instance.extraction.enable; + message = "services.graphify.instances.${name}.exports.${sink}.onExtraction requires extraction.enable."; + } + { + assertion = sink != "neo4j" || !sinkCfg.enable || sinkCfg.passwordFile != null; + message = "services.graphify.instances.${name}.exports.neo4j.passwordFile is required by Neo4j push."; + } + ]) + instance.exports); + in + [ + { + assertion = !instance.extraction.enable || instance.source.path != null || pg.enable; + message = "services.graphify.instances.${name} needs source.path and/or source.postgresql.enable when extraction is enabled."; + } + { + assertion = !pg.enable || pg.database != null; + message = "services.graphify.instances.${name}.source.postgresql.database is required when PostgreSQL introspection is enabled."; + } + { + assertion = !instance.source.cargo || instance.source.path != null; + message = "services.graphify.instances.${name}.source.cargo requires source.path."; + } + { + assertion = !instance.watch.enable || instance.source.path != null; + message = "services.graphify.instances.${name}.watch.enable requires source.path."; + } + { + assertion = !instance.watch.enable || instance.extraction.enable; + message = "services.graphify.instances.${name}.watch.enable requires extraction.enable for the initial graph."; + } + { + assertion = lib.hasPrefix "/" instance.stateDirectory; + message = "services.graphify.instances.${name}.stateDirectory must be absolute."; + } + { + assertion = instance.source.path == null || lib.hasPrefix "/" instance.source.path; + message = "services.graphify.instances.${name}.source.path must be absolute."; + } + { + assertion = !(instance.environment ? GRAPHIFY_OUT) && !(instance.environment ? HOME); + message = "services.graphify.instances.${name}.environment must not override module-owned GRAPHIFY_OUT or HOME paths."; + } + { + assertion = !server.enable || lib.hasPrefix "/" server.path; + message = "services.graphify.instances.${name}.server.path must begin with '/'."; + } + { + assertion = !server.enable || loopback || server.apiKeyFile != null || server.unsafeAllowUnauthenticated; + message = "services.graphify.instances.${name} refuses unauthenticated non-loopback MCP; set server.apiKeyFile or unsafeAllowUnauthenticated."; + } + { + assertion = instance.llm.backend == null || instance.llm.customProvider || builtins.elem instance.llm.backend builtInBackends; + message = "services.graphify.instances.${name}.llm.backend is not built in; set llm.customProvider for providersFile entries."; + } + { + assertion = !instance.llm.customProvider || instance.llm.providersFile != null; + message = "services.graphify.instances.${name}.llm.customProvider requires llm.providersFile."; + } + { + assertion = instance.llm.apiKeyFile == null || keyVariable != null; + message = "services.graphify.instances.${name}.llm.apiKeyFile requires a known backend or apiKeyEnvironmentVariable."; + } + ] + ++ sinkAssertions) + enabledInstances); +in { + options.services.graphify = { + enable = mkEnableOption "Graphify extraction and MCP services"; + + package = mkOption { + type = types.nullOr types.package; + default = graphifyPackage; + defaultText = lib.literalExpression "graphify.packages..full"; + description = "Graphify package. The upstream module supplies the full runtime package."; + }; + + user = mkOption { + type = types.str; + default = "graphify"; + }; + group = mkOption { + type = types.str; + default = "graphify"; + }; + instances = mkOption { + type = types.attrsOf (types.submodule instanceOptions); + default = {}; + description = "Named Graphify graphs with independent sources, state, schedules, and MCP listeners."; + }; + }; + + config = mkIf cfg.enable (mkMerge [ + { + assertions = + [ + { + assertion = cfg.package != null; + message = "services.graphify.package must be set when Graphify is enabled."; + } + { + assertion = cfg.instances != {}; + message = "services.graphify.instances must contain at least one instance."; + } + ] + ++ instanceAssertions; + + users.groups.${cfg.group} = {}; + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + home = "/var/lib/graphify"; + createHome = true; + }; + + systemd.tmpfiles.rules = flatten (mapAttrsToList (_: instance: [ + "d ${instance.stateDirectory} 0750 ${cfg.user} ${cfg.group} - -" + "d ${instance.stateDirectory}/.graphify 0750 ${cfg.user} ${cfg.group} - -" + ]) + enabledInstances); + + networking.firewall.allowedTCPPorts = + unique (mapAttrsToList (_: instance: instance.server.port) + (filterAttrs (_: instance: instance.server.enable && instance.server.openFirewall) enabledInstances)); + } + + { + systemd.services = mkMerge (mapAttrsToList (name: instance: let + extractUnit = "graphify-${name}-extract"; + pgService = instance.source.postgresql.systemdService; + providerBind = + optional (instance.llm.providersFile != null) + "${toString instance.llm.providersFile}:${instance.stateDirectory}/.graphify/providers.json"; + in + mkMerge ([ + (mkIf instance.extraction.enable { + ${extractUnit} = { + description = "Extract Graphify graph ${name}"; + wantedBy = optional instance.extraction.startAtBoot "multi-user.target"; + after = optional (pgService != null) pgService; + requires = optional (pgService != null) pgService; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.OnSuccess = + mapAttrsToList + (sink: _: "graphify-${name}-${sink}.service") + (filterAttrs (_: sinkCfg: sinkCfg.enable && sinkCfg.onExtraction) instance.exports); + serviceConfig = + commonServiceConfig instance + // { + Type = "oneshot"; + ExecStart = extractionScript name instance; + LoadCredential = + optional (instance.source.postgresql.pgpassFile != null) "postgresql-pgpass:${toString instance.source.postgresql.pgpassFile}" + ++ optional (instance.llm.apiKeyFile != null) "llm-api-key:${toString instance.llm.apiKeyFile}"; + BindReadOnlyPaths = providerBind; + }; + }; + }) + (mkIf instance.watch.enable { + "graphify-${name}-watch" = { + description = "Watch Graphify corpus ${name}"; + wantedBy = ["multi-user.target"]; + after = ["${extractUnit}.service"]; + requires = ["${extractUnit}.service"]; + environment = baseEnvironment instance // {GRAPHIFY_OUT = graphOut instance;}; + path = instance.runtimePackages; + serviceConfig = + commonServiceConfig instance + // { + ExecStart = "${lib.getExe cfg.package} watch ${lib.escapeShellArg instance.source.path} --debounce ${toString instance.watch.debounce}"; + Restart = "on-failure"; + RestartSec = "5s"; + }; + }; + }) + (mkIf instance.server.enable { + "graphify-${name}" = { + description = "Graphify MCP server ${name}"; + wantedBy = ["multi-user.target"]; + after = optional instance.extraction.enable "${extractUnit}.service"; + requires = optional instance.extraction.enable "${extractUnit}.service"; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.ConditionPathExists = graphPath instance; + serviceConfig = + commonServiceConfig instance + // { + ExecStart = serverScript name instance; + Restart = "on-failure"; + RestartSec = "5s"; + LoadCredential = optional (instance.server.apiKeyFile != null) "mcp-api-key:${toString instance.server.apiKeyFile}"; + }; + }; + }) + ] + ++ mapAttrsToList (sink: sinkCfg: + mkIf sinkCfg.enable { + "graphify-${name}-${sink}" = { + description = "Push Graphify graph ${name} to ${sink}"; + after = optional instance.extraction.enable "${extractUnit}.service"; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.ConditionPathExists = graphPath instance; + serviceConfig = + commonServiceConfig instance + // { + Type = "oneshot"; + ExecStart = exportScript name instance sink sinkCfg; + LoadCredential = optional (sinkCfg.passwordFile != null) "graph-database-password:${toString sinkCfg.passwordFile}"; + }; + }; + }) + instance.exports)) + enabledInstances); + + systemd.timers = mkMerge ([ + (mapAttrs' (name: instance: + nameValuePair "graphify-${name}-extract" { + wantedBy = ["timers.target"]; + timerConfig = { + OnCalendar = instance.extraction.onCalendar; + Persistent = true; + Unit = "graphify-${name}-extract.service"; + }; + }) (filterAttrs (_: instance: instance.extraction.enable && instance.extraction.onCalendar != null) enabledInstances)) + ] + ++ flatten (mapAttrsToList (name: instance: + mapAttrsToList (sink: sinkCfg: + mkIf (sinkCfg.enable && sinkCfg.onCalendar != null) { + "graphify-${name}-${sink}" = { + wantedBy = ["timers.target"]; + timerConfig = { + OnCalendar = sinkCfg.onCalendar; + Persistent = true; + Unit = "graphify-${name}-${sink}.service"; + }; + }; + }) + instance.exports) + enabledInstances)); + } + ]); +} diff --git a/pyproject.toml b/pyproject.toml index b8ea4ad31..e4784f00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/test_backend_extras.py b/tests/test_backend_extras.py index f513c57dc..4da79a007 100644 --- a/tests/test_backend_extras.py +++ b/tests/test_backend_extras.py @@ -34,6 +34,12 @@ def test_anthropic_in_all_extra(): assert any("anthropic" in dep for dep in extras["all"]), "[all] must include anthropic" +def test_postgres_in_all_extra(): + extras = _extras() + assert "psycopg[binary]" in extras["postgres"] + assert "psycopg[binary]" in extras["all"], "[all] must include PostgreSQL support" + + def test_backend_pkg_hint_points_at_uv_tool_and_extra(): msg = _backend_pkg_hint("anthropic", "anthropic") assert "uv tool install" in msg diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index 4133002c0..dc02a65b3 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -1,13 +1,84 @@ """Tests for `graphify extract` CLI dispatch path in graphify.__main__.""" from __future__ import annotations -import os +import json import pytest import graphify.__main__ as mainmod +def _postgres_result(): + """Minimal valid database extraction returned by the introspection stub.""" + source = "postgresql:/db.example.test/app" + return { + "nodes": [ + { + "id": "postgresql:app:public.widgets", + "label": "public.widgets", + "type": "table", + "file_type": "code", + "source_file": source, + "source_location": f"{source}:1", + "confidence": "EXTRACTED", + } + ], + "edges": [], + } + + +@pytest.mark.parametrize("with_path", [False, True], ids=["postgres-only", "path-and-postgres"]) +def test_extract_postgres_reaches_introspection_and_writes_output( + monkeypatch, tmp_path, with_path +): + """PostgreSQL-only mode must initialize the empty detection result. + + The no-path form previously skipped ``detect()`` as intended, then crashed + with ``UnboundLocalError`` when common reporting code read ``detection``. + The path form is covered alongside it to preserve combined source + schema + extraction while fixing the database-only branch. + """ + out_dir = tmp_path / "out" + argv = ["graphify", "extract"] + if with_path: + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def application_entry():\n return 1\n") + argv.append(str(corpus)) + argv.extend( + [ + "--postgres", + "postgresql://reader:secret@db.example.test/app", + "--out", + str(out_dir), + "--no-cluster", + ] + ) + + calls = [] + + def _introspect(dsn): + calls.append(dsn) + return _postgres_result() + + monkeypatch.setattr("graphify.pg_introspect.introspect_postgres", _introspect) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + + with pytest.raises(SystemExit) as exc_info: + mainmod.main() + + assert exc_info.value.code == 0 + assert calls == ["postgresql://reader:secret@db.example.test/app"] + + graph_path = out_dir / "graphify-out" / "graph.json" + graph = json.loads(graph_path.read_text(encoding="utf-8")) + labels = {node.get("label") for node in graph["nodes"]} + assert "public.widgets" in labels + if with_path: + assert any(str(label).startswith("application_entry") for label in labels) + + def _make_corpus(tmp_path): """Minimal corpus: one Go code file + one Markdown doc. @@ -104,15 +175,6 @@ def _one_chunk_succeeded(paths, **kwargs): monkeypatch.setattr( "graphify.llm.extract_corpus_parallel", _one_chunk_succeeded ) - cache_call = {} - - def _capture_semantic_cache(*args, **kwargs): - cache_call.update(kwargs) - return 0 - - monkeypatch.setattr( - "graphify.cache.save_semantic_cache", _capture_semantic_cache - ) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -132,450 +194,6 @@ def _capture_semantic_cache(*args, **kwargs): assert (out_dir / "graphify-out" / "graph.json").exists(), ( "graph.json must be written on the happy path" ) - assert { - str(path) for path in cache_call["allowed_source_files"] - } == {str(corpus / "README.md")} - - -def test_incremental_partial_run_preserves_untouched_semantic_hash( - monkeypatch, tmp_path -): - """#1948 caller-side guard: an incremental run that only re-dispatches the - CHANGED subset must not blank semantic_hash for live-but-untouched files. - - clear_semantic must be derived from what was actually SENT to the backend - this run (semantic_files), not from the full live corpus (files_by_type): - with the latter, every unchanged doc lands in the clear set on every - incremental run, so the very next run re-extracts the whole corpus, - forever.""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - (corpus / "OTHER.md").write_text("# Other\nAn independent second doc.\n") - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - dispatched: list[list[str]] = [] - - def _stamp_everything_sent(paths, **kwargs): - sent = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) - dispatched.append(sent) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [{"id": f"n-{rel}", "source_file": rel, - "file_type": "document"} for rel in sent], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr( - "graphify.llm.extract_corpus_parallel", _stamp_everything_sent - ) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _run_extract(): - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - # Run 1: full scan — both docs dispatched and stamped. - _run_extract() - manifest_path = out_dir / "graphify-out" / "manifest.json" - m1 = json.loads(manifest_path.read_text()) - assert m1["README.md"].get("semantic_hash") - assert m1["OTHER.md"].get("semantic_hash") - - # Run 2: only README.md changes → the incremental gate dispatches it alone. - (corpus / "README.md").write_text("# Notes\nChanged content, new hash.\n") - _run_extract() - assert dispatched[-1] == ["README.md"], ( - f"run 2 should dispatch only the changed doc, got {dispatched[-1]}" - ) - m2 = json.loads(manifest_path.read_text()) - assert m2["README.md"].get("semantic_hash") - # The heart of the guard: an untouched, never-dispatched live doc keeps - # its stamp across a partial incremental run. - assert m2["OTHER.md"].get("semantic_hash"), ( - "untouched doc's semantic_hash was blanked by a partial incremental " - "run — clear_semantic was derived from the full live corpus instead " - "of the dispatched subset (#1948)" - ) - - -def test_truncated_doc_semantic_hash_is_cleared_for_requeue(monkeypatch, tmp_path): - """#1948 x #1950 interaction: a doc stamped complete on a prior run that - TRUNCATES (partial) this run must have its stale semantic_hash cleared, so - detect_incremental re-queues it — not inherit the old hash and look - unchanged. Partial files are dropped by _stamped_manifest_files, so they - land in clear_semantic (dispatched-but-not-stamped).""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - partial_run = {"on": False} - - def _extract(paths, **kwargs): - rels = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - node = {"id": "n-readme", "source_file": "README.md", "file_type": "document"} - if partial_run["on"] and "README.md" in rels: - node["_partial"] = True # this run truncated README.md - return {"nodes": [node] if "README.md" in rels else [], - "edges": [], "hyperedges": [], "input_tokens": 10, "output_tokens": 5} - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _extract) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _run(): - monkeypatch.setattr(mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)]) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0) - - manifest_path = out_dir / "graphify-out" / "manifest.json" - _run() # run 1: complete - assert json.loads(manifest_path.read_text())["README.md"].get("semantic_hash") - - # run 2: README.md changes and truncates (partial) this time. - (corpus / "README.md").write_text("# Notes\nNew, longer content that truncated.\n") - partial_run["on"] = True - _run() - m2 = json.loads(manifest_path.read_text()) - assert not m2.get("README.md", {}).get("semantic_hash"), ( - "a truncated doc's stale semantic_hash must be cleared so it is " - "re-queued next run (#1948 x #1950)" - ) - - -def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): - """#1897: fresh extraction returns nodes with ROOT-RELATIVE source_file, - while the #933 manifest filter compared them against detect()'s ABSOLUTE - paths — so `f in _sem_extracted` was always False and every freshly - extracted doc was dropped from the manifest (only code/zero-node files - survived). Both sides must be resolved against the scan root; a genuinely - omitted doc (zero nodes) must still stay unstamped (#933 is intentional).""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - (corpus / "OMITTED.md").write_text("# never extracted\n") - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - def _fresh_relative(paths, **kwargs): - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - # Root-relative source_file, exactly what a fresh extraction produces. - # OMITTED.md gets no nodes/edges — the model skipped it. - return { - "nodes": [{"id": "readme", "source_file": "README.md", - "file_type": "document"}], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _fresh_relative) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert manifest_path.exists() - manifest = json.loads(manifest_path.read_text()) - - assert "README.md" in manifest, ( - f"freshly-extracted doc missing from manifest (#1897): {sorted(manifest)}" - ) - assert manifest["README.md"].get("semantic_hash"), ( - "freshly-extracted doc must carry a non-empty semantic_hash" - ) - # Code files are always stamped. - assert manifest.get("main.go", {}).get("semantic_hash") - # The zero-node doc stays unstamped so detect_incremental re-queues it (#933). - assert "OMITTED.md" not in manifest, ( - "zero-node doc must not be stamped in the manifest" - ) - - -def test_stamped_manifest_files_normalizes_both_sides(tmp_path): - """Unit test for the #1897 helper: relative (fresh) and absolute (cache-hit) - source_file values must both match detect()'s absolute file lists; docs with - no output are filtered; code files pass through untouched.""" - from graphify.cli import _stamped_manifest_files - - fresh_doc = tmp_path / "fresh.md"; fresh_doc.write_text("# fresh") - cached_doc = tmp_path / "cached.md"; cached_doc.write_text("# cached") - omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") - code = tmp_path / "app.py"; code.write_text("x = 1") - - files_by_type = { - "code": [str(code)], - "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], - } - sem_result = { - # fresh extraction: root-relative source_file - "nodes": [{"id": "n1", "source_file": "fresh.md"}], - # cache replay: absolute source_file (edge-only coverage counts too) - "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], - } - - out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) - assert out["code"] == [str(code)] - assert out["document"] == [str(fresh_doc), str(cached_doc)] - - -def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): - """#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a - concept) is valid output — the semantic cache persists it per source_file — - so it must be stamped. Before the fix the stamping loop only inspected - ``nodes``/``edges``, leaving such a doc unstamped and re-queued forever.""" - from graphify.cli import _stamped_manifest_files - - hyper_doc = tmp_path / "hyper.md"; hyper_doc.write_text("# hyper") - omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") - - files_by_type = {"document": [str(hyper_doc), str(omitted_doc)]} - sem_result = { - "nodes": [], - "edges": [], - "hyperedges": [ - {"id": "h1", "label": "L", "nodes": ["a", "b", "c"], - "relation": "participate_in", "source_file": "hyper.md"}, - ], - } - - out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) - assert str(hyper_doc) in out["document"], ( - "a hyperedge-only doc must be stamped (#1920)" - ) - # A doc with no output at all still stays unstamped (#933). - assert str(omitted_doc) not in out["document"] - - -def test_manifest_stamps_hyperedge_only_docs(monkeypatch, tmp_path): - """#1920 end-to-end: a fresh extraction whose only output for a doc is a - hyperedge stamps that doc's semantic_hash, so it is not re-dispatched.""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - def _hyperedge_only(paths, **kwargs): - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [], - "edges": [], - "hyperedges": [{"id": "h1", "label": "Shared", "nodes": ["a", "b", "c"], - "relation": "participate_in", "source_file": "README.md"}], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _hyperedge_only) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - manifest = json.loads((out_dir / "graphify-out" / "manifest.json").read_text()) - assert manifest.get("README.md", {}).get("semantic_hash"), ( - f"hyperedge-only doc must be stamped (#1920): {sorted(manifest)}" - ) - - -# --- #1894: --force and deep-mode dispatch over a warm cache ----------------- - -def _recording_extractor(calls): - """extract_corpus_parallel stand-in that records each dispatch.""" - def _extract(paths, **kwargs): - calls.append({"paths": [str(p) for p in paths], "kwargs": kwargs}) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [{"id": "readme", "source_file": "README.md", - "file_type": "document"}], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - return _extract - - -def _run_extract(monkeypatch, argv): - monkeypatch.setattr(mainmod.sys, "argv", argv) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - -def test_extract_mode_deep_dispatches_over_warm_cache(monkeypatch, tmp_path): - """#1894 repro: over a warm manifest + warm standard semantic cache, - `extract --mode deep` was a silent no-op — the incremental gate dispatched - zero files before the cache was ever consulted, and the cache key ignored - mode anyway. Deep must re-dispatch on the first deep run (deep namespace - cold) and be served from cache/semantic-deep/ on the second.""" - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - # No --out: the default layout (graphify-out/ beside the sources) keeps the - # CLI-level cache write's root anchored at the corpus, so the stub's - # root-relative source_file resolves (real runs also checkpoint per chunk - # inside llm.extract_corpus_parallel, which this stub replaces). - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - # Run 1: cold standard extraction — warms manifest + plain semantic cache. - _run_extract(monkeypatch, base) - assert len(calls) == 1 - - # Sanity: a warm standard re-run dispatches nothing (expected behavior). - _run_extract(monkeypatch, base) - assert len(calls) == 1 - - # The repro: warm tree + --mode deep MUST dispatch. - _run_extract(monkeypatch, base + ["--mode", "deep"]) - assert len(calls) == 2, ( - "--mode deep over a warm cache must re-dispatch (#1894)" - ) - assert calls[1]["paths"] == [str(corpus / "README.md")] - assert calls[1]["kwargs"].get("deep_mode") is True - - # Second deep run: served from the (now warm) deep namespace, no dispatch. - _run_extract(monkeypatch, base + ["--mode", "deep"]) - assert len(calls) == 2, ( - "second deep run must be served from cache/semantic-deep/" - ) - # The deep entry landed in its own namespace, not cache/semantic/. Entries are - # nested under a p{prompt-fingerprint}/ subdir (#1939), hence the recursive glob. - assert any((corpus / "graphify-out" / "cache" / "semantic-deep").glob("**/*.json")) - - -def test_extract_force_flag_redispatches_and_stamps_manifest(monkeypatch, tmp_path): - """extract accepts --force: a warm tree re-dispatches every semantic file - (cache read skipped, incremental gate off) and the manifest is still - stamped afterward (#1897-compatible full coverage).""" - import json - - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - _run_extract(monkeypatch, base) - assert len(calls) == 1 - _run_extract(monkeypatch, base) # warm: no dispatch - assert len(calls) == 1 - - _run_extract(monkeypatch, base + ["--force"]) - assert len(calls) == 2, ( - "--force over a warm tree must re-dispatch every semantic file" - ) - assert calls[1]["paths"] == [str(corpus / "README.md")] - - # The forced run still wrote the semantic cache and stamped the manifest. - # Entries nest under a p{prompt-fingerprint}/ subdir (#1939). - assert any((corpus / "graphify-out" / "cache" / "semantic").glob("**/*.json")) - manifest = json.loads( - (corpus / "graphify-out" / "manifest.json").read_text() - ) - assert manifest.get("README.md", {}).get("semantic_hash"), ( - "forced re-dispatch must still stamp the manifest" - ) - assert manifest.get("main.go", {}).get("semantic_hash") - - -def test_extract_graphify_force_env_redispatches(monkeypatch, tmp_path): - """GRAPHIFY_FORCE=1 behaves like --force (env parity with `update`).""" - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - _run_extract(monkeypatch, base) - assert len(calls) == 1 - _run_extract(monkeypatch, base) # warm: no dispatch - assert len(calls) == 1 - - monkeypatch.setenv("GRAPHIFY_FORCE", "1") - _run_extract(monkeypatch, base) - assert len(calls) == 2, "GRAPHIFY_FORCE=1 must force a re-dispatch" - - -def test_cache_check_mode_deep_reads_deep_namespace(monkeypatch, tmp_path, capsys): - """cache-check --mode deep consults cache/semantic-deep/; without the flag - it keeps reading cache/semantic/ (deep entries are invisible to it).""" - from graphify.cache import save_semantic_cache - - doc = tmp_path / "doc.md" - doc.write_text("# Doc\n") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - files_from = tmp_path / "files.txt" - files_from.write_text(str(doc) + "\n") - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), - "--root", str(tmp_path)]) - assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out - - _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), - "--root", str(tmp_path), "--mode", "deep"]) - assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out def _code_only_corpus(tmp_path): @@ -627,73 +245,6 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): assert len(json.loads(graph.read_text()).get("nodes", [])) > 0 -def test_missing_manifest_code_only_preserves_semantic_layer(monkeypatch, tmp_path): - """#1925: `graphify extract --code-only` with a MISSING manifest.json must - not degrade to a full scan that discards the committed semantic layer. An - existing graph.json is a sufficient incremental baseline, so doc/paper/image - nodes (excluded by --code-only, not deleted) are preserved; a genuinely - deleted source is still evicted (#1909 semantics retained).""" - import json - - corpus = tmp_path / "proj"; corpus.mkdir() - (corpus / "keep.py").write_text("def keep():\n return 1\n") - (corpus / "README.md").write_text("# Notes\nCurated docs.\n") - out_dir = tmp_path / "out" - graphify_out = out_dir / "graphify-out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _sem_doc_count(g): - return sum(1 for n in g["nodes"] if n.get("source_file") == "README.md") - - # 1) seed a code-only graph - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - graph_path = graphify_out / "graph.json" - graph = json.loads(graph_path.read_text()) - - # 2) inject a committed semantic layer for README.md (nodes + edge + hyperedge) - graph["nodes"].append({"id": "doc_readme_a", "label": "Concept A", - "source_file": "README.md", "file_type": "document"}) - graph["nodes"].append({"id": "doc_readme_b", "label": "Concept B", - "source_file": "README.md", "file_type": "document"}) - graph.setdefault("edges", []).append( - {"source": "doc_readme_a", "target": "doc_readme_b", - "relation": "relates_to", "source_file": "README.md"}) - graph.setdefault("hyperedges", []).append( - {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], - "relation": "participate_in", "source_file": "README.md"}) - graph_path.write_text(json.dumps(graph)) - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": 1})) - - # 3) manifest goes missing (fresh clone / deliberately untracked) - (graphify_out / "manifest.json").unlink() - - # 4) re-run the SAME code-only extract - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - after = json.loads(graph_path.read_text()) - assert _sem_doc_count(after) >= 2, ( - "committed semantic doc nodes must survive a missing-manifest " - f"--code-only rebuild (#1925); got {_sem_doc_count(after)}" - ) - assert any(h.get("id") == "h1" for h in after.get("hyperedges", [])), ( - "committed hyperedge must survive the rebuild" - ) - assert any("keep" in n["id"] for n in after["nodes"]), "code nodes intact" - - # 5) a genuine deletion still evicts the doc's semantic nodes - (corpus / "README.md").unlink() - (graphify_out / "manifest.json").unlink(missing_ok=True) - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - gone = json.loads(graph_path.read_text()) - assert _sem_doc_count(gone) == 0, ( - "a genuinely deleted doc must still be evicted (#1909 semantics preserved)" - ) - - def test_extract_out_keeps_project_root_clean(monkeypatch, tmp_path): """`extract --out DIR` routes every artifact to DIR/graphify-out/ and the scanned project must not grow a graphify-out/ (or anything else) beside @@ -787,197 +338,3 @@ def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): mainmod.main() assert exc2.value.code == 0 assert "graphify timing" not in capsys.readouterr().err - - -# --------------------------------------------------------------------------- -# #1909: a newly-excluded file's nodes must be pruned from graph.json on the -# next incremental extract even when the manifest never listed the file (the -# pre-#1897 state every 0.9.16 graph is in), so the manifest-diff prune set -# (`manifest - corpus`) can never see it. -# --------------------------------------------------------------------------- - -def _two_file_corpus(tmp_path): - project = tmp_path / "project" - project.mkdir() - (project / "x.py").write_text( - "def secret_helper():\n return 42\n\n" - "def secret_caller():\n return secret_helper()\n" - ) - (project / "keep.py").write_text( - "def kept():\n return still_here()\n\n" - "def still_here():\n return 1\n" - ) - return project - - -def _node_sources(graph_path): - import json - data = json.loads(graph_path.read_text(encoding="utf-8")) - return {n.get("source_file", "") for n in data.get("nodes", [])} - - -def _run_extract(monkeypatch, argv): - monkeypatch.setattr(mainmod.sys, "argv", argv) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - -def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest( - monkeypatch, tmp_path -): - """Seed a graph with nodes for x.py, drop x.py from the manifest (pre-#1897 - manifests never listed excluded/omitted files), exclude x.py via - .graphifyignore, re-run extract: x.py's nodes must be gone even though it - was never on the deleted list.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - graph_path = out_dir / "graphify-out" / "graph.json" - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert any("x.py" in s for s in _node_sources(graph_path)), ( - "seed extract must produce nodes for x.py" - ) - - # Simulate the pre-#1897 manifest state: x.py was never manifest-listed, - # so `manifest - corpus` can never flag it. - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = {k: v for k, v in manifest.items() if "x.py" not in k} - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - - (project / ".graphifyignore").write_text("x.py\n") - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources), ( - f"newly-excluded x.py must be pruned from graph.json, still see {sources}" - ) - assert any("keep.py" in s for s in sources), ( - "unchanged keep.py nodes must survive the incremental merge" - ) - # x.py exists on disk, is excluded, and must not creep into the manifest. - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - assert not any("x.py" in k for k in manifest), ( - f"excluded x.py must not be (re)listed in the manifest: {set(manifest)}" - ) - - -def test_incremental_extract_prunes_excluded_file_listed_in_manifest( - monkeypatch, tmp_path -): - """Post-#1897 state: the excluded file IS manifest-listed. It must be - pruned from graph.json AND dropped from the manifest (#1908), and stay - settled on a further run.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - graph_path = out_dir / "graphify-out" / "graph.json" - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert any("x.py" in k for k in json.loads(manifest_path.read_text())) - - (project / ".graphifyignore").write_text("x.py\n") - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources) - assert any("keep.py" in s for s in sources) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - assert not any("x.py" in k for k in manifest), ( - "excluded-but-alive manifest row must be pruned (#1908)" - ) - - # Steady state: a third run neither resurrects x.py nor loses keep.py. - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources) - assert any("keep.py" in s for s in sources) - - -def test_no_cluster_incremental_prunes_newly_excluded_file( - monkeypatch, tmp_path, capsys -): - """--no-cluster's exclusion-only early exit must still scrub the excluded - file's nodes from the raw graph.json (that path never runs build_merge), - and must not report the alive file as deleted.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(project), "--no-cluster", "--out", str(out_dir)], - ) - with pytest.raises(SystemExit) as exc: - mainmod.main() - assert exc.value.code == 0 - graph_path = out_dir / "graphify-out" / "graph.json" - assert any("x.py" in s for s in _node_sources(graph_path)) - capsys.readouterr() - - (project / ".graphifyignore").write_text("x.py\n") - with pytest.raises(SystemExit) as exc: - mainmod.main() - assert exc.value.code == 0 - out_text = capsys.readouterr().out - assert "1 deleted" not in out_text, ( - "excluded-but-alive file must not be reported as deleted" - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources), ( - f"--no-cluster early exit must prune excluded sources, still see {sources}" - ) - assert any("keep.py" in s for s in sources) - - -def test_cache_check_prompt_file_scopes_hits_to_that_prompt(monkeypatch, tmp_path, capsys): - """#1939: cache-check --prompt-file only counts entries produced by that same - extraction prompt, so an upgraded prompt reports a miss (re-extract) rather - than replaying the older vintage.""" - from graphify.cache import save_semantic_cache - - doc = tmp_path / "doc.md" - doc.write_text("# Doc\n") - spec = tmp_path / "extraction-spec.md" - spec.write_text("PROMPT V1", encoding="utf-8") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, prompt_file=str(spec)) - files_from = tmp_path / "files.txt" - files_from.write_text(str(doc) + "\n") - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "cache-check", str(files_from), "--root", str(tmp_path)] - _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) - assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out - - # An upgrade rewrites the prompt: the entry must no longer satisfy the run. - spec.write_text("PROMPT V2 — rewritten by an upgrade", encoding="utf-8") - os.utime(spec, ns=(0, 0)) - _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) - assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out diff --git a/uv.lock b/uv.lock index 088ebbbdc..2fd137000 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.13" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1141,6 +1141,7 @@ all = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "openai" }, { name = "openpyxl" }, + { name = "psycopg", extra = ["binary"] }, { name = "pypdf" }, { name = "python-docx" }, { name = "starlette" }, @@ -1280,6 +1281,7 @@ requires-dist = [ { name = "openpyxl", marker = "extra == 'all'" }, { name = "openpyxl", marker = "extra == 'google'" }, { name = "openpyxl", marker = "extra == 'office'" }, + { name = "psycopg", extras = ["binary"], marker = "extra == 'all'" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'" }, { name = "pypdf", marker = "extra == 'all'", specifier = ">=6.12.0" }, { name = "pypdf", marker = "extra == 'pdf'", specifier = ">=6.12.0" }, From f0f66f5c31931481d45e0091e13913a04bed458a Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Mon, 13 Jul 2026 12:01:56 +0200 Subject: [PATCH 11/18] fix: keep graphify servers independent of extraction jobs --- nix/nixos-module.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix index 6c35274fe..b15a82b45 100644 --- a/nix/nixos-module.nix +++ b/nix/nixos-module.nix @@ -679,7 +679,6 @@ in { description = "Watch Graphify corpus ${name}"; wantedBy = ["multi-user.target"]; after = ["${extractUnit}.service"]; - requires = ["${extractUnit}.service"]; environment = baseEnvironment instance // {GRAPHIFY_OUT = graphOut instance;}; path = instance.runtimePackages; serviceConfig = @@ -696,7 +695,6 @@ in { description = "Graphify MCP server ${name}"; wantedBy = ["multi-user.target"]; after = optional instance.extraction.enable "${extractUnit}.service"; - requires = optional instance.extraction.enable "${extractUnit}.service"; environment = baseEnvironment instance; path = instance.runtimePackages; unitConfig.ConditionPathExists = graphPath instance; From be1262c519878ba678d5c8b7b4ec222bdb47f2c1 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Thu, 16 Jul 2026 10:09:15 +0200 Subject: [PATCH 12/18] fix: repair OpenCode plugin registration --- graphify/install.py | 16 ++++++++++++---- tests/test_install.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/graphify/install.py b/graphify/install.py index 0ad0750a5..96a1139fc 100644 --- a/graphify/install.py +++ b/graphify/install.py @@ -1245,6 +1245,7 @@ def _uninstall_kilo_plugin(project_dir: Path) -> None: }; """ _OPENCODE_PLUGIN_PATH = Path(".opencode") / "plugins" / "graphify.js" +_OPENCODE_PLUGIN_ENTRY = Path("plugins") / "graphify.js" _OPENCODE_CONFIG_PATH = Path(".opencode") / "opencode.json" def _install_opencode_plugin(project_dir: Path) -> None: """Write graphify.js plugin and register it in opencode.json.""" @@ -1263,7 +1264,11 @@ def _install_opencode_plugin(project_dir: Path) -> None: config = {} plugins = config.setdefault("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() + entry = _OPENCODE_PLUGIN_ENTRY.as_posix() + # OpenCode resolves plugin entries relative to opencode.json. Older + # releases registered the project-relative path, causing a duplicated + # .opencode path during plugin loading. + plugins[:] = [plugin for plugin in plugins if plugin != _OPENCODE_PLUGIN_PATH.as_posix()] if entry not in plugins: plugins.append(entry) config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") @@ -1285,9 +1290,12 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: except json.JSONDecodeError: return plugins = config.get("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() - if entry in plugins: - plugins.remove(entry) + entries = { + _OPENCODE_PLUGIN_PATH.as_posix(), + _OPENCODE_PLUGIN_ENTRY.as_posix(), + } + if any(plugin in entries for plugin in plugins): + plugins[:] = [plugin for plugin in plugins if plugin not in entries] if not plugins: config.pop("plugin") config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") diff --git a/tests/test_install.py b/tests/test_install.py index b1bcaa780..406ca126b 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -676,7 +676,19 @@ def test_opencode_agents_install_registers_plugin_in_config(tmp_path): import json as _json config = _json.loads(config_file.read_text()) - assert any("graphify.js" in p for p in config.get("plugin", [])) + assert "plugins/graphify.js" in config.get("plugin", []) + assert ".opencode/plugins/graphify.js" not in config.get("plugin", []) + + +def test_opencode_agents_install_repairs_legacy_plugin_entry(tmp_path): + import json as _json + + config_file = tmp_path / ".opencode" / "opencode.json" + config_file.parent.mkdir(parents=True, exist_ok=True) + config_file.write_text(_json.dumps({"plugin": [".opencode/plugins/graphify.js"]})) + _agents_install(tmp_path, "opencode") + config = _json.loads(config_file.read_text()) + assert config["plugin"] == ["plugins/graphify.js"] def test_opencode_agents_install_merges_existing_config(tmp_path): From a87894aa5ece17da3191b63ba6acf987c21b919a Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Fri, 17 Jul 2026 23:42:58 +0200 Subject: [PATCH 13/18] Normalize no-cluster graph output through endpoint reconciliation Rewrite the --no-cluster path in both extract and watch to use build_from_json and node_link_data instead of raw dedup, so the unclustered graph passes the diagnostic integrity gate. Also update documentation (AGENTS.md, README, --help text) to describe the normalized output and add the endpoint-safe graph regression test. --- AGENTS.md | 2 +- README.md | 4 ++-- graphify/__main__.py | 4 ++-- graphify/cli.py | 43 +++++++++++++++++++++++-------------------- graphify/watch.py | 30 ++++++++++++++++++------------ tests/test_watch.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b919654c4..0012ee1a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,4 +5,4 @@ This project has a graphify knowledge graph at graphify-out/. Rules: - Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) +- After modifying code files in this session, run `graphify update .` and then `graphify diagnose multigraph --graph graphify-out/graph.json --json` to keep the graph current and verify endpoint integrity (AST-only, no API cost) diff --git a/README.md b/README.md index 8129ab168..3685e311c 100644 --- a/README.md +++ b/README.md @@ -802,7 +802,7 @@ graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow lo graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt -graphify extract ./docs --no-cluster # raw extraction only, skip clustering +graphify extract ./docs --no-cluster # normalized unclustered graph, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) @@ -834,7 +834,7 @@ graphify --version # print installed version graphify watch ./src graphify check-update ./src graphify update ./src -graphify update ./src --no-cluster # skip reclustering, write raw AST graph only +graphify update ./src --no-cluster # skip reclustering, write a normalized unclustered graph graphify update ./src --force # overwrite even if new graph has fewer nodes graphify cluster-only ./my-project graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location diff --git a/graphify/__main__.py b/graphify/__main__.py index d97d48fda..d06dcfc61 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -538,7 +538,7 @@ def _run_cli() -> None: print(" update re-extract code files and update the graph (no LLM needed)") print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") - print(" --no-cluster skip clustering, write raw extraction only") + print(" --no-cluster skip clustering, write a normalized unclustered graph") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" --graph path to graph.json (default /graphify-out/graph.json)") @@ -606,7 +606,7 @@ def _run_cli() -> None: print(" --out DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") - print(" --no-cluster skip clustering, write raw extraction only") + print(" --no-cluster skip clustering, write a normalized unclustered graph") print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") diff --git a/graphify/cli.py b/graphify/cli.py index 7ca95199f..69accfea7 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3049,17 +3049,17 @@ def _progress(idx: int, total: int, _result: dict) -> None: ) if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + # --no-cluster: write an unclustered graph without community + # detection or an analysis sidecar. Build it through the same + # endpoint reconciliation as the clustered path; a raw merged + # extraction can contain external endpoints and parallel edges, + # which makes graph.json unsafe for downstream consumers (#1781). + from graphify.build import build_from_json as _build_unclustered from graphify.export import ( backup_if_protected as _backup, existing_graph_node_count as _existing_graph_node_count, ) + from networkx.readwrite import json_graph as _json_graph if ( incremental_mode and not code_files @@ -3095,16 +3095,19 @@ def _progress(idx: int, total: int, _result: dict) -> None: stages.total() sys.exit(0) - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) + _unclustered = _build_unclustered(merged, root=target) + try: + _normalized = _json_graph.node_link_data(_unclustered, edges="links") + except TypeError: + _normalized = _json_graph.node_link_data(_unclustered) + for _link in _normalized.get("links", []): + _true_src = _link.pop("_src", None) + _true_tgt = _link.pop("_tgt", None) + if _true_src is not None and _true_tgt is not None: + _link["source"] = _true_src + _link["target"] = _true_tgt + _normalized["input_tokens"] = merged["input_tokens"] + _normalized["output_tokens"] = merged["output_tokens"] # RT-parity for the raw path: an incomplete build must not force a # partial graph over a larger complete one here either. The clustered # path gets this from to_json's #479 guard; this path never calls @@ -3114,7 +3117,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH _existing_n = _existing_graph_node_count(graph_json_path) _malformed = _existing_n is _MALFORMED_GRAPH - _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n + _shrinks = isinstance(_existing_n, int) and len(_normalized["nodes"]) < _existing_n if _malformed or _shrinks: _detail = ( f"the existing {graph_json_path} is present but unparseable " @@ -3133,14 +3136,14 @@ def _progress(idx: int, total: int, _result: dict) -> None: sys.exit(1) _backup(graphify_out) from graphify.paths import write_json_atomic as _write_json_atomic - _write_json_atomic(graph_json_path, merged, indent=2) + _write_json_atomic(graph_json_path, _normalized, indent=2) stages.mark("write") cost = _estimate_cost( backend, merged["input_tokens"], merged["output_tokens"] ) print( f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"{len(_normalized['nodes'])} nodes, {len(_normalized['links'])} edges " f"(no clustering)" ) if merged["input_tokens"] or merged["output_tokens"]: diff --git a/graphify/watch.py b/graphify/watch.py index 37edc9cb1..5d78272ac 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -818,8 +818,11 @@ def _rebuild_code( ``block_on_lock=True`` to wait instead of skip (used by the interactive ``graphify update`` CLI). - ``no_cluster`` skips community detection and writes raw merged extraction - JSON to graphify-out/graph.json (mirrors ``extract --no-cluster``). + ``no_cluster`` skips community detection and writes the merged extraction + after the same endpoint reconciliation used by the clustered path. The + result is an unclustered graph, not an intermediate raw extraction, so + graph.json remains safe for consumers that expect every link endpoint to + name a node. Returns True on success, False on error or skipped-due-to-lock. """ @@ -1081,16 +1084,19 @@ def _add_deleted_source(path: Path) -> None: out.mkdir(exist_ok=True) if no_cluster: - # Normalise to "links" key so schema is consistent with the full clustered path. - # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); - # without it, --no-cluster + repeated `update` accumulate duplicates and edge - # counts diverge across build modes (#1317). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - candidate_graph_data = { - **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, - "nodes": _dedupe_nodes(result.get("nodes", [])), - "links": _dedupe_edges(result.get("edges", [])), - } + # Keep the no-cluster output on the same normalization path as the + # full build. The old raw dump preserved unresolved external + # endpoints and parallel edges, which made a freshly generated + # graph.json fail the diagnostic integrity gate even though the + # clustered build was healthy (#1781). + normalized_graph = build_from_json(result, root=project_root) + candidate_graph_data = _topology_from_graph(normalized_graph) + for link in candidate_graph_data.get("links", []): + true_src = link.pop("_src", None) + true_tgt = link.pop("_tgt", None) + if true_src is not None and true_tgt is not None: + link["source"] = true_src + link["target"] = true_tgt candidate_graph_text = _json_text(candidate_graph_data) same_graph = False if existing_graph.exists(): diff --git a/tests/test_watch.py b/tests/test_watch.py index e665e7738..1dd3a660d 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -363,6 +363,35 @@ def test_rebuild_code_deleted_cwd_uses_graphify_repo_root(tmp_path, monkeypatch) os.chdir(old_cwd) +def test_no_cluster_writes_endpoint_safe_graph(tmp_path): + """The unclustered output must be a valid graph, not raw extraction JSON.""" + from graphify.diagnostics import diagnose_file + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "main.py").write_text( + "import missing_module\n\n" + "def main():\n" + " return missing_module.value\n", + encoding="utf-8", + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + node_ids = {node["id"] for node in data["nodes"]} + + assert all( + link["source"] in node_ids and link["target"] in node_ids + for link in data["links"] + ) + summary = diagnose_file(graph_path) + assert summary["dangling_endpoint_edges"] == 0 + assert summary["directed_same_endpoint_collapsed_edges"] == 0 + assert summary["undirected_same_endpoint_collapsed_edges"] == 0 + + def test_rebuild_code_evicts_nodes_from_deleted_files(tmp_path): """#1007: graphify update (_rebuild_code with no changed_paths) must remove nodes and edges from files deleted since the last run.""" From 7647433790c173a1b46100911a3e717e91579de8 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Fri, 17 Jul 2026 23:43:05 +0200 Subject: [PATCH 14/18] Suppress zero-node warning for skipped JSON data files JSON data files are intentionally skipped by the structural extractor. They should not trigger the zero-node aggregate warning, which was designed for failed code extractions (#1666). --- graphify/extract.py | 5 ++++- tests/test_extract.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index a030c6ee0..e44a97489 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4416,7 +4416,10 @@ def extract( _empty_sources: list[str] = [] for i, _p in enumerate(paths): _res = per_file[i] or {} - if _res.get("nodes") or _res.get("error"): + # JSON data files are intentionally skipped by the structural extractor + # (#1224). They are not failed code extractions and must not repeatedly + # trigger the zero-node warning (#1666). + if _res.get("nodes") or _res.get("error") or _res.get("skipped"): continue if _get_extractor(_p) is not None: _empty_sources.append(str(_p)) diff --git a/tests/test_extract.py b/tests/test_extract.py index f9f7a5ce7..e68ec86cb 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1753,6 +1753,16 @@ def test_extract_json_top_level_array_skipped(tmp_path): assert result["edges"] == [] +def test_extract_json_data_file_does_not_trigger_zero_node_warning(tmp_path, capsys): + """Intentionally skipped data JSON must stay quiet in the aggregate pass.""" + data = tmp_path / "records.json" + data.write_text(json.dumps([{"id": 1}, {"id": 2}])) + + extract([data], cache_root=tmp_path / "out", parallel=False) + + assert "zero nodes" not in capsys.readouterr().err + + def test_extract_json_config_by_filename_still_extracted(tmp_path): """tsconfig.json must still be AST-extracted even without telltale keys.""" cfg = tmp_path / "tsconfig.json" From b44ecc480c59fb1394eb1f52dc6f6dc88ac367d9 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 18 Jul 2026 08:31:05 +0200 Subject: [PATCH 15/18] feat: index nix and pkl sources --- graphify/detect.py | 2 +- graphify/extract.py | 4 ++ graphify/extractors/nix.py | 82 ++++++++++++++++++++++++++++++++++++++ graphify/extractors/pkl.py | 77 +++++++++++++++++++++++++++++++++++ tests/test_detect.py | 5 +++ tests/test_extract.py | 20 +++++++++- 6 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 graphify/extractors/nix.py create mode 100644 graphify/extractors/pkl.py diff --git a/graphify/detect.py b/graphify/detect.py index c2638a0ef..65ace9e39 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.nix', '.pkl', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index e44a97489..fb16caf56 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -44,8 +44,10 @@ from graphify.extractors.go import extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 +from graphify.extractors.nix import extract_nix # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 +from graphify.extractors.pkl import extract_pkl # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 @@ -3927,6 +3929,8 @@ def add_existing_edge(edge: dict) -> None: ".sh": extract_bash, ".bash": extract_bash, ".json": extract_json, + ".nix": extract_nix, + ".pkl": extract_pkl, ".tf": extract_terraform, ".tfvars": extract_terraform, ".hcl": extract_terraform, diff --git a/graphify/extractors/nix.py b/graphify/extractors/nix.py new file mode 100644 index 000000000..e3d9921ea --- /dev/null +++ b/graphify/extractors/nix.py @@ -0,0 +1,82 @@ +"""Dependency-free structural extraction for Nix expressions.""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_BINDING_RE = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_'-]*)\s*=\s*") +_FUNCTION_RE = re.compile( + r"(?m)^\s*([A-Za-z_][A-Za-z0-9_'-]*)\s*=\s*(?:\([^\n]*\)|[A-Za-z_][A-Za-z0-9_'-]*)\s*:\s*" +) +_IMPORT_RE = re.compile(r"(? dict: + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + source_file = str(path) + stem = _file_stem(path) + file_id = _make_id(stem) + nodes = [{"id": file_id, "label": path.name, "file_type": "code", + "source_file": source_file, "source_location": "L1"}] + edges: list[dict] = [] + seen = {file_id} + bindings: dict[str, str] = {} + + def line(match: re.Match) -> int: + return source.count("\n", 0, match.start()) + 1 + + def add_node(name: str, at: int, kind: str = "binding") -> str: + node_id = _make_id(stem, name) + if node_id not in seen: + seen.add(node_id) + nodes.append({"id": node_id, "label": name, "file_type": "code", + "source_file": source_file, "source_location": f"L{at}", + "kind": kind}) + return node_id + + def add_edge(source: str, target: str, relation: str, at: int, context: str | None = None) -> None: + if source == target: + return + edge = {"source": source, "target": target, "relation": relation, + "confidence": "EXTRACTED", "source_file": source_file, + "source_location": f"L{at}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + for match in _BINDING_RE.finditer(source): + name = match.group(1) + if name in {"let", "in", "with", "inherit", "assert", "rec"}: + continue + kind = "function" if _FUNCTION_RE.match(source, match.start()) else "binding" + binding_id = add_node(name, line(match), kind) + bindings.setdefault(name, binding_id) + add_edge(file_id, binding_id, "contains", line(match)) + + for match in _IMPORT_RE.finditer(source): + target = match.group(1) + target_id = _make_id(_file_stem(path.parent / target)) + if target_id not in seen: + seen.add(target_id) + nodes.append({"id": target_id, "label": Path(target).name, "file_type": "concept", + "source_file": source_file, "source_location": f"L{line(match)}", + "kind": "relative-import"}) + add_edge(file_id, target_id, "imports", line(match), "literal-relative-path") + + for match in _INTERPOLATION_RE.finditer(source): + target = bindings.get(match.group(1)) + if target: + add_edge(file_id, target, "references", line(match), "interpolation") + for match in _INHERIT_RE.finditer(source): + for name in re.findall(r"[A-Za-z_][A-Za-z0-9_'-]*", match.group(1)): + target = bindings.get(name) + if target: + add_edge(file_id, target, "references", line(match), "inherit") + return {"nodes": nodes, "edges": edges} diff --git a/graphify/extractors/pkl.py b/graphify/extractors/pkl.py new file mode 100644 index 000000000..2b0bfe07c --- /dev/null +++ b/graphify/extractors/pkl.py @@ -0,0 +1,77 @@ +"""Dependency-free structural extraction for Pkl configuration modules.""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_DECL_RE = re.compile( + r"(?m)^\s*(?:(abstract)\s+)?(module|class|typealias|function|modulemethod)\s+([A-Za-z_][A-Za-z0-9_]*)" +) +_PROPERTY_RE = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z_][A-Za-z0-9_?.<>\[\]]*)") +_IMPORT_RE = re.compile(r"(?m)^\s*(import|amends|extends)\s+([^\n]+)") + + +def extract_pkl(path: Path) -> dict: + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + source_file = str(path) + stem = _file_stem(path) + file_id = _make_id(stem) + nodes = [{"id": file_id, "label": path.name, "file_type": "code", + "source_file": source_file, "source_location": "L1"}] + edges: list[dict] = [] + seen = {file_id} + + def line(match: re.Match) -> int: + return source.count("\n", 0, match.start()) + 1 + + def add_node(name: str, at: int, kind: str) -> str: + node_id = _make_id(stem, name) + if node_id not in seen: + seen.add(node_id) + nodes.append({"id": node_id, "label": name, "file_type": "code", + "source_file": source_file, "source_location": f"L{at}", + "kind": kind}) + return node_id + + def add_edge(source: str, target: str, relation: str, at: int, context: str | None = None) -> None: + if source == target: + return + edge = {"source": source, "target": target, "relation": relation, + "confidence": "EXTRACTED", "source_file": source_file, + "source_location": f"L{at}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + symbols: dict[str, str] = {} + for match in _DECL_RE.finditer(source): + name, kind = match.group(3), match.group(2) + symbol_id = add_node(name, line(match), kind) + symbols.setdefault(name, symbol_id) + add_edge(file_id, symbol_id, "contains", line(match)) + for match in _PROPERTY_RE.finditer(source): + name, type_name = match.group(1), match.group(2) + if name in {"module", "class", "typealias", "function", "extends", "amends"}: + continue + property_id = add_node(name, line(match), "property") + symbols.setdefault(name, property_id) + add_edge(file_id, property_id, "contains", line(match)) + type_id = add_node(type_name, line(match), "type") + add_edge(property_id, type_id, "references", line(match), "type") + for match in _IMPORT_RE.finditer(source): + relation, expression = match.group(1), match.group(2).strip() + target = expression.strip('"') + target_id = _make_id("pkl", target) + if target_id not in seen: + seen.add(target_id) + nodes.append({"id": target_id, "label": target, "file_type": "concept", + "source_file": source_file, "source_location": f"L{line(match)}", + "kind": "module-reference"}) + add_edge(file_id, target_id, "imports" if relation == "import" else relation, + line(match), "module-reference") + return {"nodes": nodes, "edges": edges} diff --git a/tests/test_detect.py b/tests/test_detect.py index 837d8cf3d..348ee8510 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -12,6 +12,11 @@ def test_classify_python(): def test_classify_typescript(): assert classify_file(Path("bar.ts")) == FileType.CODE + +def test_classify_nix_and_pkl(): + assert classify_file(Path("flake.nix")) == FileType.CODE + assert classify_file(Path("Schema.pkl")) == FileType.CODE + def test_classify_powershell_module(): # #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap). assert classify_file(Path("Utils.psm1")) == FileType.CODE diff --git a/tests/test_extract.py b/tests/test_extract.py index e68ec86cb..9e23956e5 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -7,11 +7,29 @@ import pytest from graphify.build import build_from_json -from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, _DISPATCH +from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, extract_nix, extract_pkl, _DISPATCH FIXTURES = Path(__file__).parent / "fixtures" +def test_extract_nix_records_bindings_and_literal_imports(tmp_path): + path = tmp_path / "default.nix" + path.write_text('''{ inputs, ... }:\nlet\n pkgs = import ./nixpkgs.nix;\n value = "${pkgs}";\nin\n{ inherit value; }\n''', encoding="utf-8") + result = extract_nix(path) + labels = {node["label"] for node in result["nodes"]} + assert {"pkgs", "value", "nixpkgs.nix"} <= labels + assert any(edge["relation"] == "imports" for edge in result["edges"]) + + +def test_extract_pkl_records_declarations_and_module_relationships(tmp_path): + path = tmp_path / "Schema.pkl" + path.write_text('''module Schema\nimport "pkl:base"\nclass Host {}\nname: String\n''', encoding="utf-8") + result = extract_pkl(path) + labels = {node["label"] for node in result["nodes"]} + assert {"Schema", "Host", "name", "String", "pkl:base"} <= labels + assert any(edge["relation"] == "imports" for edge in result["edges"]) + + def test_make_id_strips_dots_and_underscores(): assert _make_id("_auth") == "auth" assert _make_id(".httpx._client") == "httpx_client" From 0dd18b0eb3f75570260d060f6b60e62aa2067533 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 18 Jul 2026 15:27:09 +0200 Subject: [PATCH 16/18] feat: route semantic extraction through ACP Use the official ACP client as Graphify's provider-neutral transport and preserve codex-cli as a compatibility alias. Keep subscription authentication and adapter configuration outside Graphify. --- .github/workflows/ci.yaml | 6 +- README.md | 15 ++- flake.nix | 22 +++- graphify/acp.py | 257 ++++++++++++++++++++++++++++++++++++++ graphify/cli.py | 23 +++- graphify/llm.py | 112 ++++++++++++++++- nix/nixos-module.nix | 34 ++++- pyproject.toml | 3 +- simit.toml | 1 + tests/fake_acp_agent.py | 75 +++++++++++ tests/test_acp_backend.py | 94 ++++++++++++++ uv.lock | 22 +++- 12 files changed, 649 insertions(+), 15 deletions(-) create mode 100644 graphify/acp.py create mode 100644 tests/fake_acp_agent.py create mode 100644 tests/test_acp_backend.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8de65289e..9621fc438 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,13 +19,13 @@ jobs: XDG_CACHE_HOME: "/tmp/.cache" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Nix - uses: cachix/install-nix-action@v31 + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31 - name: Check generated flake wiring - run: nix run git+https://codeberg.org/caniko/simit.git -- init flake --check --diff + run: nix run --no-write-lock-file git+https://codeberg.org/caniko/simit.git -- init flake --check --diff - name: Check flake evaluation run: nix flake check --no-build diff --git a/README.md b/README.md index 3685e311c..0e309af7b 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,8 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `gemini` | Google Gemini API | `uv tool install "graphifyy[gemini]"` | | `anthropic` | Anthropic Claude API (`--backend claude`, uses `ANTHROPIC_API_KEY`) | `uv tool install "graphifyy[anthropic]"` | | `bedrock` | AWS Bedrock (uses IAM, no API key) | `uv tool install "graphifyy[bedrock]"` | +| `acp` | Generic Agent Client Protocol provider (including provider-managed subscriptions; no Graphify API key) | install/configure an ACP adapter | +| `codex-cli` | Deprecated alias for `acp`; uses the configured ACP adapter and Codex subscription | install/configure `codex-acp` | | `azure` | Azure OpenAI Service (`--backend azure`, uses `AZURE_OPENAI_API_KEY` + `AZURE_OPENAI_ENDPOINT`) | `uv tool install "graphifyy[openai]"` | | `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | @@ -591,9 +593,16 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `AZURE_OPENAI_API_VERSION` | Azure API version override | optional — default `2024-12-01-preview` | | `AZURE_OPENAI_DEPLOYMENT` or `GRAPHIFY_AZURE_MODEL` | Azure deployment name | optional — default `gpt-4o` | | `AWS_*` / `~/.aws/credentials` | AWS Bedrock — standard credential chain | `--backend bedrock` (no API key, uses IAM) | +| `GRAPHIFY_CODEX_CLI_MODEL` | Deprecated fallback model override for the `codex-cli` alias | use `GRAPHIFY_ACP_MODEL` instead | +| `GRAPHIFY_CODEX_BIN` | Unsupported legacy direct-CLI setting | migrate to `GRAPHIFY_ACP_BIN=codex-acp`; the adapter may use `CODEX_PATH` | +| `GRAPHIFY_ACP_BIN` | ACP adapter command (normally `codex-acp`) | `--backend acp`; otherwise resolves `codex-acp` from `PATH` | +| `GRAPHIFY_ACP_ARGS_JSON` | JSON array of ACP adapter arguments | optional — defaults to `[]` | +| `GRAPHIFY_ACP_CONFIG_JSON` | JSON object of ACP session configuration options | optional — defaults to read-only mode | +| `GRAPHIFY_ACP_MODEL` | Optional model selected through ACP session configuration | optional — adapter default | +| `GRAPHIFY_ACP_PARALLEL` | Allow concurrent ACP extraction sessions | optional — default is serial | | `GRAPHIFY_MAX_WORKERS` | AST parallelism thread count | optional — also `--max-workers` flag | | `GRAPHIFY_MAX_OUTPUT_TOKENS` | Raise output cap for dense corpora | optional — e.g. `32768` for large files | -| `GRAPHIFY_API_TIMEOUT` | Per-call timeout in seconds for HTTP, claude-cli, and Anthropic SDK backends (default: 600) | optional — also `--api-timeout` flag | +| `GRAPHIFY_API_TIMEOUT` | Per-call timeout in seconds for HTTP, claude-cli, ACP, and SDK backends (default: 600) | optional — also `--api-timeout` flag | | `GRAPHIFY_MAX_RETRIES` | How many times to retry a rate-limited (429) request before giving up (default: 6; honors `Retry-After`) | optional — raise for strict per-org limits (e.g. kimi); `0` disables | | `GRAPHIFY_FORCE` | Force graph rebuild even with fewer nodes | optional — also `--force` flag | | `GRAPHIFY_GOOGLE_WORKSPACE` | Auto-enable Google Workspace export | optional — set to `1` | @@ -783,7 +792,7 @@ graphify antigravity install # .agents/rules + .agents/workflows (Google A graphify antigravity uninstall graphify extract ./docs # headless LLM extraction for CI (no IDE needed) -graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, or claude-cli +graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, claude-cli, or acp graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) - no API key needed for loopback OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # any OpenAI-compatible server (llama.cpp, vLLM, LM Studio) @@ -792,6 +801,8 @@ GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # overr GRAPHIFY_OLLAMA_KEEP_ALIVE=0 graphify extract ./docs --backend ollama # unload model after each chunk (saves VRAM on small GPUs) graphify extract ./docs --backend bedrock # AWS Bedrock via IAM - no API key, uses AWS credential chain graphify extract ./docs --backend claude-cli # route through Claude Code CLI - no API key, uses your Claude subscription +graphify extract ./docs --backend codex-cli # deprecated alias for ACP; still uses the Codex subscription through codex-acp +graphify extract ./docs --backend acp # route through a generic ACP adapter, such as codex-acp graphify extract ./docs --backend azure # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT) graphify extract ./docs --max-workers 16 # AST parallelism (also GRAPHIFY_MAX_WORKERS) graphify extract --postgres "postgresql://user:pass@host/db" # introspect live PostgreSQL schema directly diff --git a/flake.nix b/flake.nix index 8ac4b675e..49798c99b 100644 --- a/flake.nix +++ b/flake.nix @@ -122,7 +122,7 @@ # The retry-cap tests exercise the OpenAI-compatible Ollama # path, so include that optional extra in the test-only # environment without pulling every runtime extra. - graphifyy = ["dev" "ollama"]; + graphifyy = ["dev" "ollama" "acp"]; }); in (old.passthru.tests or {}) @@ -164,6 +164,14 @@ virtualenv = editablePythonSet.mkVirtualEnv "graphify-dev-env" workspace.deps.all; graphifyEnv = pythonSet.mkVirtualEnv "graphify-env" workspace.deps.default; + graphifyAcpEnv = pythonSet.mkVirtualEnv "graphify-acp-env" (workspace.deps.default + // { + graphifyy = ["acp"]; + }); + graphifyOpenaiEnv = pythonSet.mkVirtualEnv "graphify-openai-env" (workspace.deps.default + // { + graphifyy = ["openai"]; + }); graphifyFullEnv = pythonSet.mkVirtualEnv "graphify-full-env" (workspace.deps.default // { graphifyy = ["all"]; @@ -211,6 +219,14 @@ environment = graphifyFullEnv; suffix = "-full"; }; + graphifyAcpPackage = mkGraphifyPackage { + environment = graphifyAcpEnv; + suffix = "-acp"; + }; + graphifyOpenaiPackage = mkGraphifyPackage { + environment = graphifyOpenaiEnv; + suffix = "-openai"; + }; moduleSample = inputs.nixpkgs.lib.nixosSystem { system = pkgs.stdenv.hostPlatform.system; @@ -341,6 +357,8 @@ packages = { default = graphifyPackage; + acp = graphifyAcpPackage; + openai = graphifyOpenaiPackage; full = graphifyFullPackage; }; @@ -349,7 +367,7 @@ full-package = pkgs.runCommand "graphify-full-package-check" {} '' test -x ${graphifyFullPackage}/bin/graphify test -x ${graphifyFullPackage}/bin/graphify-mcp - ${graphifyFullEnv}/bin/python -c 'import anthropic, boto3, falkordb, mcp, neo4j, openai, psycopg' + ${graphifyFullEnv}/bin/python -c 'import acp, anthropic, boto3, falkordb, mcp, neo4j, openai, psycopg' touch $out ''; nixos-module = pkgs.runCommand "graphify-nixos-module-check" {} '' diff --git a/graphify/acp.py b/graphify/acp.py new file mode 100644 index 000000000..382ff044e --- /dev/null +++ b/graphify/acp.py @@ -0,0 +1,257 @@ +"""Small ACP client used by Graphify's provider-managed semantic backend. + +The client deliberately speaks ACP through the official Python SDK. The +adapter command is supplied by the host (normally Infernix's ``codex-acp`` +wrapper), so Graphify does not need to know which subscription or agent +implementation is behind it. +""" +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import tempfile +from collections.abc import Mapping +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class AcpResult: + text: str + input_tokens: int = 0 + output_tokens: int = 0 + model: str = "acp" + stop_reason: str = "end_turn" + + +def _json_object(value: str, variable: str) -> dict[str, Any]: + if not value.strip(): + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{variable} must contain a JSON object: {exc}") from exc + if not isinstance(parsed, dict): + raise ValueError(f"{variable} must contain a JSON object") + invalid = { + key: option + for key, option in parsed.items() + if not isinstance(key, str) or not isinstance(option, (str, bool)) + } + if invalid: + raise ValueError(f"{variable} values must be strings or booleans") + return parsed + + +def _config_options(response: Any) -> set[str]: + return { + str(getattr(option, "id", "")) + for option in (getattr(response, "config_options", None) or []) + if getattr(option, "id", None) + } + + +def _session_modes(response: Any) -> set[str]: + modes = getattr(response, "modes", None) + return { + str(getattr(mode, "id", "")) + for mode in (getattr(modes, "available_modes", None) or []) + if getattr(mode, "id", None) + } + + +class _GraphifyClient: + def __init__(self) -> None: + self.messages: list[str] = [] + self.usage: Any = None + + async def request_permission(self, options: list[Any], session_id: str, tool_call: Any, **kwargs: Any) -> Any: + from acp.schema import DeniedOutcome, RequestPermissionResponse + + # ACP agents may request a tool even when the session is configured as + # read-only. Graphify is a pure extractor, so all such requests are + # denied rather than delegated to a host callback. + return RequestPermissionResponse(outcome=DeniedOutcome(outcome="cancelled")) + + async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: + update_kind = str(getattr(update, "session_update", "")) + if update_kind == "agent_message_chunk": + content = update.content + if getattr(content, "type", None) == "text": + self.messages.append(content.text) + elif update_kind == "usage_update": + self.usage = update + + +async def _run_acp( + prompt: str, + *, + command: str, + args: list[str], + config: Mapping[str, Any], + model: str, + images: list[Any], + timeout: float, + cwd: Path, +) -> AcpResult: + try: + from acp import PROTOCOL_VERSION, image_block, spawn_agent_process, text_block + from acp.schema import ClientCapabilities, Implementation + except ImportError as exc: + raise ImportError( + "ACP extraction requires graphifyy's `acp` extra. " + "Install graphifyy[acp] or use the Nix acp package." + ) from exc + + resolved_command = command.strip() + if not resolved_command: + raise RuntimeError( + "No ACP adapter command configured. Set GRAPHIFY_ACP_BIN or install an ACP provider." + ) + if os.path.sep not in resolved_command and shutil.which(resolved_command) is None: + raise RuntimeError( + f"ACP adapter {resolved_command!r} was not found on PATH. " + "Set GRAPHIFY_ACP_BIN to the provider command." + ) + + environment = os.environ.copy() + environment.setdefault("NO_BROWSER", "1") + client = _GraphifyClient() + async with spawn_agent_process( + client, + resolved_command, + *args, + env=environment, + cwd=cwd, + ) as (connection, _process): + await asyncio.wait_for( + connection.initialize( + protocol_version=PROTOCOL_VERSION, + client_capabilities=ClientCapabilities(), + client_info=Implementation(name="graphify", version="0.9.18"), + ), + timeout=timeout, + ) + session = await asyncio.wait_for( + connection.new_session(cwd=str(cwd), mcp_servers=[]), + timeout=timeout, + ) + session_id = session.session_id + advertised = _config_options(session) + requested = dict(config) + if model: + requested["model"] = model + + # ACP v1 exposes provider settings after session/new. Only send + # settings the agent advertises; this keeps the client generic across + # adapters while avoiding the legacy adapter-specific `-c` flags. + for option, value in requested.items(): + if option in advertised: + await asyncio.wait_for( + connection.set_config_option(option, session_id, value), timeout=timeout + ) + + if "mode" not in advertised: + modes = _session_modes(session) + if "read-only" in modes: + await asyncio.wait_for( + connection.set_session_mode(session_id, "read-only"), timeout=timeout + ) + + content: list[Any] = [text_block(prompt)] + for image in images: + if getattr(image, "raw", None): + content.append(image_block(image.b64, image.media_type, uri=str(image.path))) + response = await asyncio.wait_for(connection.prompt(session_id, content), timeout=timeout) + + usage = getattr(response, "usage", None) or client.usage + input_tokens = int(getattr(usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(usage, "output_tokens", 0) or 0) + stop_reason = str(getattr(response, "stop_reason", "end_turn")) + return AcpResult( + text="".join(client.messages), + input_tokens=input_tokens, + output_tokens=output_tokens, + model=model or "acp", + stop_reason=stop_reason, + ) + + +def _run_in_thread(coro: Any) -> Any: + """Run ACP from sync code even when the caller already owns an event loop.""" + result: Future[Any] = Future() + + def runner() -> None: + try: + result.set_result(asyncio.run(coro)) + except BaseException as exc: # propagate the original exception + result.set_exception(exc) + + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="graphify-acp") as pool: + pool.submit(runner).result() + return result.result() + + +def run_acp( + prompt: str, + *, + model: str | None = None, + images: list[Any] | None = None, + max_tokens: int = 8192, + extraction: bool = False, + deep_mode: bool = False, +) -> AcpResult: + """Run one isolated, read-only ACP prompt and collect its text/usage.""" + command = os.environ.get("GRAPHIFY_ACP_BIN", "").strip() or "codex-acp" + try: + args_value = json.loads(os.environ.get("GRAPHIFY_ACP_ARGS_JSON", "[]")) + except json.JSONDecodeError as exc: + raise ValueError(f"GRAPHIFY_ACP_ARGS_JSON must contain a JSON array: {exc}") from exc + if not isinstance(args_value, list) or not all(isinstance(item, str) for item in args_value): + raise ValueError("GRAPHIFY_ACP_ARGS_JSON must contain a JSON array of strings") + + selected_model = ( + model + or os.environ.get("GRAPHIFY_ACP_MODEL", "").strip() + or os.environ.get("GRAPHIFY_CODEX_CLI_MODEL", "").strip() + ).strip() + config = _json_object(os.environ.get("GRAPHIFY_ACP_CONFIG_JSON", ""), "GRAPHIFY_ACP_CONFIG_JSON") + config.setdefault("mode", "read-only") + + message = prompt + if extraction: + from graphify.llm import _extraction_system + + message = ( + _extraction_system(deep=deep_mode) + + "\n\n---\nNow extract the knowledge graph from the following source file(s) " + + "and output ONLY the JSON object described above. No prose, no preamble, no markdown fences. " + + f"Keep the response within roughly {max_tokens} output tokens.\n\n" + + prompt + ) + + timeout = float(os.environ.get("GRAPHIFY_API_TIMEOUT", "600") or "600") + if timeout <= 0: + timeout = 600.0 + with tempfile.TemporaryDirectory(prefix="graphify-acp-") as working_directory: + coro = _run_acp( + message, + command=command, + args=args_value, + config=config, + model=selected_model, + images=images or [], + timeout=timeout, + cwd=Path(working_directory), + ) + try: + import asyncio as _asyncio + + _asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + return _run_in_thread(coro) diff --git a/graphify/cli.py b/graphify/cli.py index 69accfea7..960474b8e 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2323,7 +2323,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": # has an API key set. if len(sys.argv) < 3: print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama|acp] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " "[--no-gitignore] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " @@ -2765,6 +2765,27 @@ def _parse_float(name: str, raw: str) -> float: file=sys.stderr, ) sys.exit(1) + elif backend in ("acp", "codex-cli"): + import shutil as _shutil + if ( + backend == "codex-cli" + and os.environ.get("GRAPHIFY_CODEX_BIN", "").strip() + and not os.environ.get("GRAPHIFY_ACP_BIN", "").strip() + ): + print( + "error: codex-cli now uses ACP; replace GRAPHIFY_CODEX_BIN " + "with GRAPHIFY_ACP_BIN=codex-acp and set CODEX_PATH on the adapter if needed.", + file=sys.stderr, + ) + sys.exit(1) + allow_no_key = _shutil.which(os.environ.get("GRAPHIFY_ACP_BIN", "codex-acp")) is not None + if not allow_no_key: + print( + f"error: backend '{backend}' requires an ACP adapter on PATH " + "(normally codex-acp, configured with GRAPHIFY_ACP_BIN).", + file=sys.stderr, + ) + sys.exit(1) if not allow_no_key: print( f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", diff --git a/graphify/llm.py b/graphify/llm.py index ef094b53f..b4a41d042 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -174,6 +174,25 @@ def _get_tokenizer(): # CLI's Read tool rather than as inline base64 (see `_call_claude_cli`). "vision": True, }, + "acp": { + # Provider-managed ACP authentication. The adapter owns subscription + # and API-key handling; Graphify deliberately never reads an OpenAI key + # for this backend. + "default_model": "", + "model_env_key": "GRAPHIFY_ACP_MODEL", + "pricing": {"input": 0.0, "output": 0.0}, + "max_tokens": 16384, + "vision": True, + }, + # Deprecated compatibility alias. `_normalize_backend` routes this through + # the generic ACP backend; it never invokes `codex exec` directly. + "codex-cli": { + "default_model": "", + "model_env_key": "GRAPHIFY_CODEX_CLI_MODEL", + "pricing": {"input": 0.0, "output": 0.0}, + "max_tokens": 16384, + "vision": True, + }, } @@ -811,6 +830,33 @@ def _backend_supports_vision(backend: str) -> bool: return bool(BACKENDS.get(backend, {}).get("vision", False)) +_WARNED_BACKEND_ALIASES: set[str] = set() + + +def _normalize_backend(backend: str) -> str: + """Normalize deprecated backend names to their supported transports.""" + if backend != "codex-cli": + return backend + + if os.environ.get("GRAPHIFY_CODEX_BIN", "").strip() and not os.environ.get( + "GRAPHIFY_ACP_BIN", "" + ).strip(): + raise RuntimeError( + "The deprecated codex-cli backend now uses ACP, but GRAPHIFY_CODEX_BIN " + "points to the Codex CLI rather than an ACP adapter. Set GRAPHIFY_ACP_BIN " + "to codex-acp (and CODEX_PATH for the adapter when needed), then remove " + "GRAPHIFY_CODEX_BIN." + ) + if backend not in _WARNED_BACKEND_ALIASES: + print( + "[graphify] WARNING: --backend codex-cli is deprecated and now routes " + "through ACP. Use --backend acp and GRAPHIFY_ACP_* settings.", + file=sys.stderr, + ) + _WARNED_BACKEND_ALIASES.add(backend) + return "acp" + + def _image_notes(refs: list[_ImageRef], *, with_paths: bool = False) -> str: """Text block listing the images so the model emits one node per image. @@ -1425,6 +1471,41 @@ def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bo return result +def _call_acp( + user_message: str, + max_tokens: int = 8192, + *, + model: str | None = None, + deep_mode: bool = False, + images: list[_ImageRef] | None = None, +) -> dict: + """Call the configured ACP provider through the official Python SDK.""" + from graphify.acp import run_acp + + response = run_acp( + user_message, + model=model, + max_tokens=max_tokens, + extraction=True, + deep_mode=deep_mode, + images=images, + ) + raw_content = response.text + result = _parse_llm_json(raw_content or "{}") + result["input_tokens"] = response.input_tokens + result["output_tokens"] = response.output_tokens + result["model"] = response.model + result["finish_reason"] = "length" if response.stop_reason == "max_tokens" else "stop" + if _response_is_hollow(raw_content, result) and result["finish_reason"] != "length": + print( + "[graphify] ACP provider returned a hollow response; treating as " + "truncation so adaptive retry can bisect the chunk.", + file=sys.stderr, + ) + result["finish_reason"] = "length" + return result + + def _azure_client(api_key: str, endpoint: str): """Construct an AzureOpenAI client with env-driven api_version and timeout.""" try: @@ -1561,8 +1642,10 @@ def extract_files_direct( "No LLM backend configured. Set one of: GEMINI_API_KEY, ANTHROPIC_API_KEY, " "OPENAI_API_KEY, DEEPSEEK_API_KEY, MOONSHOT_API_KEY, " "AZURE_OPENAI_API_KEY+AZURE_OPENAI_ENDPOINT, OLLAMA_BASE_URL, " - "or AWS credentials. Pass backend= explicitly to select a provider." + "AWS credentials, or an ACP adapter such as codex-acp. Pass backend= explicitly " + "to select a provider." ) + backend = _normalize_backend(backend) if backend not in BACKENDS: raise ValueError(f"Unknown backend {backend!r}. Available: {sorted(BACKENDS)}") @@ -1581,7 +1664,7 @@ def extract_files_direct( file=sys.stderr, ) key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): + if not key and backend not in ("bedrock", "claude-cli", "acp"): raise ValueError( f"No API key for backend '{backend}'. " f"Set {_format_backend_env_keys(backend)} or pass api_key=." @@ -1605,6 +1688,14 @@ def extract_files_direct( result = _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) elif backend == "claude-cli": result = _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + elif backend == "acp": + result = _call_acp( + user_msg, + max_tokens=max_out, + model=model, + deep_mode=deep_mode, + images=image_refs, + ) elif backend == "bedrock": result = _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) elif backend == "azure": @@ -2082,6 +2173,7 @@ def extract_corpus_parallel( Accepts ``str`` paths as well as ``Path``; string entries are coerced up front so packing/slicing helpers can rely on ``Path`` semantics (#1386). """ + backend = _normalize_backend(backend) files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files] # Split oversized splittable documents into slices that cover the whole file # before packing, so content past _FILE_CHAR_CAP is extracted instead of @@ -2125,6 +2217,10 @@ def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | # over session state. Force serial unless the user explicitly opts in. if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1": max_concurrency = 1 + # ACP providers own subscription-backed agent sessions; keep calls serial + # by default to avoid concurrent sessions and rate-limit bursts. + if backend == "acp" and os.environ.get("GRAPHIFY_ACP_PARALLEL", "").strip() != "1": + max_concurrency = 1 def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # Persist each chunk's semantic results to the cache as soon as it # completes. Without this, the semantic cache is only written once, at @@ -2358,6 +2454,7 @@ def _call_llm( exist in this module, so the LLM tiebreaker silently no-op'd on `ImportError` (F-038). Adding the function here re-enables it. """ + backend = _normalize_backend(backend) if backend not in BACKENDS: raise ValueError(f"Unknown backend {backend!r}") cfg = BACKENDS[backend] @@ -2366,7 +2463,7 @@ def _call_llm( ollama_url = os.environ.get("OLLAMA_BASE_URL", cfg.get("base_url", "")) _validate_ollama_base_url(ollama_url) key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): + if not key and backend not in ("bedrock", "claude-cli", "acp"): raise ValueError( f"No API key for backend '{backend}'. Set {_format_backend_env_keys(backend)}." ) @@ -2434,6 +2531,13 @@ def _rec(inp, out) -> None: ) return envelope.get("result", "") + if backend == "acp": + from graphify.acp import run_acp + + response = run_acp(prompt, model=mdl, max_tokens=max_tokens) + _rec(response.input_tokens, response.output_tokens) + return response.text + if backend == "bedrock": try: @@ -2833,6 +2937,8 @@ def label_communities( max_concurrency = 1 if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1": max_concurrency = 1 + if backend == "acp" and os.environ.get("GRAPHIFY_ACP_PARALLEL", "").strip() != "1": + max_concurrency = 1 workers = max(1, min(max_concurrency, n_batches)) def _run_batch(batch_idx: int): diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix index b15a82b45..9e0b8dc4d 100644 --- a/nix/nixos-module.nix +++ b/nix/nixos-module.nix @@ -13,6 +13,8 @@ "bedrock" "claude" "claude-cli" + "acp" + "codex-cli" "deepseek" "gemini" "kimi" @@ -210,6 +212,28 @@ default = null; description = "Custom providers.json mounted read-only at HOME/.graphify/providers.json."; }; + acp = mkOption { + type = types.submodule { + options = { + binary = mkOption { + type = types.nullOr types.str; + default = null; + description = "ACP provider command. Defaults to codex-acp on PATH."; + }; + args = mkOption { + type = types.listOf types.str; + default = []; + description = "ACP adapter arguments encoded as JSON for the client."; + }; + configOptions = mkOption { + type = types.attrsOf (types.oneOf [types.str types.bool]); + default = {mode = "read-only";}; + description = "ACP session configuration options applied after session/new."; + }; + }; + }; + default = {}; + }; }; }; @@ -368,7 +392,7 @@ type = types.listOf types.package; default = []; example = lib.literalExpression "[ pkgs.claude-code pkgs.gws ]"; - description = "External executables added to service PATH, such as claude for claude-cli or gws for Google Workspace export."; + description = "External executables added to service PATH, such as claude, codex-acp, or gws."; }; }; }; @@ -388,6 +412,7 @@ baseEnvironment = instance: let pg = instance.source.postgresql; backend = instance.llm.backend; + acpBackend = builtins.elem backend ["acp" "codex-cli"]; baseUrlVariable = if backend != null then backendBaseUrlVariables.${backend} or null @@ -407,6 +432,13 @@ // optionalAttrs (instance.llm.baseUrl != null && baseUrlVariable != null) { ${baseUrlVariable} = instance.llm.baseUrl; } + // optionalAttrs acpBackend ({ + GRAPHIFY_ACP_ARGS_JSON = builtins.toJSON instance.llm.acp.args; + GRAPHIFY_ACP_CONFIG_JSON = builtins.toJSON instance.llm.acp.configOptions; + GRAPHIFY_ACP_MODEL = if instance.llm.model != null then instance.llm.model else ""; + } // optionalAttrs (instance.llm.acp.binary != null) { + GRAPHIFY_ACP_BIN = instance.llm.acp.binary; + }) // instance.environment; extractionArguments = name: instance: diff --git a/pyproject.toml b/pyproject.toml index e4784f00b..9d4f54ef6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ bedrock = ["boto3"] anthropic = ["anthropic"] gemini = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] +acp = ["agent-client-protocol>=0.10.1,<0.12"] chinese = ["jieba"] sql = ["tree-sitter-sql"] # extract_pascal() uses tree-sitter-pascal for AST-quality extraction (more @@ -80,7 +81,7 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "agent-client-protocol>=0.10.1,<0.12", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/simit.toml b/simit.toml index 98687d863..0a7e7e8d2 100644 --- a/simit.toml +++ b/simit.toml @@ -10,3 +10,4 @@ checks = ["pytest"] runtime = "nix" runner = "ubuntu-latest" components = ["flake-wiring", "flake-evaluation", "checks"] +platform = "github" diff --git a/tests/fake_acp_agent.py b/tests/fake_acp_agent.py new file mode 100644 index 000000000..817c6cc44 --- /dev/null +++ b/tests/fake_acp_agent.py @@ -0,0 +1,75 @@ +"""Minimal ACP v1 agent used by Graphify's transport integration tests.""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + + +def _send(message: dict) -> None: + print(json.dumps(message), flush=True) + + +for line in sys.stdin: + message = json.loads(line) + request_id = message.get("id") + method = message.get("method") + if method == "initialize": + result = {"protocolVersion": 1} + elif method == "session/new": + result = { + "sessionId": "graphify-test", + "configOptions": [ + { + "id": "model", + "name": "Model", + "type": "select", + "currentValue": "default", + "options": [], + }, + { + "id": "mode", + "name": "Mode", + "type": "select", + "currentValue": "agent", + "options": [], + }, + ], + } + elif method == "session/set_config_option": + if log_path := os.environ.get("GRAPHIFY_FAKE_ACP_CONFIG_LOG"): + with Path(log_path).open("a", encoding="utf-8") as log: + log.write(json.dumps(message["params"], sort_keys=True) + "\n") + result = {"configOptions": []} + elif method == "session/prompt": + _send( + { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": "graphify-test", + "update": { + "sessionUpdate": "agent_message_chunk", + "content": { + "type": "text", + "text": '{"nodes": [], "edges": [], "hyperedges": []}', + }, + }, + }, + } + ) + result = { + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 7, "totalTokens": 18}, + } + else: + _send( + { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32601, "message": f"unknown method {method}"}, + } + ) + continue + _send({"jsonrpc": "2.0", "id": request_id, "result": result}) diff --git a/tests/test_acp_backend.py b/tests/test_acp_backend.py new file mode 100644 index 000000000..46525e28c --- /dev/null +++ b/tests/test_acp_backend.py @@ -0,0 +1,94 @@ +"""Tests for Graphify's generic ACP backend.""" +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +from graphify import llm +from graphify.acp import AcpResult, _json_object, run_acp + + +def test_acp_backend_requires_no_api_key(): + assert "acp" in llm.BACKENDS + assert llm.BACKENDS["acp"]["pricing"] == {"input": 0.0, "output": 0.0} + assert llm.estimate_cost("acp", 1_000_000, 1_000_000) == 0.0 + + +def test_extract_files_direct_dispatches_to_acp_without_an_api_key(tmp_path): + source = tmp_path / "note.md" + source.write_text("# ACP route\n") + result = {"nodes": [], "edges": [], "hyperedges": []} + with patch("graphify.llm._call_acp", return_value=result) as call: + assert llm.extract_files_direct([source], backend="acp", root=tmp_path) is result + assert call.called + + +def test_codex_cli_alias_routes_through_acp(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("GRAPHIFY_CODEX_BIN", raising=False) + monkeypatch.delenv("GRAPHIFY_ACP_BIN", raising=False) + llm._WARNED_BACKEND_ALIASES.clear() + source = tmp_path / "note.md" + source.write_text("# Compatibility route\n") + result = {"nodes": [], "edges": [], "hyperedges": []} + with patch("graphify.llm._call_acp", return_value=result) as call: + assert llm.extract_files_direct([source], backend="codex-cli", root=tmp_path) is result + assert call.called + assert "deprecated" in capsys.readouterr().err + + +def test_codex_cli_alias_rejects_legacy_direct_binary(monkeypatch): + monkeypatch.setenv("GRAPHIFY_CODEX_BIN", "/nix/store/example/bin/codex") + monkeypatch.delenv("GRAPHIFY_ACP_BIN", raising=False) + with pytest.raises(RuntimeError, match="GRAPHIFY_ACP_BIN"): + llm._normalize_backend("codex-cli") + + +def test_call_acp_maps_usage_and_stop_reason(): + response = AcpResult( + text='{"nodes": [{"label": "source"}], "edges": [], "hyperedges": []}', + input_tokens=14, + output_tokens=7, + model="gpt-test", + stop_reason="end_turn", + ) + with patch("graphify.acp.run_acp", return_value=response): + parsed = llm._call_acp("source", model="gpt-test", max_tokens=512) + assert parsed["nodes"] == [{"label": "source"}] + assert parsed["input_tokens"] == 14 + assert parsed["output_tokens"] == 7 + assert parsed["model"] == "gpt-test" + assert parsed["finish_reason"] == "stop" + + +def test_acp_parallelism_is_serial_by_default(monkeypatch): + monkeypatch.delenv("GRAPHIFY_ACP_PARALLEL", raising=False) + assert llm._normalize_backend("acp") == "acp" + + +def test_acp_config_rejects_non_scalar_values(): + with pytest.raises(ValueError, match="strings or booleans"): + _json_object('{"nested": {"unsafe": true}}', "GRAPHIFY_ACP_CONFIG_JSON") + + +def test_official_sdk_transport_and_session_options(tmp_path, monkeypatch): + config_log = tmp_path / "config.jsonl" + fake_agent = Path(__file__).with_name("fake_acp_agent.py") + monkeypatch.setenv("GRAPHIFY_ACP_BIN", sys.executable) + monkeypatch.setenv("GRAPHIFY_ACP_ARGS_JSON", json.dumps([str(fake_agent)])) + monkeypatch.setenv("GRAPHIFY_ACP_CONFIG_JSON", '{"mode": "read-only"}') + monkeypatch.setenv("GRAPHIFY_FAKE_ACP_CONFIG_LOG", str(config_log)) + + result = run_acp("extract", model="gpt-test") + + assert result.text == '{"nodes": [], "edges": [], "hyperedges": []}' + assert result.input_tokens == 11 + assert result.output_tokens == 7 + options = [json.loads(line) for line in config_log.read_text().splitlines()] + assert {option["configId"]: option["value"] for option in options} == { + "model": "gpt-test", + "mode": "read-only", + } diff --git a/uv.lock b/uv.lock index 2fd137000..62b8fddf1 100644 --- a/uv.lock +++ b/uv.lock @@ -14,6 +14,18 @@ resolution-markers = [ "python_full_version < '3.11'", ] +[[package]] +name = "agent-client-protocol" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/93/396d02c91b387b2678f23081869ca47bd495fc6c78789022c683180cf5f1/agent_client_protocol-0.11.0.tar.gz", hash = "sha256:8920a3b27bdcc852fd7c73066f47ab53257dbfbe57b505e20a40c69f80a5391c", size = 87402, upload-time = "2026-07-05T17:07:56.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/48/a1367dded7765f5489f9840389949d5aeac53a73aeb1b4e6c0ab61e1617a/agent_client_protocol-0.11.0-py3-none-any.whl", hash = "sha256:2d8570cd4911f8af9fbab4808f7b05f09204a756f346c7409ecdcae3f37cdb2f", size = 68089, upload-time = "2026-07-05T17:07:55.518Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -1090,7 +1102,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.13" +version = "0.9.18" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1127,7 +1139,11 @@ dependencies = [ ] [package.optional-dependencies] +acp = [ + { name = "agent-client-protocol" }, +] all = [ + { name = "agent-client-protocol" }, { name = "anthropic" }, { name = "boto3" }, { name = "falkordb" }, @@ -1249,6 +1265,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "agent-client-protocol", marker = "extra == 'acp'", specifier = ">=0.10.1,<0.12" }, + { name = "agent-client-protocol", marker = "extra == 'all'", specifier = ">=0.10.1,<0.12" }, { name = "anthropic", marker = "extra == 'all'" }, { name = "anthropic", marker = "extra == 'anthropic'" }, { name = "boto3", marker = "extra == 'all'" }, @@ -1333,7 +1351,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "acp", "chinese", "sql", "pascal", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [ From a8e937f8845e01755d81513db3cc7d7dffb0ebdd Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 18 Jul 2026 15:38:45 +0200 Subject: [PATCH 17/18] fix: drain final ACP update before return --- graphify/acp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/graphify/acp.py b/graphify/acp.py index 382ff044e..535e39c40 100644 --- a/graphify/acp.py +++ b/graphify/acp.py @@ -168,6 +168,11 @@ async def _run_acp( content.append(image_block(image.b64, image.media_type, uri=str(image.path))) response = await asyncio.wait_for(connection.prompt(session_id, content), timeout=timeout) + # The SDK may deliver the final session/update notification just after + # the prompt response future resolves. Let that callback run before + # collecting the streamed assistant text. + await asyncio.sleep(0) + usage = getattr(response, "usage", None) or client.usage input_tokens = int(getattr(usage, "input_tokens", 0) or 0) output_tokens = int(getattr(usage, "output_tokens", 0) or 0) From 01ee9fbccbac3315aa06604d21109e6a3ed4c7f7 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sun, 19 Jul 2026 09:07:01 +0200 Subject: [PATCH 18/18] fix: isolate ACP adapter environment Forward only the minimal ACP child environment by default and require explicit names for provider-specific variables. Normalize the deprecated codex-cli alias before community-labeling concurrency guards so every ACP path shares the serial session policy. --- README.md | 1 + graphify/acp.py | 63 +++++++++++++++++++++++++++++++++++++-- graphify/llm.py | 4 +++ tests/test_acp_backend.py | 35 +++++++++++++++++++++- tests/test_labeling.py | 12 ++++++++ 5 files changed, 112 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0e309af7b..39b923f17 100644 --- a/README.md +++ b/README.md @@ -600,6 +600,7 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_ACP_CONFIG_JSON` | JSON object of ACP session configuration options | optional — defaults to read-only mode | | `GRAPHIFY_ACP_MODEL` | Optional model selected through ACP session configuration | optional — adapter default | | `GRAPHIFY_ACP_PARALLEL` | Allow concurrent ACP extraction sessions | optional — default is serial | +| `GRAPHIFY_CHILD_ENV_ALLOWLIST` | Comma-separated additional environment variable names forwarded to the ACP adapter | optional — unrelated parent secrets are not inherited | | `GRAPHIFY_MAX_WORKERS` | AST parallelism thread count | optional — also `--max-workers` flag | | `GRAPHIFY_MAX_OUTPUT_TOKENS` | Raise output cap for dense corpora | optional — e.g. `32768` for large files | | `GRAPHIFY_API_TIMEOUT` | Per-call timeout in seconds for HTTP, claude-cli, ACP, and SDK backends (default: 600) | optional — also `--api-timeout` flag | diff --git a/graphify/acp.py b/graphify/acp.py index 535e39c40..bb2593103 100644 --- a/graphify/acp.py +++ b/graphify/acp.py @@ -10,6 +10,7 @@ import asyncio import json import os +import re import shutil import tempfile from collections.abc import Mapping @@ -28,6 +29,65 @@ class AcpResult: stop_reason: str = "end_turn" +_ACP_CHILD_ENV_DEFAULTS = frozenset( + { + # Process plumbing and harmless terminal/runtime preferences. + "HOME", + "PATH", + "LANG", + "LC_ALL", + "LC_CTYPE", + "LANGUAGE", + "TMPDIR", + "TMP", + "TEMP", + "NO_BROWSER", + "TERM", + "TERM_PROGRAM", + "COLORTERM", + "CI", + "NO_COLOR", + # Standard user-data/config locations used by ACP adapters. CODEX_HOME + # is the provider-managed subscription location used by codex-acp. + "XDG_CONFIG_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "XDG_RUNTIME_DIR", + "CODEX_HOME", + } +) +_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +def _child_environment() -> dict[str, str]: + """Build the minimal environment inherited by an ACP adapter. + + Repository text is untrusted input and ACP adapters may expose tools, so + inheriting the complete Graphify environment would unnecessarily expose + unrelated API keys and tokens. Provider-specific variables remain + possible through the explicit, comma-separated + ``GRAPHIFY_CHILD_ENV_ALLOWLIST`` contract. + """ + names = set(_ACP_CHILD_ENV_DEFAULTS) + requested = os.environ.get("GRAPHIFY_CHILD_ENV_ALLOWLIST", "") + for raw_name in requested.split(","): + name = raw_name.strip() + if not name: + continue + if _ENV_NAME.fullmatch(name) is None: + raise ValueError( + "GRAPHIFY_CHILD_ENV_ALLOWLIST contains an invalid environment " + f"variable name: {name!r}" + ) + names.add(name) + + environment = {name: os.environ[name] for name in names if name in os.environ} + # This is an ACP client invariant, not an inherited caller preference. + environment["NO_BROWSER"] = "1" + return environment + + def _json_object(value: str, variable: str) -> dict[str, Any]: if not value.strip(): return {} @@ -118,8 +178,7 @@ async def _run_acp( "Set GRAPHIFY_ACP_BIN to the provider command." ) - environment = os.environ.copy() - environment.setdefault("NO_BROWSER", "1") + environment = _child_environment() client = _GraphifyClient() async with spawn_agent_process( client, diff --git a/graphify/llm.py b/graphify/llm.py index b4a41d042..88b8427d4 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -2920,6 +2920,10 @@ def label_communities( written. Callers that want graceful degradation should use :func:`generate_community_labels`. """ + # Keep the deprecated spelling on the same ACP transport and concurrency + # policy as corpus extraction. Without normalization here, community + # labeling could bypass ACP's default serial session guard. + backend = _normalize_backend(backend) labels = _placeholder_community_labels(communities) cap = len(communities) if max_communities is None else max_communities lines, labeled_cids = _community_label_lines(G, communities, gods, cap, top_k) diff --git a/tests/test_acp_backend.py b/tests/test_acp_backend.py index 46525e28c..88ec547af 100644 --- a/tests/test_acp_backend.py +++ b/tests/test_acp_backend.py @@ -9,7 +9,7 @@ import pytest from graphify import llm -from graphify.acp import AcpResult, _json_object, run_acp +from graphify.acp import AcpResult, _child_environment, _json_object, run_acp def test_acp_backend_requires_no_api_key(): @@ -74,6 +74,38 @@ def test_acp_config_rejects_non_scalar_values(): _json_object('{"nested": {"unsafe": true}}', "GRAPHIFY_ACP_CONFIG_JSON") +def test_acp_child_environment_does_not_forward_secrets(monkeypatch): + monkeypatch.setenv("HOME", "/tmp/graphify-home") + monkeypatch.setenv("PATH", "/bin") + monkeypatch.setenv("CODEX_HOME", "/tmp/codex-home") + monkeypatch.setenv("OPENAI_API_KEY", "secret-openai") + monkeypatch.setenv("GITHUB_TOKEN", "secret-github") + monkeypatch.setenv("ANTHROPIC_API_KEY", "secret-anthropic") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret-aws") + monkeypatch.setenv("ACP_PROVIDER_TOKEN", "explicit-provider-token") + monkeypatch.setenv("GRAPHIFY_CHILD_ENV_ALLOWLIST", "ACP_PROVIDER_TOKEN") + + environment = _child_environment() + + assert environment["HOME"] == "/tmp/graphify-home" + assert environment["CODEX_HOME"] == "/tmp/codex-home" + assert environment["ACP_PROVIDER_TOKEN"] == "explicit-provider-token" + for secret_name in ( + "OPENAI_API_KEY", + "GITHUB_TOKEN", + "ANTHROPIC_API_KEY", + "AWS_SECRET_ACCESS_KEY", + ): + assert secret_name not in environment + assert environment["NO_BROWSER"] == "1" + + +def test_acp_child_environment_rejects_invalid_allowlist_names(monkeypatch): + monkeypatch.setenv("GRAPHIFY_CHILD_ENV_ALLOWLIST", "VALID_NAME,NOT-VALID") + with pytest.raises(ValueError, match="invalid environment variable name"): + _child_environment() + + def test_official_sdk_transport_and_session_options(tmp_path, monkeypatch): config_log = tmp_path / "config.jsonl" fake_agent = Path(__file__).with_name("fake_acp_agent.py") @@ -81,6 +113,7 @@ def test_official_sdk_transport_and_session_options(tmp_path, monkeypatch): monkeypatch.setenv("GRAPHIFY_ACP_ARGS_JSON", json.dumps([str(fake_agent)])) monkeypatch.setenv("GRAPHIFY_ACP_CONFIG_JSON", '{"mode": "read-only"}') monkeypatch.setenv("GRAPHIFY_FAKE_ACP_CONFIG_LOG", str(config_log)) + monkeypatch.setenv("GRAPHIFY_CHILD_ENV_ALLOWLIST", "GRAPHIFY_FAKE_ACP_CONFIG_LOG") result = run_acp("extract", model="gpt-test") diff --git a/tests/test_labeling.py b/tests/test_labeling.py index 7acbd30e6..b0229090d 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -412,6 +412,18 @@ def test_label_communities_forces_serial_for_ollama(monkeypatch): assert state["peak"] == 1, "ollama must be forced serial" +def test_label_communities_forces_serial_for_codex_cli_alias(monkeypatch): + """The deprecated alias must inherit ACP's serial session policy.""" + G, communities = _many_communities(8) + fake_batch, state = _peak_tracker() + monkeypatch.setattr("graphify.llm._label_batch_with_retry", fake_batch) + monkeypatch.delenv("GRAPHIFY_ACP_PARALLEL", raising=False) + monkeypatch.delenv("GRAPHIFY_CODEX_BIN", raising=False) + monkeypatch.delenv("GRAPHIFY_ACP_BIN", raising=False) + label_communities(G, communities, backend="codex-cli", batch_size=1, max_concurrency=8) + assert state["peak"] == 1, "codex-cli alias must inherit ACP's serial session policy" + + def test_label_communities_salvages_truncated_reply(monkeypatch): # #1690: a reply truncated mid-object (a stingy token budget or model # preamble) used to hard-fail the whole batch with `Expecting value: line 1