diff --git a/graphify/detect.py b/graphify/detect.py index c2638a0ef..7193d5e33 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.pkb', '.pks', '.pls', '.plb', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index bc10f7d7d..2c9e52afd 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3904,6 +3904,10 @@ def add_existing_edge(edge: dict) -> None: ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".pkb": extract_sql, + ".pks": extract_sql, + ".pls": extract_sql, + ".plb": extract_sql, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, @@ -3947,6 +3951,10 @@ def add_existing_edge(edge: dict) -> None: # extract() to tell the user which extra restores the language. _EXTRA_FOR_EXTENSION = { ".sql": "sql", + ".pkb": "sql", + ".pks": "sql", + ".pls": "sql", + ".plb": "sql", ".tf": "terraform", ".tfvars": "terraform", ".hcl": "terraform", diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index b9ce6fd8c..d1aa3bee8 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -8,15 +8,32 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: - """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" + """Extract tables, views, functions, and relationships from .sql/.pkb/.pks files via tree-sitter. + + Supports both tree-sitter-sql (standard SQL) and tree-sitter-plsql (Oracle PL/SQL). + PL/SQL grammar is tried first, falling back to standard SQL. + """ try: - import tree_sitter_sql as tssql from tree_sitter import Language, Parser except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} + return {"nodes": [], "edges": [], "error": "tree_sitter not installed. Run: pip install tree-sitter"} + + try: + import tree_sitter_plsql as tsplsql + HAS_PLSQL = True + except ImportError: + HAS_PLSQL = False + try: + import tree_sitter_sql as tssql + except ImportError: + return {"nodes": [], "edges": [], + "error": "neither tree_sitter_plsql nor tree_sitter_sql installed"} try: - language = Language(tssql.language()) + if HAS_PLSQL: + language = Language(tsplsql.language()) + else: + language = Language(tssql.language()) parser = Parser(language) source = ( content.encode("utf-8") if isinstance(content, str) @@ -42,6 +59,11 @@ def _read(n) -> str: return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") def _obj_name(n) -> str | None: + if HAS_PLSQL: + # PL/SQL grammar uses identifier (not object_reference) for names + for c in n.children: + if c.type == "identifier": + return _read(c) for c in n.children: if c.type == "object_reference": return _read(c) @@ -61,17 +83,153 @@ def _add_edge(src: str, tgt: str, relation: str, line: int) -> None: "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) + def _walk_from_refs(node, caller_nid: str, line: int) -> None: + """Recursively find FROM/JOIN table references inside a node.""" + if node.type in ("from", "join"): + for c in node.children: + if c.type == "relation": + for cc in c.children: + if cc.type == "object_reference": + tbl = _read(cc) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + c.start_point[0] + 1) + + # PL/SQL: walk into sql_statement_* nodes for table references + if HAS_PLSQL and node.type.startswith("sql_statement_"): + _walk_plsql_sql_stmt(node, caller_nid, line) + + for child in node.children: + _walk_from_refs(child, caller_nid, line) + + def _walk_plsql_sql_stmt(node, caller_nid: str, line: int) -> None: + """Extract table references from PL/SQL SQL statement nodes. + + PL/SQL wraps SQL statements in sql_statement_select/insert/update/delete/merge + nodes. Table references are inside table_list > table_list_element > referenced_element > identifier. + Also handles INTO clause (INSERT target table) and UPDATE target table. + """ + t = node.type + + if t == "sql_statement_select": + for child in node.children: + if child.type == "kw_from": + # Walk for table_list + for sibling in node.children: + if sibling.type == "table_list": + for tbl_elem in sibling.children: + if tbl_elem.type == "table_list_element": + for ref in tbl_elem.children: + if ref.type == "referenced_element": + for id_node in ref.children: + if id_node.type == "identifier": + tbl = _read(id_node) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + id_node.start_point[0] + 1) + + elif t in ("sql_statement_insert", "sql_statement_update", "sql_statement_delete"): + # INSERT/UPDATE/DELETE — extract target table and FROM references + for child in node.children: + if child.type == "single_table_insert": + for sub in child.children: + if sub.type == "referenced_element": + for id_node in sub.children: + if id_node.type == "identifier": + tbl = _read(id_node) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + id_node.start_point[0] + 1) + elif sub.type == "kw_from": + # Subquery FROM inside INSERT ... SELECT + for sibling in child.children: + if sibling.type == "table_list": + _walk_plsql_table_list(sibling, caller_nid) + elif child.type == "referenced_element": + for id_node in child.children: + if id_node.type == "identifier": + tbl = _read(id_node) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + id_node.start_point[0] + 1) + + def _walk_plsql_table_list(node, caller_nid: str) -> None: + """Walk a table_list node for table references.""" + for tbl_elem in node.children: + if tbl_elem.type == "table_list_element": + for ref in tbl_elem.children: + if ref.type == "referenced_element": + for id_node in ref.children: + if id_node.type == "identifier": + tbl = _read(id_node) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + id_node.start_point[0] + 1) + + def _plsql_create_table_name(node) -> str | None: + """Get table name from PL/SQL CREATE TABLE (uses identifier child).""" + for c in node.children: + if c.type == "identifier": + return _read(c) + return None + + def _plsql_extract_references(node, tbl_nid: str, line: int) -> None: + """Extract REFERENCES edges from PL/SQL CREATE TABLE columns/constraints.""" + seen_refs: set[str] = set() + + for child in node.children: + if child.type == "table_element": + for sub in child.children: + if sub.type == "table_column_definition": + # Inline REFERENCES — PL/SQL grammar may produce ERROR here, + # fall back to regex. + ref_text = _read(sub) + for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", ref_text, re.IGNORECASE): + ref_name = rm.group(1) + if ref_name.lower() not in seen_refs: + seen_refs.add(ref_name.lower()) + ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) + _add_edge(tbl_nid, ref_nid, "references", line) + + elif sub.type == "table_constraint": + for constraint in sub.children: + if constraint.type == "table_constraint_foreign_key": + ref_name = None + found_ref = False + for cc in constraint.children: + if cc.type == "kw_references": + found_ref = True + elif found_ref and cc.type == "referenced_element": + for id_node in cc.children: + if id_node.type == "identifier": + ref_name = _read(id_node) + break + if ref_name: + break + if ref_name and ref_name.lower() not in seen_refs: + seen_refs.add(ref_name.lower()) + ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) + _add_edge(tbl_nid, ref_nid, "references", line) + def walk(node) -> None: t = node.type line = node.start_point[0] + 1 if t == "create_table": - name = _obj_name(node) + if HAS_PLSQL: + name = _plsql_create_table_name(node) + else: + name = _obj_name(node) if name: nid = _make_id(stem, name) _add_node(nid, name, line) table_nids[name.lower()] = nid - # Foreign key REFERENCES + + if HAS_PLSQL: + # PL/SQL table structure: table_element > table_column_definition / table_constraint + _plsql_extract_references(node, nid, line) + else: + # Standard SQL structure for col in node.children: if col.type == "column_definitions": has_error = any(cd.type == "ERROR" for cd in col.children) @@ -191,6 +349,52 @@ def walk(node) -> None: tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name) _add_edge(trig_nid, tbl_nid, "triggers", line) + # ── PL/SQL-specific node types ────────────────────────────────────── + elif HAS_PLSQL and t == "create_package": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + # Walk children for function/procedure/type/cursor declarations + _walk_from_refs(node, nid, line) + + elif HAS_PLSQL and t == "create_package_body": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + _walk_from_refs(node, nid, line) + + elif HAS_PLSQL and t in ("function_declaration", "function_definition"): + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + elif HAS_PLSQL and t in ("procedure_declaration", "procedure_definition"): + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + elif HAS_PLSQL and t in ( + "cursor_definition", + "type_definition_ref_cursor", + "type_definition_record", + "type_definition_collection", + "type_definition_sub", + ): + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + + # ── End PL/SQL-specific nodes ─────────────────────────────────────── + elif t == "ERROR": # tree-sitter-sql cannot parse PL/pgSQL CREATE FUNCTION/PROCEDURE # bodies (OUT/INOUT params, tagged dollar quotes, PERFORM, :=) and @@ -250,26 +454,15 @@ def walk(node) -> None: for child in node.children: walk(child) - def _walk_from_refs(node, caller_nid: str, line: int) -> None: - """Recursively find FROM/JOIN table references inside a node.""" - if node.type in ("from", "join"): - for c in node.children: - if c.type == "relation": - for cc in c.children: - if cc.type == "object_reference": - tbl = _read(cc) - tbl_nid = _make_id(stem, tbl) - _add_edge(caller_nid, tbl_nid, "reads_from", - c.start_point[0] + 1) - for child in node.children: - _walk_from_refs(child, caller_nid, line) - for stmt in root.children: if stmt.type == "statement": for child in stmt.children: walk(child) elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function", "ERROR"): walk(stmt) + elif HAS_PLSQL: + # PL/SQL grammar: top-level nodes have no "statement" wrapper + walk(stmt) # Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree # (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely). @@ -292,4 +485,90 @@ def _walk_from_refs(node, caller_nid: str, line: int) -> None: _add_edge(tbl_nid, ref_nid, "references", tbl_line) emitted.add((tbl_nid, ref_nid)) + # ── PL/SQL regex fallback for SQL*Plus extracts ──────────────────────── + # These files start with PACKAGE BODY name (no CREATE OR REPLACE), which + # the PL/SQL grammar doesn't recognize as package headers. Use regex to + # recover package/procedure/function/cursor/type nodes and table references. + if HAS_PLSQL: + src_text = source.decode("utf-8", errors="replace") + + # 1. Package / Package Body declarations + for m in re.finditer( + r"PACKAGE\s+(?:BODY\s+)?(\w+)\s+(?:IS|AS)", + src_text, re.IGNORECASE, + ): + name = m.group(1) + nid = _make_id(stem, name) + pkg_line = src_text[: m.start()].count("\n") + 1 + _add_node(nid, name, pkg_line) + + # 2. Procedure declarations (outside BEGIN/END blocks) + _NON_TABLES = { + "select", "where", "null", "true", "false", + "first", "rows", "next", "only", + } + seen_proc_func: set[str] = set() + for m in re.finditer( + r"(?:^|\n)\s*(?:PROCEDURE|FUNCTION)\s+(\w+)", + src_text, re.IGNORECASE, + ): + name = m.group(1) + if name.upper() in _NON_TABLES or name.lower() in seen_proc_func: + continue + seen_proc_func.add(name.lower()) + nid = _make_id(stem, name) + func_line = src_text[: m.start()].count("\n") + 1 + _add_node(nid, f"{name}()", func_line) + + # 3. Cursor definitions + seen_cursors: set[str] = set() + for m in re.finditer( + r"CURSOR\s+(\w+)\s+IS", + src_text, re.IGNORECASE, + ): + name = m.group(1) + if name.lower() in seen_cursors: + continue + seen_cursors.add(name.lower()) + nid = _make_id(stem, name) + cur_line = src_text[: m.start()].count("\n") + 1 + _add_node(nid, name, cur_line) + + # 4. Type definitions (TYPE name IS REF CURSOR | RECORD | TABLE OF | SUBTYPE) + seen_types: set[str] = set() + for m in re.finditer( + r"(?:TYPE|SUBTYPE)\s+(\w+)\s+IS", + src_text, re.IGNORECASE, + ): + name = m.group(1) + if name.lower() in seen_types: + continue + seen_types.add(name.lower()) + nid = _make_id(stem, name) + type_line = src_text[: m.start()].count("\n") + 1 + _add_node(nid, name, type_line) + + # 5. FROM/JOIN/INTO table references (heavy regex, only if we found packages) + em_reade = {(e["source"], e["target"]) for e in edges if e["relation"] == "reads_from"} + has_pkg = any(n.get("label", "") for n in nodes if n["id"] != file_nid) + if has_pkg: + for m in re.finditer(r"\b(?:FROM|JOIN)\s+(\w+)", src_text, re.IGNORECASE): + # Heuristic: only include if it looks like a table (not a keyword or variable) + tbl = m.group(1) + if tbl.upper() in _NON_TABLES or len(tbl) <= 2: + continue + tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) + if tbl_nid not in seen_ids: + # Don't create new nodes for table refs — just emit the edge + pass + # Find the nearest package/procedure node as the caller + tbl_line = src_text[: m.start()].count("\n") + 1 + for node in reversed(nodes): + if node["id"] != file_nid: + caller_nid = node["id"] + if (caller_nid, tbl_nid) not in em_reade: + em_reade.add((caller_nid, tbl_nid)) + _add_edge(caller_nid, tbl_nid, "reads_from", tbl_line) + break + return {"nodes": nodes, "edges": edges}