diff --git a/Makefile.cbm b/Makefile.cbm index 28eb21e27..b2003265a 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -345,6 +345,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_gitdiff.c \ src/pipeline/pass_configures.c \ src/pipeline/pass_configlink.c \ + src/pipeline/pass_doclinks.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ src/pipeline/pass_envscan.c \ @@ -559,7 +560,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..88dd08307 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1366,8 +1366,8 @@ static const char skill_content[] = "\n" "## Edge Types\n" "CALLS, HTTP_CALLS, ASYNC_CALLS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n" - "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, FILE_CHANGES_WITH,\n" - "SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" + "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n" + "FILE_CHANGES_WITH, SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" "CONTAINS_PACKAGE\n" "\n" "## Cypher Examples (for query_graph)\n" diff --git a/src/pipeline/pass_doclinks.c b/src/pipeline/pass_doclinks.c new file mode 100644 index 000000000..d27a22ce0 --- /dev/null +++ b/src/pipeline/pass_doclinks.c @@ -0,0 +1,495 @@ +/* + * pass_doclinks.c — Documentation → file reference linking (pre-dump pass). + * + * Markdown docs reference other repo files constantly — a coding-standards + * doc links to the modules it governs, a README points at entry points — + * but none of that surfaced as graph edges, so fan-in queries were blind to + * documentation hubs: on a docs-heavy repo the top fan-in answer was off by + * an order of magnitude because the most-referenced doc had zero inbound + * edges. + * + * Three strategies emit REFERENCES_FILE edges between EXISTING File nodes + * (targets that don't resolve to an indexed file are dropped — the pass + * never invents nodes): + * MD 1. Inline link: [text](relative/path.ext) (not http/mailto/#anchor) + * MD 2. Backtick path: `path/with/slash.ext` or `file.ext` + * MD 3. Bare mention: relative/path.ext (slash + extension) + * + * Targets resolve relative to the referencing file's directory AND the repo + * root (docs are written both ways). Repeated references between the same + * file pair are collapsed to one edge carrying a "count" property; the edge + * keeps the highest-confidence strategy that matched. + * + * Operates on the graph buffer before dump to .db file. + */ +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/compat_fs.h" +#include "foundation/limits.h" + +#include +#include +#include +#include +#include +#include + +#define SLEN(s) (sizeof(s) - SKIP_ONE) + +/* ── Doc link confidence scores ──────────────────────────────────── */ +/* Markdown strategies */ +#define DOCLINK_MD_INLINE 0.95 +#define DOCLINK_MD_BACKTICK 0.85 +#define DOCLINK_MD_BARE 0.70 + +/* Edge type emitted by this pass. */ +#define DOCLINK_EDGE_TYPE "REFERENCES_FILE" + +enum { + DOCLINK_MAX_REFS = CBM_SZ_256, /* distinct targets per referencing file */ + DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */ +}; + +/* ── Path classification ─────────────────────────────────────────── */ + +/* Extension of the path's basename (including the dot), or NULL. */ +static const char *doclink_path_ext(const char *path) { + if (!path) { + return NULL; + } + const char *base = strrchr(path, '/'); + base = base ? base + SKIP_ONE : path; + return strrchr(base, '.'); +} + +static bool doclink_is_markdown_path(const char *path) { + const char *ext = doclink_path_ext(path); + return ext && (strcmp(ext, ".md") == 0 || strcmp(ext, ".mdx") == 0); +} + +/* ── File reading (mirrors pass_semantic.c read_file, minus TS pad) ── */ + +static char *doclink_read_file(const char *path) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + char *buf = malloc((size_t)size + SKIP_ONE); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + if (nread > (size_t)size) { + nread = (size_t)size; + } + buf[nread] = '\0'; + return buf; +} + +/* ── Path normalization + resolution ─────────────────────────────── */ + +/* Normalize "a/./b/../c" into "a/c". Rejects paths that escape the repo + * root (leading ".."), empty results, and over-long/over-deep inputs. */ +static bool doclink_normalize(const char *in, char *out, size_t out_sz) { + size_t seg_starts[DOCLINK_MAX_SEGS]; + int depth = 0; + size_t out_len = 0; + const char *p = in; + out[0] = '\0'; + while (*p) { + const char *seg = p; + const char *slash = strchr(p, '/'); + size_t seg_len = slash ? (size_t)(slash - p) : strlen(p); + p = slash ? slash + SKIP_ONE : p + seg_len; + if (seg_len == 0 || (seg_len == SKIP_ONE && seg[0] == '.')) { + continue; + } + if (seg_len == PAIR_LEN && seg[0] == '.' && seg[SKIP_ONE] == '.') { + if (depth == 0) { + return false; /* escapes the repo root */ + } + depth--; + out_len = seg_starts[depth]; + out[out_len] = '\0'; + continue; + } + if (depth >= DOCLINK_MAX_SEGS || out_len + seg_len + PAIR_LEN >= out_sz) { + return false; + } + seg_starts[depth++] = out_len; + if (out_len > 0) { + out[out_len++] = '/'; + } + memcpy(out + out_len, seg, seg_len); + out_len += seg_len; + out[out_len] = '\0'; + } + return out_len > 0; +} + +/* Per-file accumulator: dedupes repeated references to the same target. */ +typedef struct { + int64_t target_id; + int count; + double confidence; + const char *strategy; /* static string literal */ +} doclink_ref_t; + +typedef struct { + cbm_gbuf_t *gb; + CBMHashTable *files_by_path; /* rel_path → cbm_gbuf_node_t* (borrowed) */ + const cbm_gbuf_node_t *src; /* referencing File node */ + char src_dir[CBM_SZ_512]; /* its directory ("" at repo root) */ + doclink_ref_t refs[DOCLINK_MAX_REFS]; + int ref_count; + bool truncated; +} doclink_ctx_t; + +/* Resolve a reference against the referencing file's directory, then the + * repo root, mirroring how humans write doc links. Returns the already- + * indexed File node or NULL — unresolvable references are dropped. */ +static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref) { + const char *r = ref; + while (r[0] == '.' && r[SKIP_ONE] == '/') { + r += PAIR_LEN; + } + if (r[0] == '/') { + r++; /* "/docs/x.md" is repo-root-relative by doc convention */ + } + if (r[0] == '\0') { + return NULL; + } + + char norm[CBM_SZ_512]; + if (dc->src_dir[0] != '\0') { + char joined[CBM_SZ_512]; + int n = snprintf(joined, sizeof(joined), "%s/%s", dc->src_dir, r); + if (n > 0 && (size_t)n < sizeof(joined) && doclink_normalize(joined, norm, sizeof(norm))) { + const cbm_gbuf_node_t *node = cbm_ht_get(dc->files_by_path, norm); + if (node) { + return node; + } + } + } + if (doclink_normalize(r, norm, sizeof(norm))) { + return cbm_ht_get(dc->files_by_path, norm); + } + return NULL; +} + +/* Record one match. Same-pair repeats bump the count; a higher-confidence + * strategy upgrades the edge's confidence + strategy label. */ +static void doclink_record(doclink_ctx_t *dc, const cbm_gbuf_node_t *target, double confidence, + const char *strategy) { + if (!target || target->id == dc->src->id) { + return; /* never self-reference */ + } + for (int i = 0; i < dc->ref_count; i++) { + if (dc->refs[i].target_id == target->id) { + dc->refs[i].count++; + if (confidence > dc->refs[i].confidence) { + dc->refs[i].confidence = confidence; + dc->refs[i].strategy = strategy; + } + return; + } + } + if (dc->ref_count >= DOCLINK_MAX_REFS) { + dc->truncated = true; + return; + } + dc->refs[dc->ref_count].target_id = target->id; + dc->refs[dc->ref_count].count = SKIP_ONE; + dc->refs[dc->ref_count].confidence = confidence; + dc->refs[dc->ref_count].strategy = strategy; + dc->ref_count++; +} + +/* Emit accumulated references as REFERENCES_FILE edges. Returns edge count. */ +static int doclink_flush(doclink_ctx_t *dc) { + int emitted = 0; + for (int i = 0; i < dc->ref_count; i++) { + char props[CBM_SZ_256]; + (void)snprintf(props, sizeof(props), + "{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", + dc->refs[i].strategy, dc->refs[i].confidence, dc->refs[i].count); + if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, dc->refs[i].target_id, DOCLINK_EDGE_TYPE, + props) > 0) { + emitted++; + } + } + if (dc->truncated) { + char cap_buf[CBM_SZ_16]; + (void)snprintf(cap_buf, sizeof(cap_buf), "%d", DOCLINK_MAX_REFS); + cbm_log_info("doclinks.truncated", "file", dc->src->file_path ? dc->src->file_path : "", + "cap", cap_buf); + } + dc->ref_count = 0; + dc->truncated = false; + return emitted; +} + +/* ── Markdown scanning ───────────────────────────────────────────── */ + +/* Characters allowed in a path-shaped token (backtick / bare mention). */ +static bool doclink_token_pathlike(const char *tok) { + bool last_seg_has_dot = false; + for (const char *p = tok; *p; p++) { + unsigned char c = (unsigned char)*p; + if (c == '/') { + last_seg_has_dot = false; + continue; + } + if (c == '.') { + last_seg_has_dot = true; + continue; + } + if (!isalnum(c) && c != '_' && c != '-' && c != '+' && c != '@' && c != '~') { + return false; + } + } + /* the basename must carry an extension — bare words are not paths */ + return last_seg_has_dot; +} + +/* A markdown link target worth resolving: not a URL, mailto, or pure anchor. */ +static bool doclink_md_target_ok(const char *target) { + if (target[0] == '\0' || target[0] == '#') { + return false; + } + if (strstr(target, "://") != NULL || strncmp(target, "mailto:", SLEN("mailto:")) == 0) { + return false; + } + return true; +} + +/* MD 1: inline links [text](target). Consumed spans are blanked so the + * backtick / bare-mention scans below cannot re-match the same path. */ +static void doclink_scan_md_links(doclink_ctx_t *dc, char *line) { + char *p = line; + while ((p = strstr(p, "](")) != NULL) { + char *close = strchr(p + PAIR_LEN, ')'); + if (!close) { + return; + } + char target[CBM_SZ_512]; + size_t tlen = (size_t)(close - (p + PAIR_LEN)); + if (tlen < sizeof(target)) { + memcpy(target, p + PAIR_LEN, tlen); + target[tlen] = '\0'; + char *cut = strchr(target, ' '); /* [t](path "title") */ + if (cut) { + *cut = '\0'; + } + cut = strchr(target, '#'); /* [t](path#anchor) */ + if (cut) { + *cut = '\0'; + } + if (doclink_md_target_ok(target)) { + doclink_record(dc, doclink_resolve(dc, target), DOCLINK_MD_INLINE, + "md_inline_link"); + } + } + /* blank the whole [text](target) span, link text included, so a + * path-shaped link text is not re-counted as a bare mention */ + char *open = p; + while (open > line && *open != '[') { + open--; + } + if (*open != '[') { + open = p; + } + memset(open, ' ', (size_t)(close - open) + SKIP_ONE); + p = close + SKIP_ONE; + } +} + +/* MD 2: backtick-quoted paths `src/foo.c` / `build.sh`. */ +static void doclink_scan_md_backticks(doclink_ctx_t *dc, char *line) { + char *p = line; + while ((p = strchr(p, '`')) != NULL) { + char *end = strchr(p + SKIP_ONE, '`'); + if (!end) { + return; + } + char tok[CBM_SZ_512]; + size_t tlen = (size_t)(end - (p + SKIP_ONE)); + if (tlen > 0 && tlen < sizeof(tok)) { + memcpy(tok, p + SKIP_ONE, tlen); + tok[tlen] = '\0'; + if (doclink_token_pathlike(tok)) { + doclink_record(dc, doclink_resolve(dc, tok), DOCLINK_MD_BACKTICK, + "md_backtick_path"); + } + } + memset(p, ' ', (size_t)(end - p) + SKIP_ONE); + p = end + SKIP_ONE; + } +} + +static bool doclink_md_delim(char c) { + return isspace((unsigned char)c) || strchr("()[]{}<>\"',;:`*|", c) != NULL; +} + +/* MD 3: bare relative path mentions — must contain a slash AND an extension + * (and, via doclink_resolve, an indexed file) to count. */ +static void doclink_scan_md_bare(doclink_ctx_t *dc, const char *line) { + const char *p = line; + while (*p) { + while (*p && doclink_md_delim(*p)) { + p++; + } + const char *start = p; + while (*p && !doclink_md_delim(*p)) { + p++; + } + size_t tlen = (size_t)(p - start); + char tok[CBM_SZ_512]; + if (tlen == 0 || tlen >= sizeof(tok)) { + continue; + } + memcpy(tok, start, tlen); + tok[tlen] = '\0'; + while (tlen > 0 && tok[tlen - SKIP_ONE] == '.') { + tok[--tlen] = '\0'; /* sentence-ending period */ + } + if (strchr(tok, '/') != NULL && doclink_token_pathlike(tok)) { + doclink_record(dc, doclink_resolve(dc, tok), DOCLINK_MD_BARE, "md_bare_mention"); + } + } +} + +static void doclink_scan_md_line(doclink_ctx_t *dc, char *line) { + doclink_scan_md_links(dc, line); + doclink_scan_md_backticks(dc, line); + doclink_scan_md_bare(dc, line); +} + +/* ── Per-file driver ─────────────────────────────────────────────── */ + +/* Scan one referencing file's content line by line and emit its edges. */ +static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, const char *source) { + dc->src = node; + dc->ref_count = 0; + dc->truncated = false; + dc->src_dir[0] = '\0'; + const char *slash = strrchr(node->file_path, '/'); + if (slash) { + size_t dlen = (size_t)(slash - node->file_path); + if (dlen >= sizeof(dc->src_dir)) { + return 0; + } + memcpy(dc->src_dir, node->file_path, dlen); + dc->src_dir[dlen] = '\0'; + } + + const char *p = source; + char line[CBM_SZ_4K]; + while (*p) { + const char *eol = strchr(p, '\n'); + size_t line_len = eol ? (size_t)(eol - p) : strlen(p); + if (line_len >= sizeof(line)) { + line_len = sizeof(line) - SKIP_ONE; + } + memcpy(line, p, line_len); + line[line_len] = '\0'; + p = eol ? eol + SKIP_ONE : p + line_len; + + doclink_scan_md_line(dc, line); + } + return doclink_flush(dc); +} + +/* ── Pass entry point ────────────────────────────────────────────── */ + +/* True when at least one File node is a markdown file. */ +static bool doclink_has_doc_files(const cbm_gbuf_node_t *const *files, int file_count) { + for (int i = 0; i < file_count; i++) { + if (doclink_is_markdown_path(files[i]->file_path)) { + return true; + } + } + return false; +} + +/* Scan every markdown File node's on-disk content, emitting edges. + * md_edges receives the emitted edge count. */ +static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path, + const cbm_gbuf_node_t *const *files, int file_count, int *md_edges) { + for (int i = 0; i < file_count; i++) { + if (!files[i]->file_path || !doclink_is_markdown_path(files[i]->file_path)) { + continue; + } + + char abs_path[CBM_PATH_MAX]; + int n = snprintf(abs_path, sizeof(abs_path), "%s/%s", repo_path, files[i]->file_path); + if (n <= 0 || (size_t)n >= sizeof(abs_path)) { + continue; + } + char *source = doclink_read_file(abs_path); + if (!source) { + continue; + } + int emitted = doclink_scan_file(dc, files[i], source); + free(source); + *md_edges += emitted; + } +} + +int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) { + cbm_gbuf_t *gb = ctx->gbuf; + + const cbm_gbuf_node_t **files = NULL; + int file_count = 0; + if (cbm_gbuf_find_by_label(gb, "File", &files, &file_count) != 0 || file_count == 0) { + return 0; + } + + /* Early exit: no markdown/shell files means nothing to scan. */ + if (!doclink_has_doc_files(files, file_count)) { + cbm_log_info("doclinks.skip", "reason", "no_doc_files"); + return 0; + } + if (!ctx->repo_path) { + cbm_log_info("doclinks.skip", "reason", "no_repo_path"); + return 0; + } + + doclink_ctx_t dc; + memset(&dc, 0, sizeof(dc)); + dc.gb = gb; + dc.files_by_path = cbm_ht_create((uint32_t)file_count); + if (!dc.files_by_path) { + return 0; + } + for (int i = 0; i < file_count; i++) { + if (files[i]->file_path) { + /* key borrowed from the node (owned by gbuf, outlives the pass) */ + cbm_ht_set(dc.files_by_path, files[i]->file_path, (void *)files[i]); + } + } + + int md_edges = 0; + doclink_scan_repo(&dc, ctx->repo_path, files, file_count, &md_edges); + cbm_ht_free(dc.files_by_path); + + char buf1[CBM_SZ_16]; + (void)snprintf(buf1, sizeof(buf1), "%d", md_edges); + cbm_log_info("doclinks.strategy", "name", "markdown", "edges", buf1); + cbm_log_info("doclinks.done", "total", buf1); + + return md_edges; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 65ac75183..3df5f22e6 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -931,6 +931,9 @@ static void predump_sem(cbm_pipeline_ctx_t *ctx) { static void predump_cfg(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_configlink(ctx); } +static void predump_doclinks(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_doclinks(ctx); +} static void predump_complexity(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_complexity(ctx); } @@ -940,11 +943,12 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { const char *name; bool moderate_only; /* true = skip in fast mode */ } passes[] = { - {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, - {predump_route, "route_match", false}, {predump_sim, "similarity", true}, - {predump_sem, "semantic_edges", true}, {predump_complexity, "complexity", false}, + {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, + {predump_doclinks, "doclinks", false}, {predump_route, "route_match", false}, + {predump_sim, "similarity", true}, {predump_sem, "semantic_edges", true}, + {predump_complexity, "complexity", false}, }; - enum { PREDUMP_PASS_COUNT = 6 }; + enum { PREDUMP_PASS_COUNT = 7 }; struct timespec t; for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) { /* "moderate_only" passes (similarity/semantic edges) run in FULL, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index fca1c8f06..b51ff69ed 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1438,6 +1438,14 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file return rc < 0 ? rc : CBM_NOT_FOUND; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_doclinks(ctx); + cbm_log_info("pass.timing", "pass", "incr_doclinks", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + if (rc < 0 || cbm_pipeline_check_cancel(ctx)) { + return rc < 0 ? rc : CBM_NOT_FOUND; + } + /* SIMILAR_TO + SEMANTICALLY_RELATED edges only in moderate/full modes */ if (ctx->mode <= CBM_MODE_MODERATE) { cbm_clock_gettime(CLOCK_MONOTONIC, &t); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 754fa4ffa..3a4b65664 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -617,6 +617,9 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); /* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); +/* Pre-dump pass: markdown/shell → file REFERENCES_FILE linking. */ +int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); + /* Pre-dump pass: SIMILAR_TO edges via MinHash fingerprinting. */ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx); diff --git a/tests/test_doclinks.c b/tests/test_doclinks.c new file mode 100644 index 000000000..a03542d9b --- /dev/null +++ b/tests/test_doclinks.c @@ -0,0 +1,312 @@ +/* + * test_doclinks.c — Tests for markdown/shell → file reference linking. + * + * Unit-test approach mirrors test_configlink.c: create real files in a + * tmpdir (the pass reads content from disk), set up File nodes in a gbuf, + * run the pass, check REFERENCES_FILE edges. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +/* ── Helpers ─────────────────────────────────────────────────────── */ + +/* Run the pass with a minimal ctx (same shape as run_configlink). */ +static int run_doclinks(cbm_gbuf_t *gb, const char *project, const char *repo_path) { + atomic_int cancelled; + atomic_init(&cancelled, 0); + cbm_pipeline_ctx_t ctx = { + .project_name = project, + .repo_path = repo_path, + .gbuf = gb, + .cancelled = &cancelled, + }; + return cbm_pipeline_pass_doclinks(&ctx); +} + +/* Create a File node the way pass_structure does: QN via fqn_compute with + * "__file__", file_path = repo-relative path. Returns the node id. */ +static int64_t add_file_node(cbm_gbuf_t *gb, const char *project, const char *rel) { + char *qn = cbm_pipeline_fqn_compute(project, rel, "__file__"); + const char *slash = strrchr(rel, '/'); + const char *basename = slash ? slash + 1 : rel; + int64_t id = cbm_gbuf_upsert_node(gb, "File", basename, qn, rel, 0, 0, NULL); + free(qn); + return id; +} + +/* Find the REFERENCES_FILE edge between two node ids. NULL if absent. */ +static const cbm_gbuf_edge_t *find_ref_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "REFERENCES_FILE", &edges, &count); + for (int i = 0; i < count; i++) { + if (edges[i]->source_id == src && edges[i]->target_id == tgt) { + return edges[i]; + } + } + return NULL; +} + +/* True when the edge's props JSON carries the given strategy. */ +static bool edge_has_strategy(const cbm_gbuf_edge_t *e, const char *strategy) { + char needle[64]; + snprintf(needle, sizeof(needle), "\"strategy\":\"%s\"", strategy); + return e && e->properties_json && strstr(e->properties_json, needle) != NULL; +} + +/* Total REFERENCES_FILE edge count. */ +static int ref_edge_count(cbm_gbuf_t *gb) { + return cbm_gbuf_edge_count_by_type(gb, "REFERENCES_FILE"); +} + +/* Fixture: tmpdir + project + gbuf. */ +typedef struct { + char tmpdir[256]; + char *project; + cbm_gbuf_t *gb; +} dl_fix_t; + +static bool dl_fix_init(dl_fix_t *fx) { + snprintf(fx->tmpdir, sizeof(fx->tmpdir), "/tmp/cbm_doclinks_XXXXXX"); + if (!cbm_mkdtemp(fx->tmpdir)) { + return false; + } + fx->project = cbm_project_name_from_path(fx->tmpdir); + fx->gb = cbm_gbuf_new(fx->project, fx->tmpdir); + return fx->gb != NULL; +} + +static void dl_fix_free(dl_fix_t *fx) { + cbm_gbuf_free(fx->gb); + free(fx->project); + th_rmtree(fx->tmpdir); +} + +/* ── Markdown: inline link ───────────────────────────────────────── */ + +TEST(doclinks_md_inline_link) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "docs/guide.md"), + "# Guide\n\nSee [the build script](../scripts/build.sh) for details.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\necho build\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/guide.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_GT(n, 0); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_inline_link")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.95")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: backtick path ─────────────────────────────────────── */ + +TEST(doclinks_md_backtick_path) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Run `scripts/build.sh` before pushing. `not_a_file.xyz` is unknown.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_backtick_path")); + /* `not_a_file.xyz` has no File node → no invented target */ + ASSERT_EQ(ref_edge_count(fx.gb), 1); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: bare mention ──────────────────────────────────────── */ + +TEST(doclinks_md_bare_mention) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "STANDARDS.md"), + "All handlers live in src/handlers.c and follow the pattern there.\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/handlers.c"), "int h(void) { return 0; }\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "STANDARDS.md"); + int64_t code_id = add_file_node(fx.gb, fx.project, "src/handlers.c"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, code_id); + ASSERT_NOT_NULL(e); + ASSERT_TRUE(edge_has_strategy(e, "md_bare_mention")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.70")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: non-file links ignored (http / mailto / #anchor) ──── */ + +TEST(doclinks_md_non_file_link_ignored) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "See [the site](https://example.com/scripts/build.sh) or\n" + "[mail us](mailto:dev@example.com) or [below](#usage).\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Markdown: anchor suffix on a file link still resolves ───────── */ + +TEST(doclinks_md_link_with_anchor_resolves_file) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See [setup](../INSTALL.md#quick-start).\n"); + th_write_file(TH_PATH(fx.tmpdir, "INSTALL.md"), "# Install\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/a.md"); + int64_t tgt_id = add_file_node(fx.gb, fx.project, "INSTALL.md"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, tgt_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Dedupe: repeated references collapse to one counted edge ────── */ + +TEST(doclinks_dedupe_keeps_count) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "Use [build](scripts/build.sh) daily.\n" + "Run `scripts/build.sh` first, then scripts/build.sh again.\n"); + th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n"); + + int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md"); + int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh"); + + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + /* three matches, ONE edge */ + ASSERT_EQ(n, 1); + ASSERT_EQ(ref_edge_count(fx.gb), 1); + + const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, script_id); + ASSERT_NOT_NULL(e); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"count\":3")); + /* highest-confidence match kind wins the edge label */ + ASSERT_TRUE(edge_has_strategy(e, "md_inline_link")); + ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.95")); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Path resolution: relative to the referencing file's directory ── */ + +TEST(doclinks_resolves_relative_to_file_dir) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + /* docs/deep/page.md links an upward "../../src/main.c" path and a + * sibling-relative "notes.md" path, neither repo-root-relative. */ + th_write_file(TH_PATH(fx.tmpdir, "docs/deep/page.md"), + "See [main](../../src/main.c) and [notes](notes.md).\n"); + th_write_file(TH_PATH(fx.tmpdir, "docs/deep/notes.md"), "# notes\n"); + th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n"); + + int64_t page_id = add_file_node(fx.gb, fx.project, "docs/deep/page.md"); + int64_t notes_id = add_file_node(fx.gb, fx.project, "docs/deep/notes.md"); + int64_t main_id = add_file_node(fx.gb, fx.project, "src/main.c"); + + run_doclinks(fx.gb, fx.project, fx.tmpdir); + + ASSERT_NOT_NULL(find_ref_edge(fx.gb, page_id, main_id)); + ASSERT_NOT_NULL(find_ref_edge(fx.gb, page_id, notes_id)); + + dl_fix_free(&fx); + PASS(); +} + +/* ── Never invent nodes: unresolvable targets produce nothing ────── */ + +TEST(doclinks_unresolvable_target_no_edge) { + dl_fix_t fx; + ASSERT_TRUE(dl_fix_init(&fx)); + + th_write_file(TH_PATH(fx.tmpdir, "README.md"), + "See [gone](docs/removed.md) and `also/missing.sh`.\n"); + + add_file_node(fx.gb, fx.project, "README.md"); + + int node_count_before = cbm_gbuf_node_count(fx.gb); + int n = run_doclinks(fx.gb, fx.project, fx.tmpdir); + ASSERT_EQ(n, 0); + ASSERT_EQ(ref_edge_count(fx.gb), 0); + ASSERT_EQ(cbm_gbuf_node_count(fx.gb), node_count_before); + + dl_fix_free(&fx); + PASS(); +} + +/* ── NULL repo_path (configlink-style unit setups) is a clean skip ── */ + +TEST(doclinks_null_repo_path_skips) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + add_file_node(gb, "test", "README.md"); + int n = run_doclinks(gb, "test", NULL); + ASSERT_EQ(n, 0); + cbm_gbuf_free(gb); + PASS(); +} + +/* ── Suite ───────────────────────────────────────────────────────── */ + +SUITE(doclinks) { + /* Markdown strategies */ + RUN_TEST(doclinks_md_inline_link); + RUN_TEST(doclinks_md_backtick_path); + RUN_TEST(doclinks_md_bare_mention); + RUN_TEST(doclinks_md_non_file_link_ignored); + RUN_TEST(doclinks_md_link_with_anchor_resolves_file); + + /* Dedupe + resolution + guards */ + RUN_TEST(doclinks_dedupe_keeps_count); + RUN_TEST(doclinks_resolves_relative_to_file_dir); + RUN_TEST(doclinks_unresolvable_target_no_edge); + RUN_TEST(doclinks_null_repo_path_skips); +} diff --git a/tests/test_edge_structural.c b/tests/test_edge_structural.c index 947288124..a88fd66d3 100644 --- a/tests/test_edge_structural.c +++ b/tests/test_edge_structural.c @@ -258,6 +258,7 @@ static const char *ES_ALL_EDGE_TYPES[] = {"CALLS", "INHERITS", "INFRA_MAPS", "OVERRIDE", + "REFERENCES_FILE", "SEMANTICALLY_RELATED", "SIMILAR_TO", "TESTS_FILE", diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 586452e2b..be4f34f68 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -1015,6 +1015,7 @@ static const char *ALL_EDGE_TYPES[] = {"CALLS", "INHERITS", "INFRA_MAPS", "OVERRIDE", + "REFERENCES_FILE", "SEMANTICALLY_RELATED", "SIMILAR_TO", "TESTS_FILE", diff --git a/tests/test_main.c b/tests/test_main.c index ba6f26d45..e3ddec49d 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -750,6 +750,7 @@ extern void suite_store_pragmas(void); extern void suite_store_checkpoint(void); extern void suite_traces(void); extern void suite_configlink(void); +extern void suite_doclinks(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -1047,6 +1048,9 @@ int main(int argc, char **argv) { /* Config link */ RUN_SELECTED_SUITE(configlink); + /* Doc/shell file reference link */ + RUN_SELECTED_SUITE(doclinks); + /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan);