From ae0168bc812e700f3ab55a1e06ff8f5f51c798d9 Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Fri, 24 Jul 2026 01:46:18 +0000 Subject: [PATCH 1/3] feat(semsim): explicit category/taxon filters, replacing the prefix proxy The similarity search API selected entities by a CURIE prefix standing in for "things of some category in some species". That only works while prefix, category and taxon coincide one-to-one, and they do not: mouse models are split across the MGI and MMRRC prefixes so no group can name them, while MGI alone covers mouse genes, genotypes and variants at once. Search now takes explicit `categories` / `taxa` / `prefixes`, which AND together. GET /semsim/filters enumerates the valid combinations from the KG rather than a hand-maintained enum, and POST /search takes a `filter` object. SemsimSearchGroup keeps working: it expands to the explicit filters it always implicitly meant. Enabling changes: - DEFAULT_ASSOCIATIONS selects by predicate rather than by association category. The old gene+disease allowlist structurally hid all 589,030 GenotypeToPhenotypicFeature edges -- every MGI and MMRRC mouse model -- plus variants and cases, with no way for a caller to opt in. Because the pool is now heterogeneous, the legacy groups must state their category to keep their old meaning; verified they still return a single category and the same top hits (MECP2/Mecp2/Rett syndrome for a Rett query). - _termset_search scores at the phenotype level instead of per (entity, phenotype). A phenotype's similarity to a query term does not depend on which entity carries it, so the old query re-expanded each phenotype's ancestors once per annotated entity: across mouse genotypes, 447,110 association rows covering 11,530 distinct MP terms, ~39x redundant. Scoring all 81,151 mouse genotypes against Rett syndrome goes from 288.7s to 2.3s (~125x). Verified equivalent, not assumed: both paths return all 81,151 entities with a maximum absolute score difference of 2.0e-15. Exact because `closure` holds no duplicate (subject, object) pairs, so count(*) and count(DISTINCT a) agree. - _flat computes closure sizes for entities the baked `closure_size` table omits. It covers genes and diseases but no genotypes, and _flat inner-joined it, so hybrid_search -- the production default -- returned an empty result set for every mouse model with no error at all. Filtering on a `nodes` table lacking category/in_taxon now raises instead of matching nothing, so a filter that cannot be honored never looks like a legitimately empty result. Same reasoning for SemsimianService, which can only express a single prefix and now refuses anything else rather than silently answering with an unfiltered superset. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PTYEKvvmBKgaK3893Kxog2 --- .../src/monarch_py/api/additional_models.py | 86 +++++- backend/src/monarch_py/api/semsim.py | 58 +++- backend/src/monarch_py/service/ducksim.py | 254 ++++++++++++++---- .../src/monarch_py/service/ducksim_service.py | 16 +- .../src/monarch_py/service/semsim_service.py | 23 +- backend/tests/api/test_semsim_router.py | 71 ++++- 6 files changed, 445 insertions(+), 63 deletions(-) diff --git a/backend/src/monarch_py/api/additional_models.py b/backend/src/monarch_py/api/additional_models.py index 632efd296..fe5fd42e3 100644 --- a/backend/src/monarch_py/api/additional_models.py +++ b/backend/src/monarch_py/api/additional_models.py @@ -32,6 +32,14 @@ def __str__(self): class SemsimSearchGroup(Enum): + """Legacy search groups. Superseded by the explicit category/taxon filters below. + + Each group is a CURIE prefix standing in for "entities of some category in some species", + which works only while the two coincide one-to-one. They do not: mouse models are split + across the MGI and MMRRC prefixes, so no single group can name them, and MGI alone covers + mouse genes, genotypes and variants at once. Kept working via `expand_search_group`. + """ + HGNC = "Human Genes" MGI = "Mouse Genes" RGD = "Rat Genes" @@ -40,6 +48,65 @@ class SemsimSearchGroup(Enum): MONDO = "Human Diseases" +# Explicit (category, taxa, prefixes) each legacy group expands to. This is what preserves the +# old meaning now that the association pool spans every phenotype-bearing entity: before, a +# gene-and-disease-only pool made prefix="MGI" mean "mouse genes" by accident. It no longer does, +# so the category has to be stated. +_SEARCH_GROUP_FILTERS = { + "HGNC": (["biolink:Gene"], ["NCBITaxon:9606"], ["HGNC"]), + "MGI": (["biolink:Gene"], ["NCBITaxon:10090"], ["MGI"]), + "RGD": (["biolink:Gene"], ["NCBITaxon:10116"], ["RGD"]), + "ZFIN": (["biolink:Gene"], ["NCBITaxon:7955"], ["ZFIN"]), + "WB": (["biolink:Gene"], ["NCBITaxon:6239"], ["WB"]), + "MONDO": (["biolink:Disease"], None, ["MONDO"]), +} + + +def expand_search_group(group): + """Legacy group -> (categories, taxa, prefixes). Accepts the enum, its name ("MGI") or its + value ("Mouse Genes"). Returns (None, None, None) for an unknown group so the caller can + decide whether that is an error.""" + if group is None: + return None, None, None + name = group.name if isinstance(group, SemsimSearchGroup) else str(group) + if name not in _SEARCH_GROUP_FILTERS: + try: + name = SemsimSearchGroup(name).name + except ValueError: + return None, None, None + return _SEARCH_GROUP_FILTERS[name] + + +class SemsimSearchFilter(BaseModel): + """Explicit replacement for `group`: say what kind of thing, in what species, from where. + + All three fields are optional and AND together, so "every mouse genotype regardless of + source" is `category=[Genotype], taxon=["NCBITaxon:10090"]` — the query the prefix-based API + could not express. `taxon` takes an NCBITaxon CURIE or a species label ("Mus musculus"). + """ + + category: Optional[List[EntityCategory]] = Field( + default=None, title="Biolink categories to search within (e.g. biolink:Genotype)" + ) + taxon: Optional[List[str]] = Field( + default=None, title="NCBITaxon CURIEs or species labels (e.g. NCBITaxon:10090)" + ) + prefix: Optional[List[str]] = Field( + default=None, title="Restrict to these CURIE prefixes (e.g. MMRRC) — a source filter, not a category" + ) + + def as_kwargs(self) -> dict: + """Engine kwargs for Ducksim.search / full_search / hybrid_search.""" + # `.value`, not `str()`: EntityCategory is a plain Enum, so str() gives + # "EntityCategory.GENOTYPE" rather than the "biolink:Genotype" CURIE the KG stores. That + # would match no rows and return an empty result that looks like a real answer. + return { + "categories": [c.value for c in self.category] if self.category else None, + "taxa": self.taxon or None, + "prefixes": self.prefix or None, + } + + class SemsimDirectionality(str, Enum): BIDIRECTIONAL = "bidirectional" SUBJECT_TO_OBJECT = "subject_to_object" @@ -66,13 +133,30 @@ class SemsimMultiCompareRequest(BaseModel): class SemsimSearchRequest(BaseModel): termset: List[str] = Field(..., title="Termset to search") - group: SemsimSearchGroup = Field(..., title="Group of entities to search within (e.g. Human Genes)") + group: Optional[SemsimSearchGroup] = Field( + None, title="DEPRECATED legacy entity group; prefer the explicit `filter`" + ) + filter: Optional[SemsimSearchFilter] = Field( + None, title="Explicit category / taxon / prefix filter (supersedes `group`)" + ) metric: SemsimMetric = Field(SemsimMetric.ANCESTOR_INFORMATION_CONTENT, title="Similarity metric to use") directionality: SemsimDirectionality = Field( SemsimDirectionality.BIDIRECTIONAL, title="Directionality of the search" ) limit: Optional[int] = Field(10, title="Limit the number of results", ge=1, le=50) + def resolved_filter(self) -> dict: + """Engine kwargs, preferring the explicit filter and falling back to the legacy group. + + `group` stays valid so existing clients keep working, but an explicit `filter` wins when + both are sent rather than silently intersecting with the group's implied category — two + filters quietly ANDing is exactly the kind of surprise this API change is meant to end. + """ + if self.filter is not None: + return self.filter.as_kwargs() + categories, taxa, prefixes = expand_search_group(self.group) + return {"categories": categories, "taxa": taxa, "prefixes": prefixes} + class TextAnnotationRequest(BaseModel): content: str = Field(..., title="The text content to annotate") diff --git a/backend/src/monarch_py/api/semsim.py b/backend/src/monarch_py/api/semsim.py index 1d260538f..3707fca16 100644 --- a/backend/src/monarch_py/api/semsim.py +++ b/backend/src/monarch_py/api/semsim.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Path, Query +from fastapi import APIRouter, HTTPException, Path, Query from monarch_py.api.additional_models import ( SemsimCompareRequest, @@ -9,6 +9,7 @@ SemsimSearchGroup, SemsimMultiCompareRequest, SemsimDirectionality, + expand_search_group, ) from monarch_py.api.config import semsim_service, solr from monarch_py.api.utils.similarity_utils import parse_similarity_prefix @@ -137,7 +138,11 @@ def _search( limit: int = Query(default=10, ge=1, le=50), engine: Optional[str] = EngineParam, ): - """Search for terms in a termset + """Search for terms in a termset, within one of the legacy entity groups. + + Superseded by `POST /search`, which takes explicit category/taxon filters and can express + targets no group can name — notably "all mouse models", which spans the MGI and MMRRC + prefixes. This route expands the group to those same explicit filters. Args:
termset (str): Comma separated list of term IDs to find matches for.
@@ -149,31 +154,66 @@ def _search( List[str]: List of matching terms """ terms = [term.strip() for term in termset.split(",")] + categories, taxa, prefixes = expand_search_group(group) results = semsim_service(engine).search( - termset=terms, prefix=parse_similarity_prefix(group), metric=metric, directionality=directionality, limit=limit + termset=terms, + prefix=parse_similarity_prefix(group), + metric=metric, + directionality=directionality, + limit=limit, + categories=categories, + taxa=taxa, + prefixes=prefixes, ) return results +@router.get("/filters") +def _filters(engine: Optional[str] = EngineParam): + """The (category, taxon) combinations available to search, with entity counts. + + Explicit filters are only usable if callers can find out what to pass, so this enumerates the + valid values from the KG itself rather than from a hand-maintained enum that drifts. + """ + service = semsim_service(engine) + if not hasattr(service, "engine"): + return {"detail": "filter discovery requires the ducksim backend"} + return [ + {"category": cat, "taxon": tax, "taxon_label": lab, "count": n} + for cat, tax, lab, n in service.engine.categories() + ] + + @router.post("/search") def _post_search(request: SemsimSearchRequest, engine: Optional[str] = EngineParam): """ - Search for terms in a termset
+ Search for entities whose phenotype profile matches a termset.

- Example:
+ Every mouse model, across MGI *and* MMRRC — not expressible with the legacy `group`: 
     {
-      "termset": ["HP:0002104", "HP:0012378", "HP:0012378", "HP:0012378"],
-      "group": "Human Diseases",
-      "metric": "ancestor_information_content",
+      "termset": ["HP:0002104", "HP:0012378"],
+      "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
+      "metric": "phenodigm_score",
       "limit": 5
     }
+ Only orderable MMRRC strains:
+ 
+    {"termset": ["HP:0002104"], "filter": {"category": ["biolink:Genotype"], "prefix": ["MMRRC"]}}
+
+ Legacy form, still supported:
+ 
+    {"termset": ["HP:0002104"], "group": "Human Diseases", "limit": 5}
+
+ Call GET /semsim/filters for the category/taxon values this KG supports. """ + if request.filter is None and request.group is None: + raise HTTPException(status_code=422, detail="provide either `filter` (preferred) or `group`") return semsim_service(engine).search( termset=request.termset, - prefix=parse_similarity_prefix(request.group.value), metric=request.metric, directionality=request.directionality, limit=request.limit, + **request.resolved_filter(), ) diff --git a/backend/src/monarch_py/service/ducksim.py b/backend/src/monarch_py/service/ducksim.py index f86e3f739..c5688a670 100644 --- a/backend/src/monarch_py/service/ducksim.py +++ b/backend/src/monarch_py/service/ducksim.py @@ -53,12 +53,21 @@ def _dedupe(seq): class Ducksim: """Semantic-similarity engine over `monarch-kg.duckdb` (closure + edges), computed in DuckDB.""" - # entity->phenotype associations for search, from the KG `edges` table + # entity->phenotype associations for search, from the KG `edges` table. + # + # Selected by PREDICATE rather than by association category, so every kind of phenotype-bearing + # entity is searchable and the caller decides what to include via explicit category/taxon + # filters. The previous category allowlist (gene + disease only) structurally hid the 589,030 + # GenotypeToPhenotypicFeature edges -- every MGI and MMRRC mouse model -- along with variants + # and cases, with no way for a caller to opt in. + # + # Because the pool is now heterogeneous, an unfiltered search mixes categories. Callers that + # mean "mouse genes" must say so (category=Gene, taxon=NCBITaxon:10090); a bare prefix no + # longer implies a category. `SemsimSearchGroup` expands to exactly that, so the legacy API + # keeps its old meaning. DEFAULT_ASSOCIATIONS = ( "SELECT subject AS entity, object AS phenotype FROM src.edges " - "WHERE category IN ('biolink:GeneToPhenotypicFeatureAssociation'," - "'biolink:DiseaseToPhenotypicFeatureAssociation') " - "AND predicate = 'biolink:has_phenotype' " + "WHERE predicate = 'biolink:has_phenotype' " # keep everything that isn't explicitly negated. `negated` is a VARCHAR today ('True'/'False'/ # NULL), but try_cast-to-BOOLEAN keeps this correct across casing and a future boolean column; # NULL (and any unparseable value) coalesces to false so the row is kept, never silently dropped. @@ -152,8 +161,83 @@ def _define_associations(self, assoc_sql): self.con.execute(f"CREATE VIEW _assoc AS {assoc_sql}") self._require_baked("closure_size") self.con.execute("CREATE VIEW _esize AS SELECT entity, size AS pn FROM src.closure_size") + self._define_entity_metadata() self.has_search = True + def _define_entity_metadata(self): + """`_emeta`: (entity, category, taxon) for every searchable entity — what explicit + category/taxon filtering resolves against. + + Materialized rather than left as a view over `src.nodes`: it is small (one row per + phenotype-annotated entity, ~200K) and every filtered search probes it, so paying the join + once at startup beats re-joining a 1.58M-row table per query. + + A `nodes` table lacking `category`/`in_taxon` (minimal or older artifacts) is tolerated + rather than fatal, but the missing dimension is recorded so `entity_filter` can refuse a + filter it cannot honor. Filtering on an absent column would otherwise match nothing and + look like a legitimately empty result — the same silent-empty failure `closure_size` + already causes for genotypes in `_flat`. + """ + cols = { + r[0] + for r in self.con.execute( + "SELECT column_name FROM duckdb_columns() WHERE database_name = 'src' AND table_name = 'nodes'" + ).fetchall() + } + self.entity_metadata = {d for d in ("category", "in_taxon") if d in cols} + cat = "n.category" if "category" in cols else "CAST(NULL AS VARCHAR)" + tax = "n.in_taxon" if "in_taxon" in cols else "CAST(NULL AS VARCHAR)" + taxl = "n.in_taxon_label" if "in_taxon_label" in cols else "CAST(NULL AS VARCHAR)" + self.con.execute(f""" + CREATE OR REPLACE TABLE _emeta AS + SELECT DISTINCT n.id AS entity, {cat} AS category, + {tax} AS taxon, {taxl} AS taxon_label + FROM (SELECT DISTINCT entity FROM _assoc) a JOIN src.nodes n ON n.id = a.entity + """) + self.con.execute("CREATE INDEX IF NOT EXISTS _ix_emeta ON _emeta(entity)") + + # ---- explicit filtering --------------------------------------------- + + def entity_filter(self, *, prefixes=None, categories=None, taxa=None, entities=None) -> str: + """Build the SQL WHERE clause selecting which entities a search may return. + + Every argument is an optional list and they AND together; passing none searches everything. + This is the explicit replacement for the old single-`prefix` argument, which conflated + category and taxon into a data-source proxy and so could express "Mouse Genes" but not + "mouse models" (MGI and MMRRC genotypes live under two different prefixes) nor "any + genotype in any species". + + `taxa` accepts either a CURIE ('NCBITaxon:10090') or a label ('Mus musculus'); labels are + matched case-insensitively so callers need not know which form the KG stores. + """ + for requested, dimension in ((categories, "category"), (taxa, "in_taxon")): + if requested and dimension not in getattr(self, "entity_metadata", set()): + raise RuntimeError( + f"cannot filter by {dimension}: the attached KG's `nodes` table has no " + f"'{dimension}' column, so this filter would match nothing" + ) + clauses = [] + if entities: + clauses.append(f"entity IN (SELECT unnest([{_quote_list(entities)}]::VARCHAR[]))") + if prefixes: + clauses.append(f"split_part(entity, ':', 1) IN ({_quote_list(prefixes)})") + meta = [] + if categories: + meta.append(f"category IN ({_quote_list(categories)})") + if taxa: + meta.append(f"(taxon IN ({_quote_list(taxa)}) OR lower(taxon_label) IN ({_quote_list(t.lower() for t in taxa)}))") + if meta: + clauses.append(f"entity IN (SELECT entity FROM _emeta WHERE {' AND '.join(meta)})") + return f"WHERE {' AND '.join(clauses)}" if clauses else "" + + def categories(self) -> list: + """Distinct (category, taxon, taxon_label, count) available to search — lets a caller (or + an API docs page) discover valid filter values instead of guessing them.""" + return self._read( + "SELECT category, taxon, taxon_label, count(*) AS n FROM _emeta " + "GROUP BY 1, 2, 3 ORDER BY n DESC" + ).fetchall() + # ---- labels --------------------------------------------------------- def labels(self, ids) -> dict: @@ -339,6 +423,7 @@ def _termset_search(self, query_terms, metric, entity_filter, limit, direction=" }.get(direction) if score_combiner is None: raise ValueError(f"unknown direction {direction!r}") + self._require_phenotype_tables() Q = _quote_list(set(query_terms)) score_expr = spec.sql_rank lim = "" if limit is None else f"LIMIT {int(limit)}" @@ -348,77 +433,138 @@ def _termset_search(self, query_terms, metric, entity_filter, limit, direction=" FROM qterms qt JOIN _clo c ON c.s = qt.q JOIN _ic ic ON ic.term = c.o), qsize AS (SELECT q, count(*) AS sz FROM q_anc GROUP BY q), nq AS (SELECT count(*) AS n FROM qterms), - -- DISTINCT: an entity may be annotated to the same phenotype via multiple association - -- rows (different evidence/sources). Without dedup, p_anc repeats that phenotype's - -- ancestors, inflating the count(*) intersection below past the union size -> a zero or - -- negative jaccard denominator (inf score; sqrt-of-negative for phenodigm). psize already - -- uses count(DISTINCT), so the two must agree. - ent_ph AS (SELECT DISTINCT entity AS e, phenotype AS p FROM _assoc {entity_filter}), - np AS (SELECT e, count(DISTINCT p) AS n FROM ent_ph GROUP BY e), - p_anc AS (SELECT ep.e, ep.p, c.o AS a FROM ent_ph ep JOIN _clo c ON c.s = ep.p), - psize AS (SELECT e, p, count(DISTINCT a) AS sz FROM p_anc GROUP BY e, p), - pair AS ( - SELECT pa.e, pa.p, qa.q, count(*) AS inter, max(qa.ic) AS resnik - FROM p_anc pa JOIN q_anc qa ON qa.a = pa.a GROUP BY pa.e, pa.p, qa.q - ), - scored AS ( - SELECT pr.e, pr.p, pr.q, pr.resnik, - pr.inter::DOUBLE / (ps.sz + qs.sz - pr.inter) AS jaccard - FROM pair pr JOIN psize ps ON ps.e = pr.e AND ps.p = pr.p - JOIN qsize qs ON qs.q = pr.q - ), - ranked AS (SELECT e, p, q, {score_expr} AS score FROM scored), - dir1 AS (SELECT bm.e, sum(bm.best) / np.n AS avg1 - FROM (SELECT e, p, max(score) AS best FROM ranked GROUP BY e, p) bm - JOIN np ON np.e = bm.e GROUP BY bm.e, np.n), - dir2 AS (SELECT bm.e, sum(bm.best) / (SELECT n FROM nq) AS avg2 - FROM (SELECT e, q, max(score) AS best FROM ranked GROUP BY e, q) bm - GROUP BY bm.e) + -- Scoring happens at the PHENOTYPE level, not per (entity, phenotype). A phenotype's + -- similarity to a query term does not depend on which entity carries it, so expanding + -- ancestors per association row recomputes the same ancestor set once per annotated + -- entity. Across mouse genotypes that is 447,110 rows covering 11,530 distinct MP + -- terms -- ~39x redundant, and the dominant cost of the whole query. + pair AS (SELECT pa.p, qa.q, count(*) AS inter, max(qa.ic) AS resnik + FROM _ph_anc pa JOIN q_anc qa ON qa.a = pa.a GROUP BY pa.p, qa.q), + ranked AS (SELECT s.p, s.q, {score_expr} AS score FROM + (SELECT pr.p, pr.q, pr.resnik, + pr.inter::DOUBLE / (ps.sz + qs.sz - pr.inter) AS jaccard + FROM pair pr JOIN _psize ps ON ps.p = pr.p + JOIN qsize qs ON qs.q = pr.q) s), + -- entities enter only here, by joining the per-phenotype scores onto their annotations + ent_ph AS (SELECT entity AS e, phenotype AS p FROM _ent_ph {entity_filter}), + dir1 AS (SELECT ep.e, sum(bp.best) / any_value(np.n) AS avg1 + FROM ent_ph ep + JOIN (SELECT p, max(score) AS best FROM ranked GROUP BY p) bp ON bp.p = ep.p + JOIN _np np ON np.e = ep.e GROUP BY ep.e), + dir2 AS (SELECT e, sum(best) / (SELECT n FROM nq) AS avg2 FROM + (SELECT ep.e, r.q, max(r.score) AS best + FROM ent_ph ep JOIN ranked r ON r.p = ep.p GROUP BY ep.e, r.q) GROUP BY e) SELECT coalesce(d1.e, d2.e) AS entity, {score_combiner} AS score FROM dir1 d1 FULL OUTER JOIN dir2 d2 ON d1.e = d2.e ORDER BY score DESC, entity {lim} """ return self._read(sql).fetchall() + def _require_phenotype_tables(self): + """Materialize the phenotype-level tables `_termset_search` scores against. Built once on + first search (a few seconds), reused by every later query. + + `_ent_ph` is deduped here because an entity may be annotated to the same phenotype through + several association rows (different evidence or sources). Left undeduped, an entity's + matched phenotype would be counted once per row and `dir1` would over-sum. `_np` is derived + from the same deduped table so the two necessarily agree. + """ + if getattr(self, "_pheno_ready", False): + return + self.con.execute("CREATE OR REPLACE TABLE _ent_ph AS SELECT DISTINCT entity, phenotype FROM _assoc") + self.con.execute("CREATE OR REPLACE TABLE _np AS SELECT entity AS e, count(*) AS n FROM _ent_ph GROUP BY 1") + self.con.execute(""" + CREATE OR REPLACE TABLE _ph_anc AS + SELECT DISTINCT ep.phenotype AS p, c.o AS a + FROM (SELECT DISTINCT phenotype FROM _ent_ph) ep JOIN _clo c ON c.s = ep.phenotype + """) + self.con.execute("CREATE OR REPLACE TABLE _psize AS SELECT p, count(*) AS sz FROM _ph_anc GROUP BY p") + self.con.execute("CREATE INDEX IF NOT EXISTS _ix_ph_anc_a ON _ph_anc(a)") + self.con.execute("CREATE INDEX IF NOT EXISTS _ix_ent_ph_p ON _ent_ph(phenotype)") + self._pheno_ready = True + def full_search( - self, query_terms, *, limit=10, metric="ancestor_information_content", prefix=None, direction="bidirectional" + self, + query_terms, + *, + limit=10, + metric="ancestor_information_content", + prefix=None, + direction="bidirectional", + categories=None, + taxa=None, + prefixes=None, ): - """Score every entity (optionally restricted to CURIE `prefix`) by the termset - best-match-average — one DuckDB query. Matches semsimian's Full mode; more accurate than - Hybrid (no Jaccard prefilter dropping true-top entities).""" - ef = f"WHERE split_part(entity, ':', 1) = {_quote_list([prefix])}" if prefix else "" + """Score every entity passing the filters by the termset best-match-average — one DuckDB + query. Matches semsimian's Full mode; more accurate than Hybrid (no Jaccard prefilter + dropping true-top entities). + + `categories`/`taxa`/`prefixes` are the explicit filters and AND together. `prefix` is the + legacy single-value form, kept working for existing callers. + """ + ef = self.entity_filter( + prefixes=prefixes or ([prefix] if prefix else None), categories=categories, taxa=taxa + ) return self._termset_search(query_terms, metric, ef, limit, direction) def hybrid_search( - self, query_terms, *, limit=10, metric="ancestor_information_content", prefix=None, direction="bidirectional" + self, + query_terms, + *, + limit=10, + metric="ancestor_information_content", + prefix=None, + direction="bidirectional", + categories=None, + taxa=None, + prefixes=None, ): """Hybrid search — semsimian's production mode: cheap Jaccard prefilter then termset rerank, as one query over the candidate set. (The Jaccard prefilter is direction-agnostic; the rerank honors `direction`.)""" - flat = self._flat(query_terms, prefix) + ef = self.entity_filter( + prefixes=prefixes or ([prefix] if prefix else None), categories=categories, taxa=taxa + ) + flat = self._flat(query_terms, ef) if not flat: return [] scores = sorted({j for _, j in flat}, reverse=True) k = max(math.ceil((limit / 1000.0) * len(scores)), limit) cutoff = scores[k] if k < len(scores) else scores[-1] candidates = [e for e, j in flat if j >= cutoff] - ef = f"WHERE entity IN (SELECT unnest([{_quote_list(candidates)}]::VARCHAR[]))" - return self._termset_search(query_terms, metric, ef, limit, direction) + return self._termset_search( + query_terms, metric, self.entity_filter(entities=candidates), limit, direction + ) + + def _flat(self, query_terms, entity_filter=""): + """Cheap set-Jaccard ranking of the filtered entities vs the query — Hybrid's candidate gen. - def _flat(self, query_terms, prefix): - """Cheap set-Jaccard ranking of all (prefix) entities vs the query — Hybrid's candidate gen.""" + `_esize` is the baked `closure_size` table, which covers only entities koza precomputed -- + genes, diseases, and no genotypes at all. A plain inner join therefore drops every mouse + model silently, returning an empty candidate set with no error. Sizes are computed here for + anything the baked table is missing so filtering, not precompute coverage, decides what is + searchable. + """ Q = _quote_list(set(query_terms)) - pfilter = f"AND split_part(i.entity, ':', 1) = {_quote_list([prefix])}" if prefix else "" + self._require_phenotype_tables() return self._read(f""" WITH qt(t) AS (SELECT unnest([{Q}]::VARCHAR[])), q_anc AS (SELECT DISTINCT c.o AS a FROM _clo c JOIN qt ON c.s = qt.t), qn AS (SELECT count(*) AS n FROM q_anc), + ent AS (SELECT entity, phenotype FROM _ent_ph {entity_filter}), inter AS (SELECT a.entity, count(DISTINCT c.o) AS inter - FROM _assoc a JOIN _clo c ON c.s = a.phenotype JOIN q_anc q ON q.a = c.o - GROUP BY a.entity) - SELECT i.entity, i.inter::DOUBLE / ((SELECT n FROM qn) + e.pn - i.inter) AS jaccard - FROM inter i JOIN _esize e ON e.entity = i.entity - WHERE true {pfilter} ORDER BY jaccard DESC, i.entity + FROM ent a JOIN _clo c ON c.s = a.phenotype JOIN q_anc q ON q.a = c.o + GROUP BY a.entity), + -- baked sizes where present, computed for the entities koza did not precompute + sz AS (SELECT i.entity, coalesce(e.pn, f.pn) AS pn + FROM inter i + LEFT JOIN _esize e ON e.entity = i.entity + LEFT JOIN (SELECT ep.entity, count(DISTINCT pa.a) AS pn + FROM ent ep JOIN _ph_anc pa ON pa.p = ep.phenotype + GROUP BY ep.entity) f ON f.entity = i.entity) + SELECT i.entity, i.inter::DOUBLE / ((SELECT n FROM qn) + sz.pn - i.inter) AS jaccard + FROM inter i JOIN sz ON sz.entity = i.entity + ORDER BY jaccard DESC, i.entity """).fetchall() # ---- search with full per-result detail ----------------------------- @@ -432,6 +578,9 @@ def search( prefix=None, direction="bidirectional", mode="hybrid", + categories=None, + taxa=None, + prefixes=None, ): """All-DuckDB search: rank entities (Hybrid by default; Full when `mode="full"`), then enrich the whole page with full termset detail in a constant number of queries — independent of @@ -443,7 +592,16 @@ def search( if spec is None: raise ValueError(f"unknown metric {metric!r}") ranker = self.full_search if mode == "full" else self.hybrid_search - ranked = ranker(query_terms, limit=limit, metric=metric, prefix=prefix, direction=direction) + ranked = ranker( + query_terms, + limit=limit, + metric=metric, + prefix=prefix, + direction=direction, + categories=categories, + taxa=taxa, + prefixes=prefixes, + ) if not ranked: return [] entity_ids = [e for e, _ in ranked] diff --git a/backend/src/monarch_py/service/ducksim_service.py b/backend/src/monarch_py/service/ducksim_service.py index 0750f498d..7e0e1efef 100644 --- a/backend/src/monarch_py/service/ducksim_service.py +++ b/backend/src/monarch_py/service/ducksim_service.py @@ -59,16 +59,28 @@ def multi_compare(self, request: SemsimMultiCompareRequest) -> List[SemsimSearch def search( self, termset: List[str], - prefix: str, + prefix: str = None, metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT, directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL, limit: int = 10, + categories: List[str] = None, + taxa: List[str] = None, + prefixes: List[str] = None, ) -> List[SemsimSearchResult]: # Hybrid mode matches the semsimian server; full_search would be more accurate (see engine). # The engine ranks and enriches the whole page in a constant number of DuckDB queries (no # per-result round-trips); the loop below is pure in-memory model shaping. direction = directionality.value if hasattr(directionality, "value") else str(directionality) - page = self.engine.search(termset, limit=limit, metric=str(metric), prefix=prefix, direction=direction) + page = self.engine.search( + termset, + limit=limit, + metric=str(metric), + prefix=prefix, + direction=direction, + categories=categories, + taxa=taxa, + prefixes=prefixes, + ) if not page: return [] # all-DuckDB hydration of result entities from the KG `nodes` table — no external entity store diff --git a/backend/src/monarch_py/service/semsim_service.py b/backend/src/monarch_py/service/semsim_service.py index db471688b..95baf02be 100644 --- a/backend/src/monarch_py/service/semsim_service.py +++ b/backend/src/monarch_py/service/semsim_service.py @@ -73,11 +73,32 @@ def multi_compare(self, request: SemsimMultiCompareRequest) -> List[SemsimSearch def search( self, termset: List[str], - prefix: str, + prefix: str = None, metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT, directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL, limit: int = 10, + categories: List[str] = None, + taxa: List[str] = None, + prefixes: List[str] = None, ) -> List[SemsimSearchResult]: + """The semsimian HTTP server indexes entities by a single CURIE prefix, so it can honor + only the prefix form of a filter. + + When `prefix` is set the call is a legacy group search and any accompanying + category/taxon arguments merely restate what the prefix already encodes, so they are + redundant and ignored. When it is not, the caller has asked for something explicit that + this backend cannot express — anything but a single prefix is refused rather than + answered with a broader result set that looks filtered but is not. + """ + if prefix is None: + if categories or taxa or (prefixes and len(prefixes) > 1): + raise NotImplementedError( + "the semsimian backend cannot filter by category or taxon, or across multiple " + "prefixes; use the ducksim backend (SEMSIM_BACKEND=ducksim or ?engine=ducksim)" + ) + if not prefixes: + raise ValueError("semsimian search requires a prefix") + prefix = prefixes[0] host = f"http://{self.semsim_server_host}:{self.semsim_server_port}" path = f"search/{','.join(termset)}/{prefix}:/{metric}?limit={limit}&directionality={directionality.value} " url = f"{host}/{path}" diff --git a/backend/tests/api/test_semsim_router.py b/backend/tests/api/test_semsim_router.py index b4e059c93..4a02c7104 100644 --- a/backend/tests/api/test_semsim_router.py +++ b/backend/tests/api/test_semsim_router.py @@ -86,8 +86,18 @@ def test_get_search(mock_search, termset: str, metric: SemsimMetric): response = client.get(f"/search/{termset}/{group.value}?metric={metric}&limit={limit}") directionality = SemsimDirectionality.BIDIRECTIONAL assert response.status_code == status.HTTP_200_OK + # The legacy group still sends its prefix, and additionally the explicit filters it expands + # to. The category is what keeps "Mouse Genes" meaning genes now that the association pool + # also holds genotypes and variants under the MGI prefix. mock_search.assert_called_once_with( - termset=["HP:123", "HP:456"], prefix=group.name, metric=metric, directionality=directionality, limit=limit + termset=["HP:123", "HP:456"], + prefix=group.name, + metric=metric, + directionality=directionality, + limit=limit, + categories=["biolink:Gene"], + taxa=["NCBITaxon:9606"], + prefixes=["HGNC"], ) @@ -115,11 +125,68 @@ def test_post_search(mock_search): assert response.status_code == status.HTTP_200_OK mock_search.assert_called_once_with( termset=["HP:123", "HP:456"], - prefix=group.name, metric=metric, directionality=directionality, limit=limit, + categories=["biolink:Gene"], + taxa=["NCBITaxon:9606"], + prefixes=["HGNC"], + ) + + +@patch("monarch_py.service.semsim_service.SemsimianService.search") +def test_post_search_explicit_filter(mock_search): + """An explicit filter reaches the backend as category/taxon, with no prefix — the form no + legacy group can produce, since mouse models span the MGI and MMRRC prefixes.""" + mock_search.return_value = MagicMock() + + response = client.post( + "/search/", + json={ + "termset": ["HP:123"], + "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]}, + "limit": 5, + }, + ) + + assert response.status_code == status.HTTP_200_OK + kwargs = mock_search.call_args.kwargs + assert kwargs["categories"] == ["biolink:Genotype"] + assert kwargs["taxa"] == ["NCBITaxon:10090"] + assert kwargs["prefixes"] is None + assert "prefix" not in kwargs + + +def test_post_search_explicit_filter_beats_group(): + """When both are sent the explicit filter wins outright rather than intersecting with the + group's implied category, so the result set is never narrower than what was asked for.""" + from monarch_py.api.additional_models import SemsimSearchRequest + + request = SemsimSearchRequest( + termset=["HP:123"], + group=SemsimSearchGroup.HGNC, + filter={"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]}, ) + assert request.resolved_filter() == { + "categories": ["biolink:Genotype"], + "taxa": ["NCBITaxon:10090"], + "prefixes": None, + } + + +def test_post_search_requires_a_filter(): + """Neither `filter` nor `group` is a client bug, not a request to search all 200k entities. + + Mounted on a real app rather than the bare router `client` used above, because FastAPI's + HTTPException-to-response handling lives at the app level: against a bare router the + exception would propagate and the status code never be exercised. + """ + from fastapi import FastAPI + + app = FastAPI() + app.include_router(router) + response = TestClient(app).post("/search/", json={"termset": ["HP:123"], "limit": 5}) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY @patch("monarch_py.service.semsim_service.SemsimianService.multi_compare") From 52d5fba0b6b186504d10b77dd08bc1d0dd9670d4 Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Fri, 24 Jul 2026 03:41:27 +0000 Subject: [PATCH 2/3] docs(ducksim): closure_size is baked upstream now; keep the fallback as a shim koza now bakes a closure size for every entity with a has_phenotype edge, so _flat's computed fallback is no longer what makes genotype search work. Keep it for artifacts built before that fix, where degrading to a slower correct answer still beats silently returning nothing. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PTYEKvvmBKgaK3893Kxog2 --- backend/src/monarch_py/service/ducksim.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/src/monarch_py/service/ducksim.py b/backend/src/monarch_py/service/ducksim.py index c5688a670..5d82370d9 100644 --- a/backend/src/monarch_py/service/ducksim.py +++ b/backend/src/monarch_py/service/ducksim.py @@ -539,11 +539,15 @@ def hybrid_search( def _flat(self, query_terms, entity_filter=""): """Cheap set-Jaccard ranking of the filtered entities vs the query — Hybrid's candidate gen. - `_esize` is the baked `closure_size` table, which covers only entities koza precomputed -- - genes, diseases, and no genotypes at all. A plain inner join therefore drops every mouse - model silently, returning an empty candidate set with no error. Sizes are computed here for - anything the baked table is missing so filtering, not precompute coverage, decides what is - searchable. + `_esize` is koza's baked `closure_size`. It once covered only genes and diseases, so a + plain inner join dropped every genotype silently and returned an empty candidate set with + no error. koza now bakes a size for every entity carrying a has_phenotype edge + (monarch-initiative/koza `fix(information-content)`), which covers all 213,273 of them. + + The LEFT JOIN and computed fallback stay as a compatibility shim: artifacts built before + that fix are still in circulation, and degrading to a slower correct answer is better than + silently returning none. Against a current artifact the fallback matches nothing and costs + nothing. """ Q = _quote_list(set(query_terms)) self._require_phenotype_tables() From 54a49fc1b29a729723d6b74d33ba0d35cd636654 Mon Sep 17 00:00:00 2001 From: Kevin Schaper Date: Fri, 24 Jul 2026 05:26:27 +0000 Subject: [PATCH 3/3] feat(semsim): search-by-profile, driven by an entity's own phenotypes Case entities were already reachable as search TARGETS once the association pool was selected by predicate rather than category -- filter to biolink:Case and a termset search ranks patients. The missing half is using a Case as a search SOURCE: a phenopacket has ~20 HPO terms, and no one is going to retype them to ask "which mouse models this patient". search_by_profile takes any phenotype-annotated entity id, reads its profile from the KG, and searches with an explicit target filter. Because the profile comes from the same _assoc pool for every entity type, one call expresses: Case -> Disease likely diagnosis for a patient Case -> Genotype mouse models for a patient (needs mouse taxon filter) Genotype -> Case patients resembling a mouse model (the MMRRC<->phenopacket link) Engine gets a thin `profile(entity)` accessor; the service composes it with the existing search and returns (query_profile, results) so a caller can weight each result by how many terms drove it. The endpoint is guarded to the ducksim backend -- semsimian has no KG to read a profile from -- returning 501 rather than a confusing failure, and 404 when the entity has no phenotypes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PTYEKvvmBKgaK3893Kxog2 --- .../src/monarch_py/api/additional_models.py | 17 ++++++++ backend/src/monarch_py/api/semsim.py | 41 +++++++++++++++++++ backend/src/monarch_py/service/ducksim.py | 10 +++++ .../src/monarch_py/service/ducksim_service.py | 36 ++++++++++++++++ backend/tests/unit/test_ducksim.py | 20 +++++++++ 5 files changed, 124 insertions(+) diff --git a/backend/src/monarch_py/api/additional_models.py b/backend/src/monarch_py/api/additional_models.py index fe5fd42e3..d2957de32 100644 --- a/backend/src/monarch_py/api/additional_models.py +++ b/backend/src/monarch_py/api/additional_models.py @@ -158,6 +158,23 @@ def resolved_filter(self) -> dict: return {"categories": categories, "taxa": taxa, "prefixes": prefixes} +class SemsimProfileSearchRequest(BaseModel): + """Search driven by an entity's own phenotype profile instead of a hand-typed termset. + + `entity` is any phenotype-annotated node — a phenopacket Case, a MONDO disease, an MGI/MMRRC + mouse model — and `filter` says what to rank against it, so patient->model, model->patient and + disease->model are the same request with different filters. + """ + + entity: str = Field(..., title="Entity whose phenotype profile is the query (e.g. a Case id)") + filter: SemsimSearchFilter = Field(..., title="Explicit category / taxon / prefix filter for the targets") + metric: SemsimMetric = Field(SemsimMetric.ANCESTOR_INFORMATION_CONTENT, title="Similarity metric to use") + directionality: SemsimDirectionality = Field( + SemsimDirectionality.BIDIRECTIONAL, title="Directionality of the search" + ) + limit: Optional[int] = Field(10, title="Limit the number of results", ge=1, le=50) + + class TextAnnotationRequest(BaseModel): content: str = Field(..., title="The text content to annotate") prefix: Optional[List[str]] = Field( diff --git a/backend/src/monarch_py/api/semsim.py b/backend/src/monarch_py/api/semsim.py index 3707fca16..4963341f4 100644 --- a/backend/src/monarch_py/api/semsim.py +++ b/backend/src/monarch_py/api/semsim.py @@ -6,6 +6,7 @@ SemsimCompareRequest, SemsimMetric, SemsimSearchRequest, + SemsimProfileSearchRequest, SemsimSearchGroup, SemsimMultiCompareRequest, SemsimDirectionality, @@ -217,3 +218,43 @@ def _post_search(request: SemsimSearchRequest, engine: Optional[str] = EnginePar limit=request.limit, **request.resolved_filter(), ) + + +@router.post("/search-by-profile") +def _post_search_by_profile(request: SemsimProfileSearchRequest, engine: Optional[str] = EngineParam): + """ + Search using an entity's OWN phenotype profile as the query.
+
+ Mouse models for a patient (phenopacket Case -> mouse genotypes):
+ 
+    {
+      "entity": "phenopacket.store:PMID_38991538_Individual_1",
+      "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
+      "metric": "phenodigm_score"
+    }
+
+ Patients matching a mouse model (mouse genotype -> Cases):
+ 
+    {"entity": "MMRRC:000415-UCD", "filter": {"category": ["biolink:Case"]}}
+
+ The likely diagnosis for a patient (Case -> diseases):
+ 
+    {"entity": "phenopacket.store:PMID_38991538_Individual_1", "filter": {"category": ["biolink:Disease"]}}
+
+ Needs the ducksim backend (the entity's profile is read from the KG). + """ + service = semsim_service(engine) + if not hasattr(service, "search_by_profile"): + raise HTTPException( + status_code=501, detail="search-by-profile requires the ducksim backend (?engine=ducksim)" + ) + profile, results = service.search_by_profile( + entity_id=request.entity, + metric=request.metric, + directionality=request.directionality, + limit=request.limit, + **request.filter.as_kwargs(), + ) + if not profile: + raise HTTPException(status_code=404, detail=f"no phenotype profile found for entity {request.entity!r}") + return {"entity": request.entity, "query_profile": profile, "results": results} diff --git a/backend/src/monarch_py/service/ducksim.py b/backend/src/monarch_py/service/ducksim.py index 5d82370d9..2bc94b587 100644 --- a/backend/src/monarch_py/service/ducksim.py +++ b/backend/src/monarch_py/service/ducksim.py @@ -250,6 +250,16 @@ def labels(self, ids) -> dict: ).fetchall() return {i: name for i, name in rows} + def profile(self, entity_id) -> list: + """The phenotype termset an entity is annotated with — its similarity query profile. + + Reads `_assoc`, so it is uniform across every entity type the association pool covers: + a disease's HPO terms, a mouse genotype's MP terms, a phenopacket Case's HPO terms. That + uniformity is the point — it lets one search primitive be driven by "this case", "this + disease" or "this mouse" interchangeably, which is what patient->model and model->patient + matching both need.""" + return self.entity_phenotypes_batch([entity_id]).get(entity_id, []) + def entity_phenotypes_batch(self, entity_ids) -> dict: """{entity -> [phenotype, ...]} for many entities in one query — enriches a whole search page without a per-entity round-trip.""" diff --git a/backend/src/monarch_py/service/ducksim_service.py b/backend/src/monarch_py/service/ducksim_service.py index 7e0e1efef..333dd3e79 100644 --- a/backend/src/monarch_py/service/ducksim_service.py +++ b/backend/src/monarch_py/service/ducksim_service.py @@ -99,6 +99,42 @@ def search( for entity_id, score, comparison in page ] + def search_by_profile( + self, + entity_id: str, + metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT, + directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL, + limit: int = 10, + categories: List[str] = None, + taxa: List[str] = None, + prefixes: List[str] = None, + ): + """Search using an entity's OWN phenotype profile as the query termset. + + The half of Case support that a termset search alone does not give you: a phenopacket Case + is a query source, not something you would retype 20 HPO ids for. Pull `entity_id`'s + profile from the KG, then search — so "diseases like this patient" (filter to Disease), + "mouse models for this patient" (Genotype + mouse taxon) and "patients like this mouse" + (Case) are one call each, differing only in the target filter. + + Returns (query_profile, results). The profile is returned because a caller ranking many + entities wants to know how many terms drove each search — a 2-term case and a 40-term case + are not equally trustworthy, and hiding the count hides that. + """ + profile = self.engine.profile(entity_id) + if not profile: + return [], [] + results = self.search( + termset=profile, + metric=metric, + directionality=directionality, + limit=limit, + categories=categories, + taxa=taxa, + prefixes=prefixes, + ) + return profile, results + # ---- model shaping -------------------------------------------------- def _hydrate(self, row: dict, entity_id: str) -> Entity: diff --git a/backend/tests/unit/test_ducksim.py b/backend/tests/unit/test_ducksim.py index d73d3f45e..a9edf323b 100644 --- a/backend/tests/unit/test_ducksim.py +++ b/backend/tests/unit/test_ducksim.py @@ -161,6 +161,26 @@ def test_negated_associations_excluded(engine): assert engine.entity_phenotypes_batch(["E:1"]) == {"E:1": ["A1"]} +def test_profile_reads_entity_termset(engine): + """`profile` returns an entity's own phenotype termset — the query source for search-by-profile. + Uniform across entity types because it reads `_assoc`; here E:3 carries two phenotypes.""" + assert engine.profile("E:3") == ["A1", "B1"] + assert engine.profile("E:1") == ["A1"] + assert engine.profile("E:4") == [] # only edge negated + assert engine.profile("NOPE:0") == [] # unknown entity + + +def test_search_by_profile_uses_entity_as_query(engine): + """search_by_profile pulls the entity's profile, then searches — E:3 (phenotypes {A1,B1}) + should retrieve itself first, and returns the driving profile so a caller can weight by it.""" + svc = DucksimService(engine=engine) + profile, results = svc.search_by_profile("E:3", limit=3, categories=None, prefixes=["E"]) + assert profile == ["A1", "B1"] + assert results[0].subject.id == "E:3" # an entity is most similar to itself + # an entity with no profile yields no query and no results, rather than searching on nothing + assert svc.search_by_profile("E:4", prefixes=["E"]) == ([], []) + + def test_search_hydrates_from_duckdb_without_entity_store(engine): """All-DuckDB search: the service builds full SemsimSearchResults — including the result entity, hydrated from the KG `nodes` table — with no external entity store (entity_implementation=None)."""