diff --git a/backend/src/monarch_py/api/additional_models.py b/backend/src/monarch_py/api/additional_models.py
index 632efd296..d2957de32 100644
--- a/backend/src/monarch_py/api/additional_models.py
+++ b/backend/src/monarch_py/api/additional_models.py
@@ -32,6 +32,14 @@ def __str__(self):
class SemsimSearchGroup(Enum):
+ """Legacy search groups. Superseded by the explicit category/taxon filters below.
+
+ Each group is a CURIE prefix standing in for "entities of some category in some species",
+ which works only while the two coincide one-to-one. They do not: mouse models are split
+ across the MGI and MMRRC prefixes, so no single group can name them, and MGI alone covers
+ mouse genes, genotypes and variants at once. Kept working via `expand_search_group`.
+ """
+
HGNC = "Human Genes"
MGI = "Mouse Genes"
RGD = "Rat Genes"
@@ -40,6 +48,65 @@ class SemsimSearchGroup(Enum):
MONDO = "Human Diseases"
+# Explicit (category, taxa, prefixes) each legacy group expands to. This is what preserves the
+# old meaning now that the association pool spans every phenotype-bearing entity: before, a
+# gene-and-disease-only pool made prefix="MGI" mean "mouse genes" by accident. It no longer does,
+# so the category has to be stated.
+_SEARCH_GROUP_FILTERS = {
+ "HGNC": (["biolink:Gene"], ["NCBITaxon:9606"], ["HGNC"]),
+ "MGI": (["biolink:Gene"], ["NCBITaxon:10090"], ["MGI"]),
+ "RGD": (["biolink:Gene"], ["NCBITaxon:10116"], ["RGD"]),
+ "ZFIN": (["biolink:Gene"], ["NCBITaxon:7955"], ["ZFIN"]),
+ "WB": (["biolink:Gene"], ["NCBITaxon:6239"], ["WB"]),
+ "MONDO": (["biolink:Disease"], None, ["MONDO"]),
+}
+
+
+def expand_search_group(group):
+ """Legacy group -> (categories, taxa, prefixes). Accepts the enum, its name ("MGI") or its
+ value ("Mouse Genes"). Returns (None, None, None) for an unknown group so the caller can
+ decide whether that is an error."""
+ if group is None:
+ return None, None, None
+ name = group.name if isinstance(group, SemsimSearchGroup) else str(group)
+ if name not in _SEARCH_GROUP_FILTERS:
+ try:
+ name = SemsimSearchGroup(name).name
+ except ValueError:
+ return None, None, None
+ return _SEARCH_GROUP_FILTERS[name]
+
+
+class SemsimSearchFilter(BaseModel):
+ """Explicit replacement for `group`: say what kind of thing, in what species, from where.
+
+ All three fields are optional and AND together, so "every mouse genotype regardless of
+ source" is `category=[Genotype], taxon=["NCBITaxon:10090"]` — the query the prefix-based API
+ could not express. `taxon` takes an NCBITaxon CURIE or a species label ("Mus musculus").
+ """
+
+ category: Optional[List[EntityCategory]] = Field(
+ default=None, title="Biolink categories to search within (e.g. biolink:Genotype)"
+ )
+ taxon: Optional[List[str]] = Field(
+ default=None, title="NCBITaxon CURIEs or species labels (e.g. NCBITaxon:10090)"
+ )
+ prefix: Optional[List[str]] = Field(
+ default=None, title="Restrict to these CURIE prefixes (e.g. MMRRC) — a source filter, not a category"
+ )
+
+ def as_kwargs(self) -> dict:
+ """Engine kwargs for Ducksim.search / full_search / hybrid_search."""
+ # `.value`, not `str()`: EntityCategory is a plain Enum, so str() gives
+ # "EntityCategory.GENOTYPE" rather than the "biolink:Genotype" CURIE the KG stores. That
+ # would match no rows and return an empty result that looks like a real answer.
+ return {
+ "categories": [c.value for c in self.category] if self.category else None,
+ "taxa": self.taxon or None,
+ "prefixes": self.prefix or None,
+ }
+
+
class SemsimDirectionality(str, Enum):
BIDIRECTIONAL = "bidirectional"
SUBJECT_TO_OBJECT = "subject_to_object"
@@ -66,7 +133,41 @@ class SemsimMultiCompareRequest(BaseModel):
class SemsimSearchRequest(BaseModel):
termset: List[str] = Field(..., title="Termset to search")
- group: SemsimSearchGroup = Field(..., title="Group of entities to search within (e.g. Human Genes)")
+ group: Optional[SemsimSearchGroup] = Field(
+ None, title="DEPRECATED legacy entity group; prefer the explicit `filter`"
+ )
+ filter: Optional[SemsimSearchFilter] = Field(
+ None, title="Explicit category / taxon / prefix filter (supersedes `group`)"
+ )
+ metric: SemsimMetric = Field(SemsimMetric.ANCESTOR_INFORMATION_CONTENT, title="Similarity metric to use")
+ directionality: SemsimDirectionality = Field(
+ SemsimDirectionality.BIDIRECTIONAL, title="Directionality of the search"
+ )
+ limit: Optional[int] = Field(10, title="Limit the number of results", ge=1, le=50)
+
+ def resolved_filter(self) -> dict:
+ """Engine kwargs, preferring the explicit filter and falling back to the legacy group.
+
+ `group` stays valid so existing clients keep working, but an explicit `filter` wins when
+ both are sent rather than silently intersecting with the group's implied category — two
+ filters quietly ANDing is exactly the kind of surprise this API change is meant to end.
+ """
+ if self.filter is not None:
+ return self.filter.as_kwargs()
+ categories, taxa, prefixes = expand_search_group(self.group)
+ return {"categories": categories, "taxa": taxa, "prefixes": prefixes}
+
+
+class SemsimProfileSearchRequest(BaseModel):
+ """Search driven by an entity's own phenotype profile instead of a hand-typed termset.
+
+ `entity` is any phenotype-annotated node — a phenopacket Case, a MONDO disease, an MGI/MMRRC
+ mouse model — and `filter` says what to rank against it, so patient->model, model->patient and
+ disease->model are the same request with different filters.
+ """
+
+ entity: str = Field(..., title="Entity whose phenotype profile is the query (e.g. a Case id)")
+ filter: SemsimSearchFilter = Field(..., title="Explicit category / taxon / prefix filter for the targets")
metric: SemsimMetric = Field(SemsimMetric.ANCESTOR_INFORMATION_CONTENT, title="Similarity metric to use")
directionality: SemsimDirectionality = Field(
SemsimDirectionality.BIDIRECTIONAL, title="Directionality of the search"
diff --git a/backend/src/monarch_py/api/semsim.py b/backend/src/monarch_py/api/semsim.py
index 1d260538f..4963341f4 100644
--- a/backend/src/monarch_py/api/semsim.py
+++ b/backend/src/monarch_py/api/semsim.py
@@ -1,14 +1,16 @@
from typing import Optional
-from fastapi import APIRouter, Path, Query
+from fastapi import APIRouter, HTTPException, Path, Query
from monarch_py.api.additional_models import (
SemsimCompareRequest,
SemsimMetric,
SemsimSearchRequest,
+ SemsimProfileSearchRequest,
SemsimSearchGroup,
SemsimMultiCompareRequest,
SemsimDirectionality,
+ expand_search_group,
)
from monarch_py.api.config import semsim_service, solr
from monarch_py.api.utils.similarity_utils import parse_similarity_prefix
@@ -137,7 +139,11 @@ def _search(
limit: int = Query(default=10, ge=1, le=50),
engine: Optional[str] = EngineParam,
):
- """Search for terms in a termset
+ """Search for terms in a termset, within one of the legacy entity groups.
+
+ Superseded by `POST /search`, which takes explicit category/taxon filters and can express
+ targets no group can name — notably "all mouse models", which spans the MGI and MMRRC
+ prefixes. This route expands the group to those same explicit filters.
Args:
termset (str): Comma separated list of term IDs to find matches for.
@@ -149,31 +155,106 @@ def _search(
List[str]: List of matching terms
"""
terms = [term.strip() for term in termset.split(",")]
+ categories, taxa, prefixes = expand_search_group(group)
results = semsim_service(engine).search(
- termset=terms, prefix=parse_similarity_prefix(group), metric=metric, directionality=directionality, limit=limit
+ termset=terms,
+ prefix=parse_similarity_prefix(group),
+ metric=metric,
+ directionality=directionality,
+ limit=limit,
+ categories=categories,
+ taxa=taxa,
+ prefixes=prefixes,
)
return results
+@router.get("/filters")
+def _filters(engine: Optional[str] = EngineParam):
+ """The (category, taxon) combinations available to search, with entity counts.
+
+ Explicit filters are only usable if callers can find out what to pass, so this enumerates the
+ valid values from the KG itself rather than from a hand-maintained enum that drifts.
+ """
+ service = semsim_service(engine)
+ if not hasattr(service, "engine"):
+ return {"detail": "filter discovery requires the ducksim backend"}
+ return [
+ {"category": cat, "taxon": tax, "taxon_label": lab, "count": n}
+ for cat, tax, lab, n in service.engine.categories()
+ ]
+
+
@router.post("/search")
def _post_search(request: SemsimSearchRequest, engine: Optional[str] = EngineParam):
"""
- Search for terms in a termset
+ Search for entities whose phenotype profile matches a termset.
- Example:
+ Every mouse model, across MGI *and* MMRRC — not expressible with the legacy `group`:
{
- "termset": ["HP:0002104", "HP:0012378", "HP:0012378", "HP:0012378"],
- "group": "Human Diseases",
- "metric": "ancestor_information_content",
+ "termset": ["HP:0002104", "HP:0012378"],
+ "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
+ "metric": "phenodigm_score",
"limit": 5
}
+ Only orderable MMRRC strains:
+
+ {"termset": ["HP:0002104"], "filter": {"category": ["biolink:Genotype"], "prefix": ["MMRRC"]}}
+
+ Legacy form, still supported:
+
+ {"termset": ["HP:0002104"], "group": "Human Diseases", "limit": 5}
+
+ Call GET /semsim/filters for the category/taxon values this KG supports.
"""
+ if request.filter is None and request.group is None:
+ raise HTTPException(status_code=422, detail="provide either `filter` (preferred) or `group`")
return semsim_service(engine).search(
termset=request.termset,
- prefix=parse_similarity_prefix(request.group.value),
metric=request.metric,
directionality=request.directionality,
limit=request.limit,
+ **request.resolved_filter(),
+ )
+
+
+@router.post("/search-by-profile")
+def _post_search_by_profile(request: SemsimProfileSearchRequest, engine: Optional[str] = EngineParam):
+ """
+ Search using an entity's OWN phenotype profile as the query.
+
+ Mouse models for a patient (phenopacket Case -> mouse genotypes):
+
+ {
+ "entity": "phenopacket.store:PMID_38991538_Individual_1",
+ "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
+ "metric": "phenodigm_score"
+ }
+
+ Patients matching a mouse model (mouse genotype -> Cases):
+
+ {"entity": "MMRRC:000415-UCD", "filter": {"category": ["biolink:Case"]}}
+
+ The likely diagnosis for a patient (Case -> diseases):
+
+ {"entity": "phenopacket.store:PMID_38991538_Individual_1", "filter": {"category": ["biolink:Disease"]}}
+
+ Needs the ducksim backend (the entity's profile is read from the KG).
+ """
+ service = semsim_service(engine)
+ if not hasattr(service, "search_by_profile"):
+ raise HTTPException(
+ status_code=501, detail="search-by-profile requires the ducksim backend (?engine=ducksim)"
+ )
+ profile, results = service.search_by_profile(
+ entity_id=request.entity,
+ metric=request.metric,
+ directionality=request.directionality,
+ limit=request.limit,
+ **request.filter.as_kwargs(),
)
+ if not profile:
+ raise HTTPException(status_code=404, detail=f"no phenotype profile found for entity {request.entity!r}")
+ return {"entity": request.entity, "query_profile": profile, "results": results}
diff --git a/backend/src/monarch_py/service/ducksim.py b/backend/src/monarch_py/service/ducksim.py
index f86e3f739..2bc94b587 100644
--- a/backend/src/monarch_py/service/ducksim.py
+++ b/backend/src/monarch_py/service/ducksim.py
@@ -53,12 +53,21 @@ def _dedupe(seq):
class Ducksim:
"""Semantic-similarity engine over `monarch-kg.duckdb` (closure + edges), computed in DuckDB."""
- # entity->phenotype associations for search, from the KG `edges` table
+ # entity->phenotype associations for search, from the KG `edges` table.
+ #
+ # Selected by PREDICATE rather than by association category, so every kind of phenotype-bearing
+ # entity is searchable and the caller decides what to include via explicit category/taxon
+ # filters. The previous category allowlist (gene + disease only) structurally hid the 589,030
+ # GenotypeToPhenotypicFeature edges -- every MGI and MMRRC mouse model -- along with variants
+ # and cases, with no way for a caller to opt in.
+ #
+ # Because the pool is now heterogeneous, an unfiltered search mixes categories. Callers that
+ # mean "mouse genes" must say so (category=Gene, taxon=NCBITaxon:10090); a bare prefix no
+ # longer implies a category. `SemsimSearchGroup` expands to exactly that, so the legacy API
+ # keeps its old meaning.
DEFAULT_ASSOCIATIONS = (
"SELECT subject AS entity, object AS phenotype FROM src.edges "
- "WHERE category IN ('biolink:GeneToPhenotypicFeatureAssociation',"
- "'biolink:DiseaseToPhenotypicFeatureAssociation') "
- "AND predicate = 'biolink:has_phenotype' "
+ "WHERE predicate = 'biolink:has_phenotype' "
# keep everything that isn't explicitly negated. `negated` is a VARCHAR today ('True'/'False'/
# NULL), but try_cast-to-BOOLEAN keeps this correct across casing and a future boolean column;
# NULL (and any unparseable value) coalesces to false so the row is kept, never silently dropped.
@@ -152,8 +161,83 @@ def _define_associations(self, assoc_sql):
self.con.execute(f"CREATE VIEW _assoc AS {assoc_sql}")
self._require_baked("closure_size")
self.con.execute("CREATE VIEW _esize AS SELECT entity, size AS pn FROM src.closure_size")
+ self._define_entity_metadata()
self.has_search = True
+ def _define_entity_metadata(self):
+ """`_emeta`: (entity, category, taxon) for every searchable entity — what explicit
+ category/taxon filtering resolves against.
+
+ Materialized rather than left as a view over `src.nodes`: it is small (one row per
+ phenotype-annotated entity, ~200K) and every filtered search probes it, so paying the join
+ once at startup beats re-joining a 1.58M-row table per query.
+
+ A `nodes` table lacking `category`/`in_taxon` (minimal or older artifacts) is tolerated
+ rather than fatal, but the missing dimension is recorded so `entity_filter` can refuse a
+ filter it cannot honor. Filtering on an absent column would otherwise match nothing and
+ look like a legitimately empty result — the same silent-empty failure `closure_size`
+ already causes for genotypes in `_flat`.
+ """
+ cols = {
+ r[0]
+ for r in self.con.execute(
+ "SELECT column_name FROM duckdb_columns() WHERE database_name = 'src' AND table_name = 'nodes'"
+ ).fetchall()
+ }
+ self.entity_metadata = {d for d in ("category", "in_taxon") if d in cols}
+ cat = "n.category" if "category" in cols else "CAST(NULL AS VARCHAR)"
+ tax = "n.in_taxon" if "in_taxon" in cols else "CAST(NULL AS VARCHAR)"
+ taxl = "n.in_taxon_label" if "in_taxon_label" in cols else "CAST(NULL AS VARCHAR)"
+ self.con.execute(f"""
+ CREATE OR REPLACE TABLE _emeta AS
+ SELECT DISTINCT n.id AS entity, {cat} AS category,
+ {tax} AS taxon, {taxl} AS taxon_label
+ FROM (SELECT DISTINCT entity FROM _assoc) a JOIN src.nodes n ON n.id = a.entity
+ """)
+ self.con.execute("CREATE INDEX IF NOT EXISTS _ix_emeta ON _emeta(entity)")
+
+ # ---- explicit filtering ---------------------------------------------
+
+ def entity_filter(self, *, prefixes=None, categories=None, taxa=None, entities=None) -> str:
+ """Build the SQL WHERE clause selecting which entities a search may return.
+
+ Every argument is an optional list and they AND together; passing none searches everything.
+ This is the explicit replacement for the old single-`prefix` argument, which conflated
+ category and taxon into a data-source proxy and so could express "Mouse Genes" but not
+ "mouse models" (MGI and MMRRC genotypes live under two different prefixes) nor "any
+ genotype in any species".
+
+ `taxa` accepts either a CURIE ('NCBITaxon:10090') or a label ('Mus musculus'); labels are
+ matched case-insensitively so callers need not know which form the KG stores.
+ """
+ for requested, dimension in ((categories, "category"), (taxa, "in_taxon")):
+ if requested and dimension not in getattr(self, "entity_metadata", set()):
+ raise RuntimeError(
+ f"cannot filter by {dimension}: the attached KG's `nodes` table has no "
+ f"'{dimension}' column, so this filter would match nothing"
+ )
+ clauses = []
+ if entities:
+ clauses.append(f"entity IN (SELECT unnest([{_quote_list(entities)}]::VARCHAR[]))")
+ if prefixes:
+ clauses.append(f"split_part(entity, ':', 1) IN ({_quote_list(prefixes)})")
+ meta = []
+ if categories:
+ meta.append(f"category IN ({_quote_list(categories)})")
+ if taxa:
+ meta.append(f"(taxon IN ({_quote_list(taxa)}) OR lower(taxon_label) IN ({_quote_list(t.lower() for t in taxa)}))")
+ if meta:
+ clauses.append(f"entity IN (SELECT entity FROM _emeta WHERE {' AND '.join(meta)})")
+ return f"WHERE {' AND '.join(clauses)}" if clauses else ""
+
+ def categories(self) -> list:
+ """Distinct (category, taxon, taxon_label, count) available to search — lets a caller (or
+ an API docs page) discover valid filter values instead of guessing them."""
+ return self._read(
+ "SELECT category, taxon, taxon_label, count(*) AS n FROM _emeta "
+ "GROUP BY 1, 2, 3 ORDER BY n DESC"
+ ).fetchall()
+
# ---- labels ---------------------------------------------------------
def labels(self, ids) -> dict:
@@ -166,6 +250,16 @@ def labels(self, ids) -> dict:
).fetchall()
return {i: name for i, name in rows}
+ def profile(self, entity_id) -> list:
+ """The phenotype termset an entity is annotated with — its similarity query profile.
+
+ Reads `_assoc`, so it is uniform across every entity type the association pool covers:
+ a disease's HPO terms, a mouse genotype's MP terms, a phenopacket Case's HPO terms. That
+ uniformity is the point — it lets one search primitive be driven by "this case", "this
+ disease" or "this mouse" interchangeably, which is what patient->model and model->patient
+ matching both need."""
+ return self.entity_phenotypes_batch([entity_id]).get(entity_id, [])
+
def entity_phenotypes_batch(self, entity_ids) -> dict:
"""{entity -> [phenotype, ...]} for many entities in one query — enriches a whole search page
without a per-entity round-trip."""
@@ -339,6 +433,7 @@ def _termset_search(self, query_terms, metric, entity_filter, limit, direction="
}.get(direction)
if score_combiner is None:
raise ValueError(f"unknown direction {direction!r}")
+ self._require_phenotype_tables()
Q = _quote_list(set(query_terms))
score_expr = spec.sql_rank
lim = "" if limit is None else f"LIMIT {int(limit)}"
@@ -348,77 +443,142 @@ def _termset_search(self, query_terms, metric, entity_filter, limit, direction="
FROM qterms qt JOIN _clo c ON c.s = qt.q JOIN _ic ic ON ic.term = c.o),
qsize AS (SELECT q, count(*) AS sz FROM q_anc GROUP BY q),
nq AS (SELECT count(*) AS n FROM qterms),
- -- DISTINCT: an entity may be annotated to the same phenotype via multiple association
- -- rows (different evidence/sources). Without dedup, p_anc repeats that phenotype's
- -- ancestors, inflating the count(*) intersection below past the union size -> a zero or
- -- negative jaccard denominator (inf score; sqrt-of-negative for phenodigm). psize already
- -- uses count(DISTINCT), so the two must agree.
- ent_ph AS (SELECT DISTINCT entity AS e, phenotype AS p FROM _assoc {entity_filter}),
- np AS (SELECT e, count(DISTINCT p) AS n FROM ent_ph GROUP BY e),
- p_anc AS (SELECT ep.e, ep.p, c.o AS a FROM ent_ph ep JOIN _clo c ON c.s = ep.p),
- psize AS (SELECT e, p, count(DISTINCT a) AS sz FROM p_anc GROUP BY e, p),
- pair AS (
- SELECT pa.e, pa.p, qa.q, count(*) AS inter, max(qa.ic) AS resnik
- FROM p_anc pa JOIN q_anc qa ON qa.a = pa.a GROUP BY pa.e, pa.p, qa.q
- ),
- scored AS (
- SELECT pr.e, pr.p, pr.q, pr.resnik,
- pr.inter::DOUBLE / (ps.sz + qs.sz - pr.inter) AS jaccard
- FROM pair pr JOIN psize ps ON ps.e = pr.e AND ps.p = pr.p
- JOIN qsize qs ON qs.q = pr.q
- ),
- ranked AS (SELECT e, p, q, {score_expr} AS score FROM scored),
- dir1 AS (SELECT bm.e, sum(bm.best) / np.n AS avg1
- FROM (SELECT e, p, max(score) AS best FROM ranked GROUP BY e, p) bm
- JOIN np ON np.e = bm.e GROUP BY bm.e, np.n),
- dir2 AS (SELECT bm.e, sum(bm.best) / (SELECT n FROM nq) AS avg2
- FROM (SELECT e, q, max(score) AS best FROM ranked GROUP BY e, q) bm
- GROUP BY bm.e)
+ -- Scoring happens at the PHENOTYPE level, not per (entity, phenotype). A phenotype's
+ -- similarity to a query term does not depend on which entity carries it, so expanding
+ -- ancestors per association row recomputes the same ancestor set once per annotated
+ -- entity. Across mouse genotypes that is 447,110 rows covering 11,530 distinct MP
+ -- terms -- ~39x redundant, and the dominant cost of the whole query.
+ pair AS (SELECT pa.p, qa.q, count(*) AS inter, max(qa.ic) AS resnik
+ FROM _ph_anc pa JOIN q_anc qa ON qa.a = pa.a GROUP BY pa.p, qa.q),
+ ranked AS (SELECT s.p, s.q, {score_expr} AS score FROM
+ (SELECT pr.p, pr.q, pr.resnik,
+ pr.inter::DOUBLE / (ps.sz + qs.sz - pr.inter) AS jaccard
+ FROM pair pr JOIN _psize ps ON ps.p = pr.p
+ JOIN qsize qs ON qs.q = pr.q) s),
+ -- entities enter only here, by joining the per-phenotype scores onto their annotations
+ ent_ph AS (SELECT entity AS e, phenotype AS p FROM _ent_ph {entity_filter}),
+ dir1 AS (SELECT ep.e, sum(bp.best) / any_value(np.n) AS avg1
+ FROM ent_ph ep
+ JOIN (SELECT p, max(score) AS best FROM ranked GROUP BY p) bp ON bp.p = ep.p
+ JOIN _np np ON np.e = ep.e GROUP BY ep.e),
+ dir2 AS (SELECT e, sum(best) / (SELECT n FROM nq) AS avg2 FROM
+ (SELECT ep.e, r.q, max(r.score) AS best
+ FROM ent_ph ep JOIN ranked r ON r.p = ep.p GROUP BY ep.e, r.q) GROUP BY e)
SELECT coalesce(d1.e, d2.e) AS entity, {score_combiner} AS score
FROM dir1 d1 FULL OUTER JOIN dir2 d2 ON d1.e = d2.e
ORDER BY score DESC, entity {lim}
"""
return self._read(sql).fetchall()
+ def _require_phenotype_tables(self):
+ """Materialize the phenotype-level tables `_termset_search` scores against. Built once on
+ first search (a few seconds), reused by every later query.
+
+ `_ent_ph` is deduped here because an entity may be annotated to the same phenotype through
+ several association rows (different evidence or sources). Left undeduped, an entity's
+ matched phenotype would be counted once per row and `dir1` would over-sum. `_np` is derived
+ from the same deduped table so the two necessarily agree.
+ """
+ if getattr(self, "_pheno_ready", False):
+ return
+ self.con.execute("CREATE OR REPLACE TABLE _ent_ph AS SELECT DISTINCT entity, phenotype FROM _assoc")
+ self.con.execute("CREATE OR REPLACE TABLE _np AS SELECT entity AS e, count(*) AS n FROM _ent_ph GROUP BY 1")
+ self.con.execute("""
+ CREATE OR REPLACE TABLE _ph_anc AS
+ SELECT DISTINCT ep.phenotype AS p, c.o AS a
+ FROM (SELECT DISTINCT phenotype FROM _ent_ph) ep JOIN _clo c ON c.s = ep.phenotype
+ """)
+ self.con.execute("CREATE OR REPLACE TABLE _psize AS SELECT p, count(*) AS sz FROM _ph_anc GROUP BY p")
+ self.con.execute("CREATE INDEX IF NOT EXISTS _ix_ph_anc_a ON _ph_anc(a)")
+ self.con.execute("CREATE INDEX IF NOT EXISTS _ix_ent_ph_p ON _ent_ph(phenotype)")
+ self._pheno_ready = True
+
def full_search(
- self, query_terms, *, limit=10, metric="ancestor_information_content", prefix=None, direction="bidirectional"
+ self,
+ query_terms,
+ *,
+ limit=10,
+ metric="ancestor_information_content",
+ prefix=None,
+ direction="bidirectional",
+ categories=None,
+ taxa=None,
+ prefixes=None,
):
- """Score every entity (optionally restricted to CURIE `prefix`) by the termset
- best-match-average — one DuckDB query. Matches semsimian's Full mode; more accurate than
- Hybrid (no Jaccard prefilter dropping true-top entities)."""
- ef = f"WHERE split_part(entity, ':', 1) = {_quote_list([prefix])}" if prefix else ""
+ """Score every entity passing the filters by the termset best-match-average — one DuckDB
+ query. Matches semsimian's Full mode; more accurate than Hybrid (no Jaccard prefilter
+ dropping true-top entities).
+
+ `categories`/`taxa`/`prefixes` are the explicit filters and AND together. `prefix` is the
+ legacy single-value form, kept working for existing callers.
+ """
+ ef = self.entity_filter(
+ prefixes=prefixes or ([prefix] if prefix else None), categories=categories, taxa=taxa
+ )
return self._termset_search(query_terms, metric, ef, limit, direction)
def hybrid_search(
- self, query_terms, *, limit=10, metric="ancestor_information_content", prefix=None, direction="bidirectional"
+ self,
+ query_terms,
+ *,
+ limit=10,
+ metric="ancestor_information_content",
+ prefix=None,
+ direction="bidirectional",
+ categories=None,
+ taxa=None,
+ prefixes=None,
):
"""Hybrid search — semsimian's production mode: cheap Jaccard prefilter then termset rerank,
as one query over the candidate set. (The Jaccard prefilter is direction-agnostic; the
rerank honors `direction`.)"""
- flat = self._flat(query_terms, prefix)
+ ef = self.entity_filter(
+ prefixes=prefixes or ([prefix] if prefix else None), categories=categories, taxa=taxa
+ )
+ flat = self._flat(query_terms, ef)
if not flat:
return []
scores = sorted({j for _, j in flat}, reverse=True)
k = max(math.ceil((limit / 1000.0) * len(scores)), limit)
cutoff = scores[k] if k < len(scores) else scores[-1]
candidates = [e for e, j in flat if j >= cutoff]
- ef = f"WHERE entity IN (SELECT unnest([{_quote_list(candidates)}]::VARCHAR[]))"
- return self._termset_search(query_terms, metric, ef, limit, direction)
+ return self._termset_search(
+ query_terms, metric, self.entity_filter(entities=candidates), limit, direction
+ )
- def _flat(self, query_terms, prefix):
- """Cheap set-Jaccard ranking of all (prefix) entities vs the query — Hybrid's candidate gen."""
+ def _flat(self, query_terms, entity_filter=""):
+ """Cheap set-Jaccard ranking of the filtered entities vs the query — Hybrid's candidate gen.
+
+ `_esize` is koza's baked `closure_size`. It once covered only genes and diseases, so a
+ plain inner join dropped every genotype silently and returned an empty candidate set with
+ no error. koza now bakes a size for every entity carrying a has_phenotype edge
+ (monarch-initiative/koza `fix(information-content)`), which covers all 213,273 of them.
+
+ The LEFT JOIN and computed fallback stay as a compatibility shim: artifacts built before
+ that fix are still in circulation, and degrading to a slower correct answer is better than
+ silently returning none. Against a current artifact the fallback matches nothing and costs
+ nothing.
+ """
Q = _quote_list(set(query_terms))
- pfilter = f"AND split_part(i.entity, ':', 1) = {_quote_list([prefix])}" if prefix else ""
+ self._require_phenotype_tables()
return self._read(f"""
WITH qt(t) AS (SELECT unnest([{Q}]::VARCHAR[])),
q_anc AS (SELECT DISTINCT c.o AS a FROM _clo c JOIN qt ON c.s = qt.t),
qn AS (SELECT count(*) AS n FROM q_anc),
+ ent AS (SELECT entity, phenotype FROM _ent_ph {entity_filter}),
inter AS (SELECT a.entity, count(DISTINCT c.o) AS inter
- FROM _assoc a JOIN _clo c ON c.s = a.phenotype JOIN q_anc q ON q.a = c.o
- GROUP BY a.entity)
- SELECT i.entity, i.inter::DOUBLE / ((SELECT n FROM qn) + e.pn - i.inter) AS jaccard
- FROM inter i JOIN _esize e ON e.entity = i.entity
- WHERE true {pfilter} ORDER BY jaccard DESC, i.entity
+ FROM ent a JOIN _clo c ON c.s = a.phenotype JOIN q_anc q ON q.a = c.o
+ GROUP BY a.entity),
+ -- baked sizes where present, computed for the entities koza did not precompute
+ sz AS (SELECT i.entity, coalesce(e.pn, f.pn) AS pn
+ FROM inter i
+ LEFT JOIN _esize e ON e.entity = i.entity
+ LEFT JOIN (SELECT ep.entity, count(DISTINCT pa.a) AS pn
+ FROM ent ep JOIN _ph_anc pa ON pa.p = ep.phenotype
+ GROUP BY ep.entity) f ON f.entity = i.entity)
+ SELECT i.entity, i.inter::DOUBLE / ((SELECT n FROM qn) + sz.pn - i.inter) AS jaccard
+ FROM inter i JOIN sz ON sz.entity = i.entity
+ ORDER BY jaccard DESC, i.entity
""").fetchall()
# ---- search with full per-result detail -----------------------------
@@ -432,6 +592,9 @@ def search(
prefix=None,
direction="bidirectional",
mode="hybrid",
+ categories=None,
+ taxa=None,
+ prefixes=None,
):
"""All-DuckDB search: rank entities (Hybrid by default; Full when `mode="full"`), then enrich
the whole page with full termset detail in a constant number of queries — independent of
@@ -443,7 +606,16 @@ def search(
if spec is None:
raise ValueError(f"unknown metric {metric!r}")
ranker = self.full_search if mode == "full" else self.hybrid_search
- ranked = ranker(query_terms, limit=limit, metric=metric, prefix=prefix, direction=direction)
+ ranked = ranker(
+ query_terms,
+ limit=limit,
+ metric=metric,
+ prefix=prefix,
+ direction=direction,
+ categories=categories,
+ taxa=taxa,
+ prefixes=prefixes,
+ )
if not ranked:
return []
entity_ids = [e for e, _ in ranked]
diff --git a/backend/src/monarch_py/service/ducksim_service.py b/backend/src/monarch_py/service/ducksim_service.py
index 0750f498d..333dd3e79 100644
--- a/backend/src/monarch_py/service/ducksim_service.py
+++ b/backend/src/monarch_py/service/ducksim_service.py
@@ -59,16 +59,28 @@ def multi_compare(self, request: SemsimMultiCompareRequest) -> List[SemsimSearch
def search(
self,
termset: List[str],
- prefix: str,
+ prefix: str = None,
metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT,
directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL,
limit: int = 10,
+ categories: List[str] = None,
+ taxa: List[str] = None,
+ prefixes: List[str] = None,
) -> List[SemsimSearchResult]:
# Hybrid mode matches the semsimian server; full_search would be more accurate (see engine).
# The engine ranks and enriches the whole page in a constant number of DuckDB queries (no
# per-result round-trips); the loop below is pure in-memory model shaping.
direction = directionality.value if hasattr(directionality, "value") else str(directionality)
- page = self.engine.search(termset, limit=limit, metric=str(metric), prefix=prefix, direction=direction)
+ page = self.engine.search(
+ termset,
+ limit=limit,
+ metric=str(metric),
+ prefix=prefix,
+ direction=direction,
+ categories=categories,
+ taxa=taxa,
+ prefixes=prefixes,
+ )
if not page:
return []
# all-DuckDB hydration of result entities from the KG `nodes` table — no external entity store
@@ -87,6 +99,42 @@ def search(
for entity_id, score, comparison in page
]
+ def search_by_profile(
+ self,
+ entity_id: str,
+ metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT,
+ directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL,
+ limit: int = 10,
+ categories: List[str] = None,
+ taxa: List[str] = None,
+ prefixes: List[str] = None,
+ ):
+ """Search using an entity's OWN phenotype profile as the query termset.
+
+ The half of Case support that a termset search alone does not give you: a phenopacket Case
+ is a query source, not something you would retype 20 HPO ids for. Pull `entity_id`'s
+ profile from the KG, then search — so "diseases like this patient" (filter to Disease),
+ "mouse models for this patient" (Genotype + mouse taxon) and "patients like this mouse"
+ (Case) are one call each, differing only in the target filter.
+
+ Returns (query_profile, results). The profile is returned because a caller ranking many
+ entities wants to know how many terms drove each search — a 2-term case and a 40-term case
+ are not equally trustworthy, and hiding the count hides that.
+ """
+ profile = self.engine.profile(entity_id)
+ if not profile:
+ return [], []
+ results = self.search(
+ termset=profile,
+ metric=metric,
+ directionality=directionality,
+ limit=limit,
+ categories=categories,
+ taxa=taxa,
+ prefixes=prefixes,
+ )
+ return profile, results
+
# ---- model shaping --------------------------------------------------
def _hydrate(self, row: dict, entity_id: str) -> Entity:
diff --git a/backend/src/monarch_py/service/semsim_service.py b/backend/src/monarch_py/service/semsim_service.py
index db471688b..95baf02be 100644
--- a/backend/src/monarch_py/service/semsim_service.py
+++ b/backend/src/monarch_py/service/semsim_service.py
@@ -73,11 +73,32 @@ def multi_compare(self, request: SemsimMultiCompareRequest) -> List[SemsimSearch
def search(
self,
termset: List[str],
- prefix: str,
+ prefix: str = None,
metric: SemsimMetric = SemsimMetric.ANCESTOR_INFORMATION_CONTENT,
directionality: SemsimDirectionality = SemsimDirectionality.BIDIRECTIONAL,
limit: int = 10,
+ categories: List[str] = None,
+ taxa: List[str] = None,
+ prefixes: List[str] = None,
) -> List[SemsimSearchResult]:
+ """The semsimian HTTP server indexes entities by a single CURIE prefix, so it can honor
+ only the prefix form of a filter.
+
+ When `prefix` is set the call is a legacy group search and any accompanying
+ category/taxon arguments merely restate what the prefix already encodes, so they are
+ redundant and ignored. When it is not, the caller has asked for something explicit that
+ this backend cannot express — anything but a single prefix is refused rather than
+ answered with a broader result set that looks filtered but is not.
+ """
+ if prefix is None:
+ if categories or taxa or (prefixes and len(prefixes) > 1):
+ raise NotImplementedError(
+ "the semsimian backend cannot filter by category or taxon, or across multiple "
+ "prefixes; use the ducksim backend (SEMSIM_BACKEND=ducksim or ?engine=ducksim)"
+ )
+ if not prefixes:
+ raise ValueError("semsimian search requires a prefix")
+ prefix = prefixes[0]
host = f"http://{self.semsim_server_host}:{self.semsim_server_port}"
path = f"search/{','.join(termset)}/{prefix}:/{metric}?limit={limit}&directionality={directionality.value} "
url = f"{host}/{path}"
diff --git a/backend/tests/api/test_semsim_router.py b/backend/tests/api/test_semsim_router.py
index b4e059c93..4a02c7104 100644
--- a/backend/tests/api/test_semsim_router.py
+++ b/backend/tests/api/test_semsim_router.py
@@ -86,8 +86,18 @@ def test_get_search(mock_search, termset: str, metric: SemsimMetric):
response = client.get(f"/search/{termset}/{group.value}?metric={metric}&limit={limit}")
directionality = SemsimDirectionality.BIDIRECTIONAL
assert response.status_code == status.HTTP_200_OK
+ # The legacy group still sends its prefix, and additionally the explicit filters it expands
+ # to. The category is what keeps "Mouse Genes" meaning genes now that the association pool
+ # also holds genotypes and variants under the MGI prefix.
mock_search.assert_called_once_with(
- termset=["HP:123", "HP:456"], prefix=group.name, metric=metric, directionality=directionality, limit=limit
+ termset=["HP:123", "HP:456"],
+ prefix=group.name,
+ metric=metric,
+ directionality=directionality,
+ limit=limit,
+ categories=["biolink:Gene"],
+ taxa=["NCBITaxon:9606"],
+ prefixes=["HGNC"],
)
@@ -115,11 +125,68 @@ def test_post_search(mock_search):
assert response.status_code == status.HTTP_200_OK
mock_search.assert_called_once_with(
termset=["HP:123", "HP:456"],
- prefix=group.name,
metric=metric,
directionality=directionality,
limit=limit,
+ categories=["biolink:Gene"],
+ taxa=["NCBITaxon:9606"],
+ prefixes=["HGNC"],
+ )
+
+
+@patch("monarch_py.service.semsim_service.SemsimianService.search")
+def test_post_search_explicit_filter(mock_search):
+ """An explicit filter reaches the backend as category/taxon, with no prefix — the form no
+ legacy group can produce, since mouse models span the MGI and MMRRC prefixes."""
+ mock_search.return_value = MagicMock()
+
+ response = client.post(
+ "/search/",
+ json={
+ "termset": ["HP:123"],
+ "filter": {"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
+ "limit": 5,
+ },
+ )
+
+ assert response.status_code == status.HTTP_200_OK
+ kwargs = mock_search.call_args.kwargs
+ assert kwargs["categories"] == ["biolink:Genotype"]
+ assert kwargs["taxa"] == ["NCBITaxon:10090"]
+ assert kwargs["prefixes"] is None
+ assert "prefix" not in kwargs
+
+
+def test_post_search_explicit_filter_beats_group():
+ """When both are sent the explicit filter wins outright rather than intersecting with the
+ group's implied category, so the result set is never narrower than what was asked for."""
+ from monarch_py.api.additional_models import SemsimSearchRequest
+
+ request = SemsimSearchRequest(
+ termset=["HP:123"],
+ group=SemsimSearchGroup.HGNC,
+ filter={"category": ["biolink:Genotype"], "taxon": ["NCBITaxon:10090"]},
)
+ assert request.resolved_filter() == {
+ "categories": ["biolink:Genotype"],
+ "taxa": ["NCBITaxon:10090"],
+ "prefixes": None,
+ }
+
+
+def test_post_search_requires_a_filter():
+ """Neither `filter` nor `group` is a client bug, not a request to search all 200k entities.
+
+ Mounted on a real app rather than the bare router `client` used above, because FastAPI's
+ HTTPException-to-response handling lives at the app level: against a bare router the
+ exception would propagate and the status code never be exercised.
+ """
+ from fastapi import FastAPI
+
+ app = FastAPI()
+ app.include_router(router)
+ response = TestClient(app).post("/search/", json={"termset": ["HP:123"], "limit": 5})
+ assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
@patch("monarch_py.service.semsim_service.SemsimianService.multi_compare")
diff --git a/backend/tests/unit/test_ducksim.py b/backend/tests/unit/test_ducksim.py
index d73d3f45e..a9edf323b 100644
--- a/backend/tests/unit/test_ducksim.py
+++ b/backend/tests/unit/test_ducksim.py
@@ -161,6 +161,26 @@ def test_negated_associations_excluded(engine):
assert engine.entity_phenotypes_batch(["E:1"]) == {"E:1": ["A1"]}
+def test_profile_reads_entity_termset(engine):
+ """`profile` returns an entity's own phenotype termset — the query source for search-by-profile.
+ Uniform across entity types because it reads `_assoc`; here E:3 carries two phenotypes."""
+ assert engine.profile("E:3") == ["A1", "B1"]
+ assert engine.profile("E:1") == ["A1"]
+ assert engine.profile("E:4") == [] # only edge negated
+ assert engine.profile("NOPE:0") == [] # unknown entity
+
+
+def test_search_by_profile_uses_entity_as_query(engine):
+ """search_by_profile pulls the entity's profile, then searches — E:3 (phenotypes {A1,B1})
+ should retrieve itself first, and returns the driving profile so a caller can weight by it."""
+ svc = DucksimService(engine=engine)
+ profile, results = svc.search_by_profile("E:3", limit=3, categories=None, prefixes=["E"])
+ assert profile == ["A1", "B1"]
+ assert results[0].subject.id == "E:3" # an entity is most similar to itself
+ # an entity with no profile yields no query and no results, rather than searching on nothing
+ assert svc.search_by_profile("E:4", prefixes=["E"]) == ([], [])
+
+
def test_search_hydrates_from_duckdb_without_entity_store(engine):
"""All-DuckDB search: the service builds full SemsimSearchResults — including the result entity,
hydrated from the KG `nodes` table — with no external entity store (entity_implementation=None)."""