Add taxonomy-driven semantic link discovery and storage

dice-report/src/main/kotlin/com/embabel/dice/report/OntologicalSemanticLinkDiscoverer.kt

@@ -0,0 +1,167 @@
+/*
+ * Copyright 2024-2026 Embabel Pty Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.embabel.dice.report
+
+import com.embabel.dice.common.DiceMetadataKeys
+import com.embabel.dice.proposition.Proposition
+import com.embabel.dice.proposition.PropositionStatus
+import com.embabel.dice.taxonomy.Taxonomy
+import org.jetbrains.annotations.ApiStatus
+import org.slf4j.LoggerFactory
+
+/**
+ * Discovers adjacency-invisible entity links grounded in a shared taxonomy ancestor.
+ *
+ * Category assignments come only from ACTIVE propositions. A candidate is suppressed
+ * when its endpoints are directly co-mentioned or share an entity neighbour.
+ */
+@ApiStatus.Experimental
+class OntologicalSemanticLinkDiscoverer(
+    private val taxonomy: Taxonomy,
+    private val maxAncestorLevels: Int = 2,
+) : SemanticLinkDiscoverer {
+
+    private val logger = LoggerFactory.getLogger(OntologicalSemanticLinkDiscoverer::class.java)
+
+    init {
+        require(maxAncestorLevels >= 0) {
+            "maxAncestorLevels must not be negative: $maxAncestorLevels"
+        }
+    }
+
+    @ApiStatus.Experimental
+    override fun discover(propositions: List<Proposition>): List<SemanticLink> {
+        val active = propositions.filter { it.status == PropositionStatus.ACTIVE }
+        logger.debug(
+            "Ontological discovery: {} proposition(s) in ({} active)",
+            propositions.size,
+            active.size,
+        )
+
+        val categoryEvidence = categoryEvidence(active)
+        val neighbours = adjacency(active)
+        val entities = categoryEvidence.keys.sorted()
+        val links = mutableListOf<SemanticLink>()
+
+        for (i in entities.indices) {
+            for (j in i + 1 until entities.size) {
+                val a = entities[i]
+                val b = entities[j]
+                val aNeighbours = neighbours[a].orEmpty()
+                val bNeighbours = neighbours[b].orEmpty()
+                if (b in aNeighbours) continue
+                if (aNeighbours.any { it in bNeighbours }) continue
+
+                val connection = bestConnection(
+                    categoryEvidence.getValue(a),
+                    categoryEvidence.getValue(b),
+                ) ?: continue
+                links += SemanticLink(
+                    sourceEntityId = a,
+                    targetEntityId = b,
+                    connectingEntityIds = emptyList(),
+                    kind = LinkKind.INFERRED,
+                    sourcePropositionIds = (
+                        categoryEvidence.getValue(a).getValue(connection.categoryA) +
+                            categoryEvidence.getValue(b).getValue(connection.categoryB)
+                        ).sorted(),
+                    reviewStatus = ReviewStatus.CANDIDATE,
+                    confidence = 1.0 / (1 + maxOf(connection.levelsA, connection.levelsB)),
+                    rationale = "shared taxonomy ancestor '${connection.ancestorLabel}' " +
+                        "($a via ${connection.categoryA}, $b via ${connection.categoryB})",
+                )
+            }
+        }
+
+        val result = links.sortedWith(
+            compareBy(
+                { it.sourceEntityId },
+                { it.targetEntityId },
+            ),
+        )
+        logger.debug("Ontological discovery: {} indirect link(s) found", result.size)
+        return result
+    }
+
+    private fun categoryEvidence(
+        active: List<Proposition>,
+    ): Map<String, Map<String, Set<String>>> {
+        val evidence = linkedMapOf<String, MutableMap<String, MutableSet<String>>>()
+        for (proposition in active) {
+            val categoryId = proposition.metadata[DiceMetadataKeys.TAXONOMY_NODE] as? String ?: continue
+            for (entityId in proposition.mentions.mapNotNull { it.resolvedId }.distinct()) {
+                evidence
+                    .getOrPut(entityId) { linkedMapOf() }
+                    .getOrPut(categoryId) { linkedSetOf() }
+                    .add(proposition.id)
+            }
+        }
+        return evidence
+    }
+
+    private fun adjacency(
+        propositions: List<Proposition>,
+    ): Map<String, Set<String>> {
+        val neighbours = linkedMapOf<String, MutableSet<String>>()
+        for (proposition in propositions) {
+            val ids = proposition.mentions.mapNotNull { it.resolvedId }.distinct()
+            for (i in ids.indices) {
+                for (j in i + 1 until ids.size) {
+                    neighbours.getOrPut(ids[i]) { linkedSetOf() }.add(ids[j])
+                    neighbours.getOrPut(ids[j]) { linkedSetOf() }.add(ids[i])
+                }
+            }
+        }
+        return neighbours
+    }
+
+    private fun bestConnection(
+        categoriesA: Map<String, Set<String>>,
+        categoriesB: Map<String, Set<String>>,
+    ): AncestryConnection? = categoriesA.keys
+        .flatMap { categoryA ->
+            categoriesB.keys.mapNotNull { categoryB ->
+                val ancestor = taxonomy.sharedAncestor(categoryA, categoryB, maxAncestorLevels)
+                    ?: return@mapNotNull null
+                AncestryConnection(
+                    categoryA = categoryA,
+                    categoryB = categoryB,
+                    ancestorId = ancestor.id,
+                    ancestorLabel = ancestor.label,
+                    levelsA = taxonomy.levelsTo(categoryA, ancestor.id),
+                    levelsB = taxonomy.levelsTo(categoryB, ancestor.id),
+                )
+            }
+        }
+        .minWithOrNull(
+            compareBy(
+                { maxOf(it.levelsA, it.levelsB) },
+                { it.levelsA + it.levelsB },
+                { it.ancestorId },
+                { it.categoryA },
+                { it.categoryB },
+            ),
+        )
+
+    private data class AncestryConnection(
+        val categoryA: String,
+        val categoryB: String,
+        val ancestorId: String,
+        val ancestorLabel: String,
+        val levelsA: Int,
+        val levelsB: Int,
+    )
+}

dice-report/src/main/kotlin/com/embabel/dice/report/SemanticLinkStore.kt

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2024-2026 Embabel Pty Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.embabel.dice.report
+
+import com.embabel.agent.core.ContextId
+import org.jetbrains.annotations.ApiStatus
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * Persistence port for reviewable [SemanticLink]s.
+ */
+@ApiStatus.Experimental
+interface SemanticLinkStore {
+
+    /**
+     * Return the natural key for [link], independent of endpoint order.
+     */
+    @ApiStatus.Experimental
+    fun idOf(link: SemanticLink): String = listOf(
+        minOf(link.sourceEntityId, link.targetEntityId),
+        maxOf(link.sourceEntityId, link.targetEntityId),
+        link.kind.name,
+    ).joinToString("|")
+
+    /**
+     * Upsert [link] in [contextId].
+     *
+     * An existing accepted or rejected review decision is retained.
+     */
+    @ApiStatus.Experimental
+    fun record(contextId: ContextId, link: SemanticLink): SemanticLink
+
+    /**
+     * Find links in [contextId], optionally restricted to [status].
+     */
+    @ApiStatus.Experimental
+    fun find(contextId: ContextId, status: ReviewStatus? = null): List<SemanticLink>
+
+    /**
+     * Update the review status of [id] in [contextId], returning `null` when absent.
+     */
+    @ApiStatus.Experimental
+    fun updateReviewStatus(contextId: ContextId, id: String, status: ReviewStatus): SemanticLink?
+}
+
+/**
+ * Thread-safe, in-process [SemanticLinkStore] for tests and single-node applications.
+ */
+@ApiStatus.Experimental
+class InMemorySemanticLinkStore : SemanticLinkStore {
+
+    private val linksByContext = ConcurrentHashMap<ContextId, ConcurrentHashMap<String, SemanticLink>>()
+
+    @ApiStatus.Experimental
+    override fun record(contextId: ContextId, link: SemanticLink): SemanticLink {
+        val stored = linksByContext
+            .computeIfAbsent(contextId) { ConcurrentHashMap() }
+            .compute(idOf(link)) { _, existing ->
+                val refreshed = link.detached()
+                when (existing?.reviewStatus) {
+                    ReviewStatus.ACCEPTED,
+                    ReviewStatus.REJECTED,
+                    -> refreshed.copy(reviewStatus = existing.reviewStatus)
+
+                    else -> refreshed
+                }
+            }!!
+        return stored.detached()
+    }
+
+    @ApiStatus.Experimental
+    override fun find(contextId: ContextId, status: ReviewStatus?): List<SemanticLink> =
+        linksByContext[contextId]
+            ?.values
+            ?.asSequence()
+            ?.filter { status == null || it.reviewStatus == status }
+            ?.sortedBy(::idOf)
+            ?.map { it.detached() }
+            ?.toList()
+            ?: emptyList()
+
+    @ApiStatus.Experimental
+    override fun updateReviewStatus(
+        contextId: ContextId,
+        id: String,
+        status: ReviewStatus,
+    ): SemanticLink? = linksByContext[contextId]
+        ?.computeIfPresent(id) { _, link -> link.copy(reviewStatus = status) }
+        ?.detached()
+
+    private fun SemanticLink.detached(): SemanticLink = copy(
+        connectingEntityIds = connectingEntityIds.toList(),
+        sourcePropositionIds = sourcePropositionIds.toList(),
+    )
+}

dice-report/src/test/kotlin/com/embabel/dice/report/InMemorySemanticLinkStoreTest.kt

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2024-2026 Embabel Pty Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.embabel.dice.report
+
+import com.embabel.agent.core.ContextId
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertNull
+import org.junit.jupiter.api.Test
+
+internal class InMemorySemanticLinkStoreTest {
+
+    private val contextA = ContextId("context-a")
+    private val contextB = ContextId("context-b")
+
+    @Test
+    fun `record stores a candidate and refreshes evidence without resurrecting a rejection`() {
+        val store = InMemorySemanticLinkStore()
+        val candidate = link(confidence = 0.4, rationale = "initial")
+
+        assertEquals(ReviewStatus.CANDIDATE, store.record(contextA, candidate).reviewStatus)
+
+        val id = store.idOf(candidate)
+        store.updateReviewStatus(contextA, id, ReviewStatus.REJECTED)
+        val refreshed = store.record(
+            contextA,
+            link(confidence = 0.9, rationale = "refreshed"),
+        )
+
+        assertEquals(ReviewStatus.REJECTED, refreshed.reviewStatus)
+        assertEquals(0.9, refreshed.confidence)
+        assertEquals("refreshed", refreshed.rationale)
+        assertEquals(listOf(refreshed), store.find(contextA))
+    }
+
+    @Test
+    fun `record preserves an accepted review decision`() {
+        val store = InMemorySemanticLinkStore()
+        val candidate = link()
+        val id = store.idOf(candidate)
+        store.record(contextA, candidate)
+        store.updateReviewStatus(contextA, id, ReviewStatus.ACCEPTED)
+
+        val refreshed = store.record(contextA, link(confidence = 0.8))
+
+        assertEquals(ReviewStatus.ACCEPTED, refreshed.reviewStatus)
+        assertEquals(0.8, refreshed.confidence)
+    }
+
+    @Test
+    fun `find filters by review status and isolates contexts`() {
+        val store = InMemorySemanticLinkStore()
+        val linkA = link()
+        val linkB = link(source = "entity-c", target = "entity-d")
+        store.record(contextA, linkA)
+        store.record(contextA, linkB)
+        store.record(contextB, linkA.copy(reviewStatus = ReviewStatus.ACCEPTED))
+        store.updateReviewStatus(contextA, store.idOf(linkB), ReviewStatus.REJECTED)
+
+        assertEquals(listOf(linkA), store.find(contextA, ReviewStatus.CANDIDATE))
+        assertEquals(
+            listOf(ReviewStatus.CANDIDATE, ReviewStatus.REJECTED),
+            store.find(contextA).map(SemanticLink::reviewStatus),
+        )
+        assertEquals(listOf(ReviewStatus.ACCEPTED), store.find(contextB).map(SemanticLink::reviewStatus))
+    }
+
+    @Test
+    fun `update returns null for unknown ids and cannot cross context boundaries`() {
+        val store = InMemorySemanticLinkStore()
+        val candidate = link()
+        val id = store.idOf(candidate)
+        store.record(contextA, candidate)
+
+        assertNull(store.updateReviewStatus(contextA, "missing", ReviewStatus.REJECTED))
+        assertNull(store.updateReviewStatus(contextB, id, ReviewStatus.REJECTED))
+        assertEquals(ReviewStatus.CANDIDATE, store.find(contextA).single().reviewStatus)
+    }
+
+    @Test
+    fun `reversed endpoints share one natural key and one stored row`() {
+        val store = InMemorySemanticLinkStore()
+        val forward = link(source = "entity-a", target = "entity-b", rationale = "forward")
+        val reversed = link(source = "entity-b", target = "entity-a", rationale = "reversed")
+
+        assertEquals(store.idOf(forward), store.idOf(reversed))
+        store.record(contextA, forward)
+        store.record(contextA, reversed)
+
+        assertEquals(1, store.find(contextA).size)
+        assertEquals("reversed", store.find(contextA).single().rationale)
+    }
+
+    private fun link(
+        source: String = "entity-a",
+        target: String = "entity-b",
+        confidence: Double = 0.5,
+        rationale: String? = null,
+    ): SemanticLink = SemanticLink(
+        sourceEntityId = source,
+        targetEntityId = target,
+        connectingEntityIds = listOf("bridge"),
+        sourcePropositionIds = listOf("proposition-1"),
+        confidence = confidence,
+        rationale = rationale,
+    )
+}