diff --git a/docs/handbook/README.md b/docs/handbook/README.md new file mode 100644 index 000000000..a1923e441 --- /dev/null +++ b/docs/handbook/README.md @@ -0,0 +1,102 @@ +# Ontos Handbook (LLM Grounding Corpus) + +These documents are **internal grounding material for the Ask Ontos copilot**, +not user-facing product documentation. They define the canonical vocabulary, +role model, lifecycle states, ontology stack, data-quality model, and +end-to-end flows of Ontos so that a retrieval tool (`search_ontos_handbook`, +grep-style match) can return citable text fragments to the LLM at answer +time. + +Naming note: this corpus used to be called "concepts", but "Concept" is +already an Ontos ontology entity (an RDFS class / SKOS concept in the +knowledge graph). To avoid overloading the noun in code, API surface, +and docs, the LLM-grounding markdown corpus is "handbook". + +## How to read this corpus {#how-to-read} + +If you are new to Ontos, start with +[end-to-end-flows.md](end-to-end-flows.md) — it walks the bottom-up and +top-down flows that the rest of the corpus elaborates. Then read the +[entities-glossary.md](entities-glossary.md) as a one-paragraph map of +every first-class thing. Then read whichever lifecycle / domain doc your +task touches. Citation discipline is required (see below) — if the +copilot can't anchor a claim to an anchor in this corpus, it should +refuse to answer rather than hallucinate. + +## Scope {#scope} + +In scope: + +- Conceptual definitions of every first-class entity (data product, + contract, domain, agreement, workflow execution, concept, semantic + link, quality check, etc.) +- Role catalog, permission model, identity resolution, demo-mode override +- Status state machines for data products, contracts, agreements, and + workflow executions +- Ontology + knowledge graph + glossary distinctions +- Data quality model — definitions, measurements, rollups, DQX + integration +- End-to-end flows — bottom-up (UC → curated catalog) and top-down + (ontology → physical assets) +- Customer-voice "common questions" sections, role-targeted persona + framings + +Out of scope: + +- API reference (use the FastAPI `/docs` endpoint) +- File / line implementation citations (these live in + `docs/architecture/` if needed) +- Tutorials, marketing material, customer playbooks + +## Citation discipline {#citation-discipline} + +When the copilot answers a conceptual question, it attaches a citation of +the form `[ref: .md#]` to the relevant claim. In v1 these +citations are **internal-only** — they are stripped from the +user-facing response. They exist so reviewers can audit grounding +quality and so the citation contract is ready when we expose them in +v2. + +Authoring rules for citation-ready sections: + +1. Every concept worth citing has an explicit HTML anchor: + `## Thing {#thing}`. +2. Anchor names are kebab-case derived from the section heading. +3. Cross-references between docs use relative anchored links: + `[Output Port](data-contract-lifecycle.md#output-port)`. +4. Each file ends with a verification footer + (`_Last verified against codebase: YYYY-MM-DD_`). + +## Adding a new concept doc {#adding-a-doc} + +1. Pick a single subject (a role, a lifecycle, an entity family, a + cross-cutting concern). Keep each file under ~3.5 pages. +2. Add anchors for every concept the LLM might be asked about — not + every paragraph, but every nameable thing. Aim for 10–20 anchors per + file. +3. Ground claims in source (managers, db models, enums). Mark anything + in flux as "in the current Ontos version" or "evolving". +4. Include a Common Questions section if the doc covers something + customers ask about repeatedly. +5. Update the file list below. +6. Re-verify and bump the verification footer. + +## Current corpus {#current-corpus} + +| File | Subject | +|---|---| +| [roles-and-rbac.md](roles-and-rbac.md) | Permission model (feature × access level), built-in roles, identity resolution, demo-mode override, per-execution authz, Ontos-admin vs workspace-admin | +| [data-product-lifecycle.md](data-product-lifecycle.md) | ODPS data product states, deliverables, consumables, delivery methods, consumer principals, version family, common questions | +| [data-contract-lifecycle.md](data-contract-lifecycle.md) | ODCS v3.1.0 contract states, schema, quality definitions, editor-of-record framing, contracts-first vs products-first, version family, common questions | +| [agreement-workflow.md](agreement-workflow.md) | Approval workflow runtime, agreement vs execution vs wizard session, approval gates, grant_permissions, webhook extras, common questions | +| [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md) | Ontology, knowledge graph, semantic links, glossary, three-tier linking, runtime graph, SPARQL, round-trip current state, common questions | +| [asset-model.md](asset-model.md) | One-pager: unified Asset entity, ontology-driven AssetType, AssetTypeCategory, entity relationships, asset reviews | +| [delivery-and-propagation.md](delivery-and-propagation.md) | Delivery Method vs Delivery Mode, Direct/Indirect/Manual modes, change-type taxonomy, concept→UC tag flow, integration with grant_permissions, common questions | +| [mcp-and-ask-ontos.md](mcp-and-ask-ontos.md) | In-product Ask Ontos copilot (grounding, permissions, refusals) vs the external MCP server (tokens, scopes, JSON-RPC), common questions | +| [data-quality.md](data-quality.md) | ODCS quality definitions, per-entity quality items, DQX end-to-end flow, source enums, surfacing, common questions | +| [end-to-end-flows.md](end-to-end-flows.md) | Bottom-up flow (UC → product → contract → concept), top-down flow (ontology → assets → tags), where they meet, common questions | +| [entities-glossary.md](entities-glossary.md) | One-paragraph definitions of every first-class entity | +| [installation-and-troubleshooting.md](installation-and-troubleshooting.md) | Distribution channels (Marketplace vs Git), first-install prerequisites, update workflow, alembic discipline, common UI errors users hit (request-role prompt, 403s, scope-missing, grant_permissions, sync layout), common questions | +| [personas-quick-reference.md](personas-quick-reference.md) | Plain-language persona framings, pages they touch, what they ask Ask Ontos | + +_Last verified against codebase: 2026-05-29_ diff --git a/docs/handbook/agreement-workflow.md b/docs/handbook/agreement-workflow.md new file mode 100644 index 000000000..8732c058d --- /dev/null +++ b/docs/handbook/agreement-workflow.md @@ -0,0 +1,306 @@ +# Agreements and the Approval Workflow + +Ontos has two flavours of workflow: + +- **Process workflows** (`workflow_type = "process"`) — event-driven + automations that react to triggers like `on_create`, `on_subscribe`, + `on_status_change`. +- **Approval workflows** (`workflow_type = "approval"`) — wizard-driven + flows that collect user input across multiple steps and produce a signed + **agreement** record on completion. + +This document covers the approval flavour and the agreement artifact it +produces. The `grant_permissions` step is shared with process workflows +and is explained in context. + +## What you see in Ontos + +### Three concepts that get conflated {#three-concepts} + +Customers (and reviewers, and sub-agents writing code against the API) +often slur three different things together. They are distinct. + +| Concept | What it is | When it exists | +|---|---|---| +| **Workflow** | The *definition* — an ordered list of step configurations with a trigger and a scope. Edited from Settings → Workflows. | Whenever you've authored a workflow definition. | +| **Wizard Session** | The *in-flight runtime state* of an approval workflow. Holds the snapshotted workflow definition (so later edits don't change what's being signed) and the per-step user inputs collected so far. | From the moment a user opens the wizard until the workflow completes or is cancelled. | +| **Agreement** | The *immutable post-completion record* — who signed what, against which entity, with which workflow definition snapshot. | Persisted at the end of a successful approval workflow. Never modified after. | + +A related but separate entity is the **Workflow Execution** +(`WorkflowExecutionDb`) — the runtime row that tracks status (`pending` / +`running` / `paused` / `succeeded` / `failed` / `cancelled`) and current +step. Wizard sessions are the user-facing surface for in-flight approval +workflows; workflow executions are the lower-level state tracker that +both process and approval workflows write to. + +### What an agreement is {#what-is-an-agreement} + +An agreement is the durable record of a completed approval workflow. It +captures who agreed to what, against which entity, under which workflow +definition. Once written, an agreement is immutable: the workflow +definition in force at sign-time is snapshotted onto the record so +later edits to the workflow do not retroactively change the signed +terms. + +Stored fields (`AgreementDb`): + +- `entity_type`, `entity_id` — what the agreement is about (typically a + data product, data contract, output port, or access grant). +- `workflow_id` — pointer to the live workflow (may be `NULL` if the + workflow was deleted; the snapshot still describes what was signed). +- `wizard_session_id` — pointer to the wizard session that produced the + agreement. +- `step_results` — JSON list of per-step results (user input, computed + values, branching outcome). +- `workflow_snapshot` — immutable JSON of the workflow definition at + sign time. +- `workflow_name`, `workflow_version` — denormalized for quick lookup. +- `pdf_storage_path` — optional generated PDF artifact. +- `created_by`, `created_at`. + +The agreement complements (does not replace) the workflow execution +record: the `WorkflowExecutionDb` row captures runtime state while the +workflow is in flight; the agreement is the post-completion artifact. + +### Approval Gates {#approval-gates} + +An **Approval Gate** is a first-class concept across Ontos's lifecycle: +a moment where a configured approver must sign off before the entity +moves to the next state. The platform uses approval gates at well-known +junctures: + +- **Contract Approval** — when a contract transitions + `proposed → under_review → approved`. +- **Sandbox Ready** — when a product moves from `draft` to `sandbox`. +- **Product Certified** — when a product is submitted for certification + (status transitions through `proposed → under_review → approved`). +- **Product Active** — when a product is published (typically + `approved → active`). + +Each gate is implemented as an approval workflow that's matched by +trigger type when the corresponding lifecycle action fires. The gate +configuration names the approvers (group, role, business owner, or +explicit email list), the policy checks to run, and any side effects +(notification, PDF generation, UC grant, etc.). + +A gate may be configured to **auto-approve** under specific conditions +(e.g., `auto_approve=true` on an output port for low-sensitivity public +data) — in which case the gate is logged but not paused for human sign +off. + +### Roles in an approval flow {#approval-roles} + +Three logical roles participate, though a single person may play +several: + +- **Initiator / requester / signer** — the user who launched the + wizard. May be a Data Consumer requesting access to a Deliverable, a + Data Producer requesting publication, or any user acknowledging a + disclaimer. +- **Business owner / approver** — the party whose acceptance is + required for the agreement to be valid. Resolved from the workflow's + `approval` or `user_action` step configuration (e.g., + `approvers: "domain_owners"`, named groups, Ontos roles whose + `approval_privileges` cover the workflow's entity type, or explicit + emails). The role picker in the workflow designer filters to roles + that can approve the entity types the workflow is configured for, so + authors don't accidentally assign a role that has no approval + authority for the target entity. +- **Co-signer** (optional) — additional principals captured via the + `co_signers` step type. Each co-signer's acknowledgement is recorded + in `step_results`. + +The wizard launcher also captures an **on-behalf-of** principal when +the flow is initiated for a group or service principal rather than the +signer themselves. The captured `on_behalf_of` value flows through the +workflow context and is persisted on derived artifacts (e.g., the data +product subscription's `on_behalf_of_type` / `on_behalf_of_value` +columns). + +### Triggering an approval workflow {#triggers} + +Approval workflows are matched by **trigger type**, not by name. The +wizard dialog dispatches on `for_*` triggers, which are 1:1 mirrors of +the corresponding `on_*` process triggers: + +| Wizard trigger | Matching process trigger | Use | +|---|---|---| +| `for_approval_response` | (responds to a paused process workflow) | Approver acts on a step paused inside a running process workflow | +| `for_subscribe` | `on_subscribe` | Consumer subscribes to / signs a contract | +| `for_request_review` | `on_request_review` | Wizard before a review request | +| `for_request_access` | `on_request_access` | Wizard before an access grant request | +| `for_request_publish` | `on_request_publish` | Wizard before publish/deploy | +| `for_request_certify` | `on_request_certify` | Wizard before certification | +| `for_request_status_change` | `on_request_status_change` | Wizard before a status change | +| `on_first_access` | (same) | One-time terms-of-use acceptance at app entry | + +`on_first_access` is a session trigger: the frontend fires it on app +mount when the current user has not yet accepted the workflow at its +latest version. + +### Webhook steps {#webhook-step} + +A `webhook` step calls an external HTTP endpoint either through a Unity +Catalog Connection (preferred for secrets handling) or by direct URL. +The step config includes `method`, `body_template`, headers, and +optional extras for caller-supplied additional headers, query +parameters, and path segments that can be templated from workflow +context. Body templates can reference workflow context variables +(`${entity.consumer_principals}`, `${entity.}`, etc.), so +the workflow can drive ITSM tickets, e-mail providers, or Slack +notifications without baking secrets into the workflow definition. + +### Delivery {#delivery-step} + +A `deliver` step (distinct from `delivery`) dispatches the completed +agreement through one or more channels: `in_app` (Ontos notification), +`email` (via EmailService), and `webhook` (HTTP POST). Email is the +only channel that may be silently stripped if no email provider is +configured; authors are expected to integrate their own email provider +via a webhook step instead. + +### Agreement immutability and re-execution {#immutability} + +Once persisted, an agreement is not edited. If a workflow definition +changes after an agreement is signed, the signed agreement still +references the original definition via `workflow_snapshot`. To re-run +an approval for the same entity (e.g., a new contract version), a new +wizard session is launched, producing a new agreement row. + +## Under the hood + +### Wizard step types {#step-types} + +A workflow is an ordered list of steps. The `StepType` enum is the +authoritative catalog; the most common types for approval workflows are +listed below. Step branching uses `on_pass` / `on_fail` references to +other `step_id` slugs. + +| Step type | Purpose | +|---|---| +| `validation` | Evaluate a compliance DSL rule; pass/fail branches. | +| `approval` | Pause execution; resume when configured approvers respond. | +| `user_action` | Collect free-form user input — reason, acceptances, custom fields. | +| `on_behalf_of` | Capture self / group / SP principal at wizard start. | +| `legal_document` | Display a legal text the signer must scroll through. | +| `acknowledgement_checklist` | Force tick-box acknowledgement of named statements. | +| `co_signers` | Collect additional acknowledging principals. | +| `policy_check` | Evaluate an existing compliance policy by UUID. | +| `conditional` | Branch on a DSL expression. | +| `notification` | Send in-app / email / webhook notification. | +| `webhook` | Call an external HTTP endpoint (via UC Connections or raw URL). | +| `generate_pdf` | Render a PDF from `step_results` + per-step `pdf_contribution`. | +| `persist_agreement` | Write the agreement record. | +| `deliver` | Dispatch the signed agreement through configured channels. | +| `grant_permissions` | Grant Unity Catalog permissions via the service principal client. | +| `assign_tag` / `remove_tag` | Mutate tags on the trigger entity. | +| `entity_action` | Apply a status action (certify, publish, etc.) on the trigger entity. | +| `create_asset_review` | Open a formal Data Asset Review for tracking. | +| `script` | Execute Python code (gated by deployment policy). | +| `pass` / `fail` | Terminal nodes. | + +### Execution state machine {#execution-state-machine} + +A `WorkflowExecutionDb` row tracks the runtime status of a single +workflow invocation. The `ExecutionStatus` enum: + +`pending`, `running`, `paused`, `succeeded`, `failed`, `cancelled`. + +- `pending` — created, not yet started. +- `running` — actively executing a step. +- `paused` — waiting for an external event (typically an `approval` or + `user_action` step's response). +- `succeeded` — reached a `pass` terminal or completed all steps. +- `failed` — reached a `fail` terminal or a step raised. +- `cancelled` — explicitly cancelled by an authorized caller. + +Per-step results live in `WorkflowStepExecutionDb` with +`StepExecutionStatus`: `pending`, `running`, `succeeded`, `failed`, +`skipped`. The `passed` boolean captures the branching outcome (used by +`validation`, `policy_check`, `conditional`). + +### The `grant_permissions` step {#grant-permissions-step} + +`grant_permissions` is the bridge from a signed agreement to real Unity +Catalog access. The step: + +1. Reads the workflow execution context, including the resolved + consumer principals from the underlying data product + (`${entity.consumer_principals}` is available in templates because + the product enrichment runs before the step). +2. Uses the Ontos service principal's workspace client to issue UC + `GRANT` statements on the configured securables. +3. Records the issued grants in the workflow step's `result_data` for + audit, so revocation can find them later. + +The step requires the service principal to hold **`MANAGE`** on each +securable it grants on. `ALL_PRIVILEGES` is **not** sufficient. UC +accepts only account-level groups; workspace-only groups will be +rejected even if they resolve in the Ontos identity layer. + +## Common questions {#common-questions} + +**"My consumer subscribed but didn't get access — what's missing?"** + +Three usual causes. (1) The approval gate is paused waiting for the +Data Product Owner to respond — the workflow execution shows +`paused`, not `failed`. (2) The `grant_permissions` step ran but the +app SP doesn't have `MANAGE` on the target UC securable. (3) The +`consumer_principals` list resolves to a workspace-only group, which +UC rejects. + +**"What's the difference between a Wizard Session and an Agreement?"** + +The Wizard Session is in-flight — the user is mid-flow, hasn't +finished, can still go back. The Agreement is what gets written at +the end, immutable. The wizard session points at the agreement once +it's persisted; the agreement points back at the wizard session for +audit. + +**"If I edit the workflow definition while a wizard is in progress, +what happens to that wizard?"** + +Nothing immediate. The wizard session snapshotted the workflow +definition at the moment it was launched. The in-flight wizard +continues with the snapshot, not the latest definition. The +resulting agreement records the snapshot too. New wizard sessions +launched after the edit pick up the new definition. + +**"How do I know which workflow runs when a consumer subscribes?"** + +It's matched by trigger type. For a subscribe action, the wizard +dispatches `for_subscribe`. Any approval workflow with trigger +`for_subscribe` and a matching scope (workspace / domain / project) +is eligible; the most specific scope wins. The Settings → Workflows +view shows all active workflows and their triggers. + +**"Can I see a list of all paused workflows waiting for me?"** + +Yes — the in-app notification center shows pending approvals, and +the Agreements / Approvals page lists workflow executions where +you're a configured approver and the status is `paused`. + +**"What's the relationship between `grant_permissions` and the +product's `consumer_principals` list?"** + +The `consumer_principals` list on the product is the authoritative +list of "who is allowed to consume this product". When a subscribe +workflow runs, that list is part of the execution context. The +`grant_permissions` step reads the resolved principals via +`${entity.consumer_principals}` and issues UC `GRANT` statements +accordingly. Updating the list on the product after access is +already granted does *not* automatically revoke — you'd need a +matching revoke workflow. + +## Cross-references {#cross-references} + +- [Permission model](roles-and-rbac.md#permission-model) for outer + feature-gate behavior vs per-execution authorization +- [Consumer principals](data-product-lifecycle.md#consumer-principals) + for what the `grant_permissions` step actually grants to +- [Data contract](data-contract-lifecycle.md#what-is-a-contract) for + the contract being signed against +- [End-to-end flows](end-to-end-flows.md) — where approval gates fall + in the producer and consumer journeys + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/asset-model.md b/docs/handbook/asset-model.md new file mode 100644 index 000000000..02dd00976 --- /dev/null +++ b/docs/handbook/asset-model.md @@ -0,0 +1,137 @@ +# Asset Model + +A quick reference for the unified Asset entity and its ontology-driven type +system. For the longer story behind why the ontology is prescriptive, +see [Ontology and Knowledge Graph](ontology-and-knowledge-graph.md#prescriptive-principle). + +## What you see in Ontos + +### What an Asset is {#what-is-an-asset} + +An **Asset** is the Ontos-side handle Ontos keeps for a governed "thing". +The thing might be a UC table, a UC view, a notebook, a model, a +dashboard, a job, a pipeline, an API endpoint, a Power BI report — any +named resource the organization wants to apply governance to. The Asset +is the Ontos record; the thing itself lives in its native system. + +Each Asset carries a name and description, a typed asset type (driven +by the ontology), an optional domain, a platform, a location (the +fully-qualified name, URL, or path), free-form properties, quick tags, +and a lifecycle status (Draft / Active / Deprecated / Retired) shown +on the Asset detail page. + +### Ontology-driven Asset Types {#asset-types-ontology-driven} + +Asset types in Ontos are **not** a hardcoded list. They are derived +from the ontology that ships with the deployment, per the +[prescriptive-ontology principle](ontology-and-knowledge-graph.md#prescriptive-principle). + +Adding a new entity type to your knowledge model is an ontology edit — +add a class with the right annotations, re-sync — not a code change. +The form fields rendered for that type in the Asset Explorer, the icon +on the type chip, the relationship options in the relationship panel, +the persona visibility — all driven by the ontology. + +### Where Assets show up {#where-assets-show-up} + +- **Data Products** — Deliverables (output ports) reference one or more + Assets as their backing surface. +- **Data Contracts** — schema objects link to asset columns via + property-level semantic links; assets implement contracts through an + "implements contract" relationship. +- **Marketplace** — Assets surface through the data products they back. +- **Semantic Links** — Assets are valid targets for semantic links; + this is how a concept gets pinned to a UC table. +- **Asset Explorer** — the unified view across asset types, with + persona-based visibility filtering so each role sees the asset types + that are relevant to their work. + +### Asset Reviews {#asset-reviews} + +The **Data Asset Review** workflow lets a Producer request that a +Steward formally inspect an Asset before it gets attached to a published +product. Reviews are first-class approval workflows that produce an +Agreement on completion. The review captures inspection notes, sign-off, +and an optional approval recommendation. + +The feature ships in the current version. The legacy "Datasets" surface +is deprecated in favor of the unified Asset Explorer. + +## Under the hood + +### Persisted Asset record {#persisted-asset-record} + +Assets persist as `AssetDb` rows in the `assets` table, carrying name, +description, typed `asset_type_id`, optional `domain_id`, platform, +`location` (FQN, URL, or path), JSON `properties`, quick tags, and +lifecycle `status` (`draft` / `active` / `deprecated` / `retired`). + +### Asset-type sync from the TTL {#asset-type-sync} + +The asset-type pipeline runs at startup: + +1. `ontos-ontology.ttl` is parsed. +2. For every class annotated `ontos:modelTier "asset"`, a row in + `AssetTypeDb` is created or updated. The row carries the UI icon, + category, persona visibility, required/optional metadata schemas + (JSON schemas), and allowed incoming/outgoing relationship types. +3. The frontend's Asset Explorer reads `/api/asset-types` at load — it + doesn't ship a hardcoded list. + +### AssetTypeCategory {#asset-type-categories} + +`AssetTypeCategory` is a coarse classification on persisted asset types: + +- `DATA` — tables, views, streams, files +- `ANALYTICS` — dashboards, reports, metrics +- `INTEGRATION` — APIs, connectors +- `SYSTEM` — internal infrastructure references +- `CUSTOM` — user-defined types from custom ontology classes + +This is separate from `AssetCategory`, which is a connector-level +classification (`DATA` / `COMPUTE` / `SEMANTIC` / `VIZ` / `STORAGE` / +`OTHER`) used by integration adapters when normalizing platform-specific +types to the unified model. + +### Entity Relationships {#entity-relationships} + +Assets connect to each other — and to other Ontos entities — through +`EntityRelationshipDb` (`entity_relationships` table). The model is +deliberately polymorphic: + +- `source_type`, `source_id` — the originating entity +- `target_type`, `target_id` — the destination entity +- `relationship_type` — a string validated against the ontology at + write time (e.g., `implementsContract`, `hasColumn`, + `belongsToSystem`, `consumesFrom`, `derived_from`) +- `properties` — optional JSON for relationship-specific metadata + +The relationship types themselves are part of the ontology — adding a +new relationship type is an ontology edit. The table is indexed on +both endpoints and on relationship type for fast lookup in either +direction. + +### Cascade delete {#cascade-delete} + +Assets participate in a cascade-delete preview: deleting an asset +identifies dependent entities (children via hierarchical +relationships, products / contracts referencing the asset) so the +caller sees the blast radius before confirming. The preview is +exposed as a tree via `DeletePreviewItem`; the actual delete uses +`CascadeDeleteRequest` and returns a per-asset success / failure list. + +### Asset Reviews — workflow plumbing {#asset-reviews-plumbing} + +Reviews execute as `workflow_type = "approval"` workflow executions +with trigger `for_request_review`. The legacy "datasets" endpoints +(`/api/datasets`) are deprecated in favor of querying assets directly +through `/api/assets`. + +## Cross-references {#cross-references} + +- [Ontology — prescriptive principle](ontology-and-knowledge-graph.md#prescriptive-principle) +- [Semantic Link — the bridge from a concept to an asset](ontology-and-knowledge-graph.md#three-tier-linking) +- [Data Product — Deliverable / Output Port](data-product-lifecycle.md#output-port) — output ports point at assets +- [End-to-end Flow A — Step 1, "Bring an asset from UC into Ontos"](end-to-end-flows.md#step-a-1) + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/data-contract-lifecycle.md b/docs/handbook/data-contract-lifecycle.md new file mode 100644 index 000000000..79281450a --- /dev/null +++ b/docs/handbook/data-contract-lifecycle.md @@ -0,0 +1,319 @@ +# Data Contract Lifecycle + +A data contract in Ontos is the technical and semantic agreement that a +data product's Deliverable implements — schema, quality expectations, SLAs, +server endpoints, support channels, pricing. Ontos implements the **Open +Data Contract Standard (ODCS) v3.1.0**. + +The model customers reach for: *Ontos is the lifecycle manager of data +contracts. Create v2, inform users about the changes from v1. Ontos can diff +and include the list of changes.* The two operational ideas underneath that +sentence are **editor-of-record** and **indirect delivery**. + +## What you see in Ontos + +### What a contract is {#what-is-a-contract} + +A data contract is the technical and semantic agreement attached to a data +product's Deliverable. The DataContractsManager owns the lifecycle. + +Required identity fields (ODCS): + +- `kind` (default `DataContract`) +- `api_version` (default `v3.1.0`) +- `name`, `version` — semver-style version string +- `status` — lifecycle state (see [Status state machine](#status-state-machine)) + +Optional ODCS top-level fields: + +- `tenant`, `data_product` (free-text product name) +- `domain_id` — link to a Data Domain +- Description block (`usage`, `purpose`, `limitations`) +- `sla_default_element`, `contract_created_ts` + +Databricks extensions: `owner_team_id`, `project_id`, certification fields, +publication scope, personal draft fields, and parent/base-name links for +explicit versioning. + +### Editor of record {#editor-of-record} + +The default mental model: **Ontos is the editor of record for the +contract**. You draft, edit, propose, review, and version the contract +in Ontos. The contract has a stable DB row, an immutable agreement +trail (when approval workflows are involved), and a diff view between +versions. + +The alternative: customers who already have YAML contracts in their +workspace repo can run **indirect delivery via volume** — Ontos pushes +the canonical YAML representation into a configured Databricks Volume, +where the workspace deployment pipelines pick it up. The seam is +deliberate: + +- The DB is the editor of record. +- The Volume is the deployment surface. + +If a team edits the YAML in their workspace and then re-imports it, +they're choosing to make the workspace the editor of record — at which +point Ontos becomes the observer. Pick one direction and stay there +per contract; round-tripping leads to drift. + +This pattern is exactly the answer to the customer question: *"Switching +contract versions >>> trigger a job and notify people."* Ontos owns the +event of v1 → v2 (with diff + approval gate + notification fan-out); +the volume / job system owns the *deployment* of the new YAML. + +### Status state machine {#status-state-machine} + +Contracts use the unified `EntityStatus` enum but support a slightly +smaller set than data products (no `sandbox`): + +`draft`, `proposed`, `under_review`, `approved`, `active`, `deprecated`, +`retired`. + +Allowed transitions (`DATA_CONTRACT_TRANSITIONS`): + +| From | To | +|---|---| +| draft | proposed, deprecated | +| proposed | draft, under_review, deprecated | +| under_review | draft, approved, deprecated | +| approved | active, draft, deprecated | +| active | deprecated | +| deprecated | retired, active | +| retired | (terminal) | + +New contracts default to `draft`. Publishing (in the marketplace sense) +is governed by `publication_scope`, distinct from status. + +### Contracts-First vs Products-First {#contracts-vs-products-first} + +Two equally-supported workflow orderings. The platform doesn't favor one; +the choice is a team-culture choice. + +**Contracts-First.** The contract is drafted before product assets are +linked. Producer and consumer iterate on schema, quality expectations, +and SLAs in a shared design. The Steward approves the contract first. +The Producer then builds the product to satisfy the contract. Lower +risk of late surprises; higher upfront design cost. + +**Products-First.** The product is composed with assets first. Contracts +are attached to Deliverables later, possibly per Deliverable. The +contract is drafted to reflect what's already there. Faster initial +delivery; risk that consumer needs surface only after the product is +half-built. + +Both flows pass through the same status state machine. Stewards in +Contracts-First gate at contract approval; in Products-First, they gate +at product certification (and at contract approval if the contract is +formal). + +### Quality checks {#quality-checks} + +A quality check on a contract is a **check definition** attached to a +schema object (or a specific column). Definitions are *not* execution +results — execution and historical scoring are tracked separately +through the Quality panel on the data product and the compliance / +notifications surfaces. The full story is in +[data-quality.md](data-quality.md#contract-check-definitions). + +A check has: + +- `level` — `object` (table-wide) or `property` (column-specific). +- `dimension` — ODCS quality dimension: `accuracy`, `completeness`, + `conformity`, `consistency`, `coverage`, `timeliness`, `uniqueness`. +- `business_impact` — `operational` or `regulatory`. +- `severity` — `info`, `warning`, `error`. +- `type` — `library` (named rule), `text`, `sql` (free `query`), or + `custom` (with `engine` and `implementation`). +- A family of comparator fields (`must_be`, `must_not_be`, `must_be_gt`, + `must_be_between_min`/`max`, etc.) for declarative thresholds. + +Profiling runs record discovery/profiling activity (DQX, LLM-suggested, +manual) and produce *suggested* quality checks that can be accepted, +rejected, or modified before being promoted into real checks on the +contract. + +### Authoritative definitions {#contract-auth-defs} + +Contracts, schema objects, and individual properties can each link to +authoritative external definitions (`url` + `type`). This is how a +contract binds itself to a business term in the glossary, a +transformation specification, or a regulatory reference. + +### Publication and certification {#publication-certification} + +`publication_scope` (`none`/`domain`/`organization`/`external`) +replaces the legacy `published` boolean for marketplace visibility. +Certification fields (`certification_level`, +`inherited_certification_level`, `certified_at`, `certified_by`, +`certification_expires_at`, `certification_notes`) operate +independently from status. + +### Relationship to data products {#relationship-to-products} + +A contract is bound to a data product through a **Deliverable** of the +product (`OutputPortDb.contract_id`). A single contract may serve +multiple Deliverables — possibly across multiple products. The +product's Consumables carry contract references for upstream +dependencies, with the contract version pinned at integration time. +Subscribers to a product receive compliance alerts when the bound +contract's quality checks fail. + +### Versioning and diffing {#versioning-and-diffing} + +Versions are explicit rows grouped by a stable family identifier — +historically `base_name` paired with the `parent_contract_id` +parent-walk, and in the current Ontos version moving toward a canonical +`version_family_id` column that survives renames. The customer-voice +framing — *Create v2, inform users about the changes from v1, Ontos can +diff and include the list of changes* — maps onto two Ontos behaviors: + +1. The diff view between two contract versions presents schema, quality + check, SLA, and server changes side-by-side. +2. The contract subscription / dependency graph drives notification + fan-out when a new version is approved. Subscribers to the binding + product (and consumers of dependent products) get notifications; + the workflow can also fire a webhook to ITSM / Slack / email. + +List views collapse by family by default (one row per family, the +most-relevant version surfaced according to caller role) with a toggle +to expand and see every version. The detail view's version navigator +lets authors and consumers move across versions without losing context. + +If you also push the YAML to a volume (Indirect delivery — see +[Delivery and Propagation](delivery-and-propagation.md#indirect-mode)), +a workspace job picks up the new version and applies it — but the diff +and the notification fan-out happens in Ontos. + +## Under the hood + +### Schema objects and properties {#schema-objects} + +A contract's schema is a tree: + +- **Schema object** (`SchemaObjectDb`) — table-equivalent. Has `name`, + `logical_type` (default `object`), optional `physical_name`, + `physical_type`, `business_name`, `description`, `tags`, + `data_granularity_description`. +- **Schema property** (`SchemaPropertyDb`) — column-equivalent. Has + `name`, `logical_type`, `physical_type`, flags (`required`, `unique`, + `primary_key`, `partitioned`, `critical_data_element`), and rich + transformation metadata (`transform_source_objects`, + `transform_logic`, `transform_description`). Properties may nest via + `parent_property_id` for struct/array types. + +Properties can carry ODCS classification (`classification`), +encrypted-name hints, examples, and a JSON blob of logical-type-specific +options (`logical_type_options_json`). + +Each schema object and property is itself an entity that can carry +semantic links — see +[Three-tier linking](ontology-and-knowledge-graph.md#three-tier-linking). + +### Servers and environments {#servers} + +A contract's `servers` block lists the environments where the contract +applies. Each `DataContractServerDb` row carries `type` (required), +`environment`, optional `description`, and a `properties` collection of +key/value pairs for connection details. Servers support ODCS v3.1.0 +stable IDs so external referrers can point at a specific server entry +across version edits. + +### Contract roles {#contract-roles} + +A contract declares its own access roles (`DataContractRoleDb`) +independent of Ontos's RBAC roles. Each entry names a role with +`access`, `first_level_approvers`, `second_level_approvers`, and +optional custom properties. These represent the **access role** layer of +the contract — the business statement of who can read or write the +underlying data — and feed into the agreement workflow as configurable +approver groups. + +### SLAs and support {#sla-support} + +- `DataContractSlaPropertyDb` — typed SLA entries with `property`, + `value`, `unit`, `element`, and `driver`. Supports ODCS SLA + semantics. +- `DataContractSupportDb` — communication channels for support + (`channel`, `url`, `tool`, `scope`, `invitation_url`). +- `DataContractPricingDb` — single-row pricing block + (`price_amount`, `price_currency`, `price_unit`). + +### Relationships {#contract-relationships} + +ODCS v3.1.0 schema-level and property-level relationships (foreign +keys) are stored in `data_contract_schema_object_relationships` and +`data_contract_schema_property_relationships`. Relationship type +defaults to `foreignKey`; `from_value` / `to_value` are JSON-serialized +to allow single-string or array references. + +## Common questions {#common-questions} + +**"Does edits to the YAML in my workspace flow back into Ontos?"** + +In the current Ontos version: not automatically. Ontos is the editor +of record; the workspace volume is the deployment surface. If your +team edits the YAML in the workspace, you're implicitly choosing the +workspace as the editor of record — re-importing the edited YAML into +Ontos is possible, but it overwrites whatever Ontos held. Pick one +direction and stick to it for a given contract. + +**"What does 'switching contract versions' do — does it trigger a +job?"** + +Switching to a new active version transitions the previous version to +`deprecated`, transitions the new version to `active`, and fires the +configured event-driven workflows (e.g., `on_status_change`). Those +workflows are how customers wire downstream side effects: trigger a +deployment job, fan out notifications to subscribers, post to ITSM, +push the YAML to a volume. Ontos owns the event; the workflows you +configure own the side effects. + +**"Where do I see what changed between v1 and v2?"** + +The contract detail page has a Compare action that selects another +version of the same `base_name` and renders a structured diff. The +diff covers schema additions / removals / type changes, quality check +adds / removes / threshold changes, SLA changes, and server changes. + +**"Can a single contract cover multiple Deliverables?"** + +Yes. One contract can be bound to multiple Deliverables, possibly +across multiple products. The reverse — multiple contracts on a +single Deliverable — is not the model; a Deliverable has at most one +`contract_id`. + +**"My contract has 12 quality checks defined but the product's +Quality panel says 0% — why?"** + +Quality check definitions are not measurements. The contract holds +the design intent; the product's Quality panel reads +`QualityItemDb` measurements (per dimension, per source). If no +profiling run, no dbt run, no DQX run, and no manual measurement has +written a `QualityItemDb` row yet, there's nothing to roll up. See +[Two systems at a glance](data-quality.md#two-systems). + +**"Why are quality checks defined per schema property and not just +per schema object?"** + +ODCS allows both: object-level checks (the whole table) and +property-level checks (one column). Most checks land at the property +level because that's where the granularity is — completeness of +`customer_id`, conformity of `country_code` to ISO 3166. Object-level +checks cover row-count expectations, freshness of the latest +partition, and other table-wide invariants. + +## Cross-references {#cross-references} + +- [Data Quality](data-quality.md) — definitions vs measurements vs + DQX +- [Data Product](data-product-lifecycle.md#what-is-a-data-product) and + [Deliverable](data-product-lifecycle.md#output-port) +- [Approval workflow](agreement-workflow.md#approval-roles) for + contract review approval gates +- [Semantic Link](ontology-and-knowledge-graph.md#three-tier-linking) + for concept assignment at schema and property level +- [Bottom-up flow Step 3](end-to-end-flows.md#step-a-3) and the + Contracts-First vs Products-First decision + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/data-product-lifecycle.md b/docs/handbook/data-product-lifecycle.md new file mode 100644 index 000000000..868ed81c5 --- /dev/null +++ b/docs/handbook/data-product-lifecycle.md @@ -0,0 +1,320 @@ +# Data Product Lifecycle + +A data product in Ontos is a versioned, governed bundle of related Databricks +(or other-platform) assets, exposed through one or more **Deliverables** (the +customer-facing name for output ports), depending on declared **Consumables** +(input ports), owned by a team, and optionally bound to one or more data +contracts. It implements the **Open Data Product Standard (ODPS) v1.0.0** +with Databricks-specific extensions. + +In conversation you'll hear "Deliverable" and "Consumable". In the persisted +ODPS model these are `output_port` and `input_port`. Both vocabularies are +correct; the customer-facing names are primary. + +## What you see in Ontos + +### What a data product is {#what-is-a-data-product} + +A data product groups tables, views, functions, models, dashboards, +notebooks, and jobs into a single cohesive unit with explicit ownership, +a lifecycle, and machine-readable input/output descriptions. The +DataProductsManager is the entry point for all lifecycle operations. + +Core ODPS fields persisted on the product: + +- `api_version` (e.g., `v1.0.0`) +- `kind` (default `DataProduct`) +- `status` — lifecycle state (see [Status state machine](#status-state-machine)) +- `name`, `version`, `domain`, `tenant` +- `product_created_ts` +- Owner team reference (`owner_team_id`) and optional project (`project_id`) +- Description block (`purpose`, `limitations`, `usage`) +- Authoritative definitions (business definition links, tutorials, etc.) +- Custom properties (extensible key/value pairs) +- Input ports (Consumables), output ports (Deliverables), management ports +- Support channels and team metadata + +Databricks extensions on the product: + +- `consumer_principals` — typed list of identities authorized to consume + output ports (see [Consumer principals](#consumer-principals)). +- `publication_scope` — visibility band for marketplace discovery + (`none`, `domain`, `organization`, `external`). +- `certification_level` — optional ordinal pointing into the configured + certification levels. +- Personal draft fields (`draft_owner_id`, `parent_product_id`, + `base_name`, `change_summary`). + +### Status state machine {#status-state-machine} + +Data products use the unified `EntityStatus` enum. Valid statuses: + +`draft`, `sandbox`, `proposed`, `under_review`, `approved`, `active`, +`deprecated`, `retired`. + +Allowed transitions (`DATA_PRODUCT_TRANSITIONS`): + +| From | To | +|---|---| +| draft | sandbox, proposed, deprecated | +| sandbox | draft, proposed, deprecated | +| proposed | draft, under_review, deprecated | +| under_review | draft, approved, deprecated | +| approved | active, draft, deprecated | +| active | deprecated | +| deprecated | retired, active | +| retired | (terminal) | + +Notes: + +- New products default to `draft` if no status is supplied at creation. +- A product can be published (made visible in the marketplace) only when + the status is at least `approved`; publishing transitions it to + `active` and stamps `published_at` / `published_by`. +- Certification is a **separate dimension** from status (it was removed + from the status enum). Certification has its own fields and lifecycle. +- `retired` is terminal. A retired product cannot be revived. + +### Ownership model {#ownership} + +Three orthogonal layers of ownership apply to a data product: + +1. **Domain** (`domain` field, free-text in ODPS — typically a Data + Domain name). Selects which Data Governance Officer / Data Steward + has oversight. +2. **Owner team** (`owner_team_id` → `teams.id`). The team is the + durable organizational owner; team members inherit edit rights. +3. **Draft owner** (`draft_owner_id`). When set, the product is a + personal draft visible only to that user (and elevated roles). + Clearing `draft_owner_id` (the "commit" action) promotes the draft + to team-visible status. + +Authorization rules (enforced by DataProductsManager when checking edit +permissions): + +- Admins can always edit. +- Team members inherit edit access through the owner team. +- A user whose email matches `draft_owner_id` can always edit (covers + both personal drafts and single-user-owned products). +- Domain-scoped (`Filtered`) users can edit products in their domains. + +### Deliverables (output ports) {#output-port} + +A **Deliverable** describes a consumable surface of the data product. A +product typically has one or more Deliverables, each pointing at a +concrete asset and shipping through a specific delivery method. + +Required ODPS fields on a Deliverable: `name`, `version`. + +Optional fields: + +- `description`, `port_type` +- `contract_id` — link to a data contract. **May be NULL by design**: a + Deliverable can be declared without a contract during early lifecycle + stages, and a contract can be attached later. +- `delivery_method_id` — reference to a configured delivery method (see + [Delivery methods](#delivery-methods)). +- `asset_type`, `asset_identifier` — Databricks-side pointer (e.g., + `catalog.schema.table` for a UC table). +- `status` — independent port-level status string. +- `server` — JSON connection details. +- `contains_pii`, `auto_approve` — policy flags consumed by the + agreement workflow. + +A Deliverable can carry a list of **input contracts** (dependencies on +other contracts/versions) and an **SBOM** (software bill of materials) +block. + +### Consumables (input ports) {#input-port} + +A **Consumable** describes what the product reads. Unlike Deliverables, +**a `contract_id` is required** on every Consumable (per ODPS v1.0.0). +The identifier resolves to the contract version the product was built +against, making upstream changes diffable against the consumed schema. + +Databricks extensions: `asset_type`, `asset_identifier`. + +### Delivery methods {#delivery-methods} + +A Deliverable references a configured Delivery Method. The named values +shipped with Ontos: + +- **Table Access** — Consumer reads via Unity Catalog `SELECT`. The + `grant_permissions` step in approval workflows wires the UC grants. + This is the most common delivery method. +- **Serving Endpoint** — Consumer hits an HTTP serving endpoint (Mosaic + AI Model Serving or equivalent). Access is brokered by serving + endpoint configuration. +- **File Export** — Consumer pulls files from a configured location + (volume, S3, ADLS, GCS). Export schedule is established when access is + provisioned. +- **Streaming** — Consumer reads from a streaming source (Kafka, DLT, + etc.). Streaming-specific connection details ride on the Deliverable. + +The list is configurable from Settings → Delivery Methods, so a +deployment can extend it with org-specific delivery patterns (Postgres +shares, JDBC handoffs, dbt project handoffs, etc.). + +### Management ports {#management-port} + +Management ports expose administrative endpoints for the product: +discovery, observability, control, dictionary. Required fields are +`name` and `content` (one of `discoverability`, `observability`, +`control`, `dictionary`). Optional fields: `port_type` (default +`rest`), `url`, `channel`, `description`. + +### Consumer principals {#consumer-principals} + +`consumer_principals` is a JSON list of typed identity references +describing who is allowed to consume the product's Deliverables. Each +entry has a `type` and a `value`: + +- `type: "group"` (default) — a Databricks workspace/account group + display name. +- `type: "service_principal"` — an SP `applicationId`. +- `type: "role"`, `type: "scope"`, etc. — reserved for future identity + methods. + +The agreement workflow propagates the resolved consumer principals into +the workflow execution context so that `grant_permissions` steps and +webhook templates can reference `${entity.consumer_principals}` when +granting Unity Catalog privileges. + +Two operational rules here: + +- The Ontos app service principal needs **`MANAGE`** on each UC + securable it grants on. `ALL_PRIVILEGES` is **not** sufficient. +- UC accepts only **account-level groups**; workspace-only groups + will be rejected even if they resolve in Ontos's identity layer. + +### Semantic links and tags {#semantic-links-tags} + +A data product can carry: + +- **Semantic links** — references to ontology concepts (concept IRIs) + via the shared `entity_semantic_links` table. This is how a product + participates in the knowledge graph and glossary navigation. See + [Semantic Link](ontology-and-knowledge-graph.md#three-tier-linking). +- **Tags** — namespaced tag references via the shared tag system. See + [Tag](entities-glossary.md#tag) in the glossary. +- **Custom properties** — ODPS-native, free-form `property`/`value` + pairs at the product level. + +### Quality measurement attachment {#quality-attachment} + +A data product does not own quality checks directly. Quality checks +live on the data contract bound to a Deliverable. The product's Quality +panel is a rollup over the contracts the product binds — see +[Data Quality](data-quality.md#measurements-and-rollup). + +Subscribing to a product implicitly registers the consumer for +compliance alerts when the bound contract's quality checks fail. + +### Publication and subscription {#publication-subscription} + +- `publication_scope` controls marketplace visibility: `none` (default), + `domain`, `organization`, `external`. +- The legacy `published` boolean column is retained for backward DB + compatibility but is superseded by `publication_scope`. +- Consumers can subscribe to a product (`DataProductSubscriptionDb`), + opting into ITSM notifications for deprecations, new versions, and + compliance violations. Subscriptions support **subscribe-on-behalf-of** + for groups and service principals (`on_behalf_of_type`, + `on_behalf_of_value`). + +### Versioning {#versioning} + +Versioning is explicit: each version is its own product row. Versions of +the same product are grouped by a stable family identifier — historically +`base_name` paired with the `parent_product_id` parent-walk, and in the +current Ontos version moving toward a canonical `version_family_id` +column that survives renames and is propagated through every clone path. +Either way, the lifecycle invariant is the same: a row per version, +linked to its predecessors, with a separate `change_summary` capturing +the human-readable diff. There is no automatic version promotion — +version transitions are author-driven. + +List views collapse by family by default (one row per family, showing +the most-relevant version for the caller's role) with an option to +expand and see every version individually. Detail views surface a +version navigator so authors and consumers can move between versions +of the same product without losing context. + +## Under the hood + +Field-level persistence details (the `DataProductDb` row, the ODPS extension columns `consumer_principals` / `publication_scope` / `certification_level`, the on-behalf-of subscription columns) are documented in `src/backend/src/db_models/data_products.py` and the ODPS v1.0.0 spec the schema implements. + +## Common questions {#common-questions} + +**"What is the difference between Deliverable and output port?"** + +Same thing. Deliverable is the customer-facing name we use in +conversation, in the UI, and in the marketplace; output port is the +ODPS-spec label used in the persisted model and in ODPS exports. Same +for Consumable vs input port. Use whichever name your audience uses. + +**"My Deliverable has a NULL contract — is that allowed?"** + +Yes. Deliverables can be declared without a contract during early +lifecycle stages (draft, sandbox, proposed). You can attach a contract +later. Consumables, by contrast, are required to reference a contract +version per ODPS — this is what makes upstream changes diffable against +what you consumed. + +**"What does `auto_approve` on a Deliverable do?"** + +It is a policy flag the agreement workflow reads. When set, the +subscribe / request-access workflow short-circuits the approval gate +and proceeds directly to `grant_permissions`. Use sparingly — it's +appropriate for low-sensitivity, public-data products where governance +overhead has no upside. + +**"How do I move my product from draft to proposed?"** + +Click the status action on the detail page. The transition fires the +`on_request_status_change` process trigger (if a workflow is matched) +or the inline status update path otherwise. The transition itself only +needs `data-products:READ_WRITE` and ownership; an approval gate, if +configured, will pause for the approver before the status flips. + +**"Why don't I see this product in the marketplace?"** + +Three usual causes. (1) `publication_scope` is `none` (default for new +products). (2) Status is not yet `active`. (3) Publication scope is +`domain` and the consumer isn't in the domain. Look at status and +publication_scope on the product detail page. + +**"What does publishing a product do — does it grant permissions +automatically?"** + +Publishing transitions status to `active` and makes the product +visible in the marketplace. It does **not** by itself grant UC +permissions to consumers. Permissions get granted when a consumer +subscribes and the subscribe workflow runs its `grant_permissions` +step. The publication step is a *visibility* operation; the +subscription step is the *access* operation. They are deliberately +separate. + +**"My product has 4 deliverables — does each one need its own +contract?"** + +You can do that, or you can let one contract serve several +Deliverables (the contract's schema can describe the union of what the +Deliverables expose). Most teams start with one contract per +Deliverable for clarity and only consolidate when they have repeated +patterns. + +## Cross-references {#cross-references} + +- [Data Contract](data-contract-lifecycle.md#what-is-a-contract) for + what the bound contract carries +- [Approval workflow](agreement-workflow.md#approval-roles) for how + subscribe / publish gates work +- [Data Quality rollup](data-quality.md#measurements-and-rollup) for + the Quality panel +- [Semantic links](ontology-and-knowledge-graph.md#three-tier-linking) + for concept assignment +- [Bottom-up flow](end-to-end-flows.md#flow-a-bottom-up) for the + end-to-end producer journey + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/data-quality.md b/docs/handbook/data-quality.md new file mode 100644 index 000000000..1bae21a91 --- /dev/null +++ b/docs/handbook/data-quality.md @@ -0,0 +1,271 @@ +# Data Quality + +Data quality in Ontos is split into **two parallel systems** that interact at +specific seams. Customers consistently get confused here — they assume the +quality checks on the contract are also the execution results, and they ask +where DQX-style profiling output lives. The short version: the contract +holds *definitions*; a separate per-entity store holds *measurements*; the +DQX workflow is the most-integrated execution path; everything else lands +via the `external` source. + +## What you see in Ontos + +### The two systems at a glance {#two-systems} + +| System | What it stores | Where you see it | +|---|---|---| +| Contract quality checks | Check **definitions** — the rule, dimension, severity, threshold | Contract detail page → **Quality Rules** section: per-object and per-column check rows authored by the Steward. The contract page also has a **Profile with DQX** button (the action that generates these rules) | +| Per-entity quality measurements | Check **measurements** — score, pass/fail, when, by which engine | Data Product detail page → **Quality panel**: rolled-up scores by dimension and source | + +A contract that has 12 checks defined and zero measurements is normal — +the contract is the design intent, the measurements are what actually +happened last Tuesday at 03:00. + +### DQX integration — what you do, step by step {#dqx-flow} + +DQX is the most-tightly-wired integration. It is a complete loop, not a +one-shot. + +**Step 1 — Steward kicks off profiling.** On the contract detail page, +click the **Profile with DQX** action. This launches a background +profiling workflow. + +**Step 2 — Workflow profiles a sample.** For each schema in the +contract, the workflow profiles a sample of the underlying data +(approximately 10% of rows, capped at 5,000) and proposes quality +rules from what it observes. + +**Step 3 — Suggestions appear inline.** The proposed rules land as +**pending suggestions** visible on the contract's Schema tab next to +the corresponding columns. They are *not yet* real checks — they're a +draft waiting for your review. + +**Step 4 — Review and accept.** From the contract's Schema tab, you see +each pending suggestion inline with the column it targets. **Accept** to +promote it to a real check; **Reject** to dismiss; or **Edit** to +modify thresholds before accepting. + +**Step 5 — Periodic re-measurement.** Later runs of profiling (or any +quality engine configured against the contract) re-measure the columns +and post new measurements against the right contract source. + +**Step 6 — Rollup feeds the Quality panel.** The Data Product detail +page Quality panel averages the latest measurement per dimension and +shows the overall quality score, the per-dimension breakdown, and the +per-source breakdown. + +**Step 7 — Subscribers get compliance alerts.** A subscription to a +data product implicitly subscribes the consumer to compliance alerts +for the bound contracts. When a measurement at `error` severity fails, +subscribers are notified via the configured notification channels. + +Profiling-run state surfaces inside Ontos — if a profiling run fails, +the contract page shows the failure status and the error so you don't +have to drill into the Databricks Workflows UI to find out what went +wrong. + +### Where quality surfaces in the UI {#where-it-surfaces} + +- **Data Product detail page → Quality panel.** Reads the rolled-up + quality summary for the contracts bound to the product. Shows + per-dimension scores and per-source breakdown. This is the "is my + product healthy?" view. +- **Data Contract detail page → Schema tab.** Shows per-check + *definitions* attached to each schema object / property, plus + *suggested* checks pending review from the most recent profiling run. + This is the "what does the contract require?" view. +- **Subscription compliance alerts.** A consumer who subscribes to a + data product receives notifications when the bound contract's quality + checks fail at `error` severity. The channels are configured per + notification type. + +### External quality sources {#external-sources} + +Customers running their own DQ pipelines outside Ontos can still get +their results to show up in the Quality panel. The supported sources +the platform recognizes: + +- **Manual** — entered by hand (a Steward filling in a one-time number). +- **dbt** — dbt test results. Supported as a source value; the dedicated + import path is not yet shipping in the current Ontos version, so use + the external path below in the meantime. +- **DQX** — the integrated path described above. +- **Great Expectations** — same status as dbt. +- **Soda** — same status. +- **External** — a deliberate catch-all. Custom DQ pipelines can post + measurements through the public quality API and the rollup treats + them like any other source. + +If you have an organization-standard DQ tool that isn't on this list, +the recipe is: post your results through the external source, populate +the dimension and score, and the rollup will pick them up. + +## Under the hood + +### ODCS check definitions on the contract {#contract-check-definitions} + +A `DataQualityCheckDb` row carries: + +- **Level** — `object` (whole table) or `property` (one column). Property + checks set `property_id`; object checks leave it null. +- **Dimension** — one of the ODCS-native dimensions: `accuracy`, + `completeness`, `conformity`, `consistency`, `coverage`, `timeliness`, + `uniqueness`. +- **Business impact** — `operational` or `regulatory`. Drives how a + violation propagates: operational fires consumer alerts; regulatory + additionally flags compliance. +- **Severity** — `info`, `warning`, or `error`. Surfaces in UI badges and + in subscriber notifications. +- **Type** — `library` (a named reusable rule), `text` (free description + the steward fills in), `sql` (a `query` field with a SQL predicate), or + `custom` (with an `engine` and `implementation` field for plugging in + external tooling). +- **A family of declarative comparator fields** — `must_be`, + `must_not_be`, `must_be_gt`, `must_be_lt`, `must_be_between_min`, + `must_be_between_max`, etc. These are how the steward expresses "this + metric must be greater than 0.99" without writing SQL. + +Definitions ride along with the contract through its lifecycle. They are +not executed by the contract itself. Something else has to actually +measure the column and report back — see the next section. + +### Per-entity measurements and rollup {#measurements-and-rollup} + +A `QualityItemDb` row is generic: scoped by `entity_type` (one of +`data_product`, `data_contract`, `asset`, `data_domain`) and `entity_id`. +Each row records one measurement at one moment by one source. + +The row carries: + +- `score_percent` (0–100) +- `checks_passed`, `checks_total` +- `measured_at` (timestamp) +- `dimension` — same enum as the contract-check dimension +- `source` — one of `manual`, `dbt`, `dqx`, `great_expectations`, `soda`, + `external`. (See [external sources](#external-sources) above.) + +`QualityManager.aggregate_for_product` is the rollup that the Data Product +detail page reads. The logic is: + +1. Find the contracts bound to the product (via output ports → contract + ID). +2. Pull all `QualityItemDb` rows for those contract IDs. +3. Keep the **latest** measurement per `(entity_type, entity_id, + dimension)` tuple. Stale measurements drop out — only the freshest + reading per dimension survives. +4. Average per-dimension and per-source. +5. Return a `QualitySummary` with `overall_score_percent`, + `by_dimension`, and `by_source`. + +Crucially, a data product does **not** own quality directly. The product's +Quality panel is a view over the contracts it binds. If you want quality +to show up on a product, attach a contract to one of its output ports and +let measurements flow into that contract. + +### DQX workflow internals {#dqx-internals} + +The Profile with DQX action launches the `dqx_profile_datasets` workflow. +For each schema in the contract, the workflow uses +`databricks.labs.dqx.profiler.profiler.DQProfiler` to profile a sample +(10% sample, capped at 5000 rows). It hands the profile to `DQGenerator` +and calls `generator.generate_dq_rules(profiles, level="error")` to +propose rules. + +Each generated rule lands as a row in the `suggested_quality_checks` +table with `status = 'pending'`. Accepting a suggestion promotes it +into a real `DataQualityCheckDb` attached to the relevant +`SchemaObjectDb` or `SchemaPropertyDb`. Later periodic profiling runs +write `QualityItemDb` rows with `source = 'dqx'`. + +Profiling-run state is tracked in `data_profiling_runs`. Each run has a +`status` (`running` / `completed` / `failed`) and a `summary_stats` blob. +A failed run surfaces `status = 'failed'` plus an `error_message`. + +### DQX → ODCS dimension mapping {#dqx-odcs-mapping} + +DQX has its own rule names. When suggestions are written, those names map +to ODCS dimensions: + +| DQX rule name pattern | ODCS dimension | +|---|---| +| `is_not_null` | `completeness` | +| `is_in`, `min_max`, `pattern` | `conformity` | +| `is_unique` | `uniqueness` | +| anything else | `accuracy` (fallback) | + +This means a profiling run will populate several dimensions at once for a +typical column. The fallback to `accuracy` is intentional — it ensures +every DQX rule lands somewhere on the ODCS scale even if there's no +obvious mapping. + +### External source enum and write path {#external-source-internals} + +The `source` enum supports `manual`, `dbt`, `dqx`, `great_expectations`, +`soda`, `external`. For the engines whose dedicated importer isn't +shipping yet (`dbt`, `great_expectations`, `soda`), the practical path +today is to write `QualityItemDb` rows via the manager with +`source='external'`, populate the dimension and score, and let the +rollup pick them up. This is the same path the integrated engines use; +the only difference is which workflow writes the rows. + +## Common questions {#common-questions} + +**"How do DQX checks made outside Ontos surface inside Ontos?"** + +Two paths. (1) If your external DQX pipeline writes back to the +`QualityItemDb` table via the manager with `source='dqx'`, they show up in +the product's Quality panel automatically. (2) If your pipeline writes to +its own custom delta tables, those results are invisible to Ontos until +something translates them into `QualityItemDb` rows. There is no current +shipping job that reads from arbitrary external DQX delta tables. + +**"If I accept 14 suggested rules from a profiling run, do they go back +into the contract YAML?"** + +Yes — at least within Ontos. Accepting a suggestion promotes it from +`suggested_quality_checks` to a real `DataQualityCheckDb` row attached to +the contract's schema object or property. The contract's ODCS export +includes those checks. If you maintain a YAML copy of the contract in +your workspace (indirect-delivery via volume), the next contract version +generated by Ontos will reflect the new checks. The seam between Ontos's +DB-of-record and an externally-edited YAML is a place where teams need to +pick one as authoritative — see +[data-contract-lifecycle.md](data-contract-lifecycle.md#editor-of-record). + +**"Does Ontos own DQ execution or just observe results?"** + +For DQX it owns execution (the workflow runs from the contract page and +writes back). For everything else, Ontos prefers to *observe* — it +expects the engine of record (your dbt project, your Great Expectations +suite, your Soda checks) to write its results to `QualityItemDb` and lets +the rollup do its work. Customers who want Ontos to be the orchestrator +of all DQ runs are at a sharp end of the spec — the current Ontos version +is biased toward observation. + +**"Where do I see a failed profiling job?"** + +Inside Ontos. The `data_profiling_runs` row has a `status` field and an +`error_message`. The contract page surfaces the latest run status. The +single-pane-of-glass goal is intentional — customers shouldn't need to +chase the Databricks Workflows UI to find out why a profiling job +failed. + +**"Can I import dbt test results?"** + +The `source='dbt'` enum exists, so the schema is ready. But the +dedicated import workflow is not currently shipping. The pragmatic path +today is to use the `external` source and write your dbt test outcomes +through the manager. Treat this as an evolving area — first-class dbt +integration is the kind of thing that becomes a discrete shipped feature +later. + +## Cross-references {#cross-references} + +- [Quality check definitions on contracts](data-contract-lifecycle.md#quality-checks) +- [Subscription compliance alerts](data-product-lifecycle.md#publication-subscription) +- [QualityItem](entities-glossary.md#quality-item) and + [Quality Check](entities-glossary.md#quality-check) in the glossary +- [Bottom-up flow Step 4](end-to-end-flows.md#flow-a-bottom-up) for where + DQ fits in the day-to-day contract-authoring journey + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/delivery-and-propagation.md b/docs/handbook/delivery-and-propagation.md new file mode 100644 index 000000000..bd6357615 --- /dev/null +++ b/docs/handbook/delivery-and-propagation.md @@ -0,0 +1,303 @@ +# Delivery and Propagation + +Two unrelated things in Ontos share the word *delivery*. Customers conflate +them constantly, and so do engineers writing tools that touch them. Sort the +two out first; the rest of this doc is about the second one. + +## What you see in Ontos + +### Two "delivery" concepts {#two-deliveries} + +| Term | What it governs | Where it shows up | +|---|---|---| +| **Delivery Method** | How a product's **data** flows to consumers — the channel customers connect to. | On a Deliverable (output port). Configured from Settings → Delivery Methods. | +| **Delivery Mode** | How a governance **change** propagates out of Ontos into the underlying system (Unity Catalog, Git, a human's queue). | On the Delivery Service. Configured from Settings → Delivery. | + +The values are also disjoint, so there is no overlap by mistake: + +- Delivery Method values are `Table Access`, `Serving Endpoint`, `File + Export`, `Streaming` — the `DeliveryMethodCategory` enum (`access`, + `endpoint`, `export`, `streaming`). +- Delivery Mode values are `Direct`, `Indirect`, `Manual` — the + `DeliveryMode` enum. + +When in doubt: *Method* answers "how does the consumer get bytes?" — *Mode* +answers "how does Ontos's decision become real in the platform?". + +The rest of this document is about Delivery Mode and the Delivery Service. +For Delivery Method, see +[Data Product Delivery Methods](data-product-lifecycle.md#delivery-methods). + +### Why the Delivery Service exists {#why-delivery-service} + +Ontos is a system of record for a lot of *decisions*: who can access what, +which tags belong on which column, which roles carry which permissions, +which versions of which products are active. Those decisions only have value +when they land in the systems that actually enforce them — Unity Catalog, +external IT change-management, the customer's own GitOps pipeline. + +The **Delivery Service** is the layer that turns Ontos's decisions into +side-effects in those systems. It owns three orthogonal concerns: + +- **Where to land the change** — Direct (apply via SDK), Indirect (persist + as YAML in Git), Manual (notify a human). +- **What kind of change** — grant, revoke, tag assignment, entity + create/update/delete, role change. +- **Whether several modes run in parallel** — the configuration is a set, + not an exclusive choice. A site can run Direct + Indirect simultaneously + so that every direct UC change also produces a Git-trackable YAML + artifact. + +Customers ask about this layer using a few characteristic phrases — "we want +to maintain everything in Ontos and load it back to our workspace", "we +need indirect delivery via a volume so our deployment pipeline can pick it +up", "can Ontos write tags back to UC for us?". Those are all questions +about Delivery Mode. + +### The three modes {#the-three-modes} + +Modes are not mutually exclusive. Every configured mode runs for every +deliverable change, and the results are aggregated. If Direct succeeds but +Indirect fails, the aggregate is `any_success=True, all_success=False`, and +both outcomes are surfaced. + +#### Direct mode {#direct-mode} + +**What it does.** Applies the change immediately, in-process, by calling +the underlying system's API. For grants and revokes, that means the +service principal's workspace client issuing a UC `GRANT` / `REVOKE`. The +result is visible in the underlying system within seconds. + +**When to use it.** When Ontos has the authority to write directly (the +service principal holds the necessary privileges) and the customer wants +governance decisions to take effect without a human in the loop. + +**What the user sees.** Status flips on the entity (e.g., subscription +becomes Active) and the new state is observable in the target system on +the next read. + +**Dry-run support.** `DELIVERY_DIRECT_DRY_RUN=true` lets the mode log what +it *would* do without actually issuing the SDK call. Used when validating +a configuration before turning live changes on. + +**Current coverage in the shipping version.** Direct mode wires +`GRANT` and `REVOKE` end-to-end via `GrantManager`. Other change types +(tag assignments, entity create/update) currently no-op in Direct mode +with a `Change type not applicable for direct mode` marker and are +expected to land via Indirect or Manual — or, for the concept-to-UC-tag +path, via the `uc_tag_sync` workflow described below. + +#### Indirect mode {#indirect-mode} + +**What it does.** Serializes the change to a YAML file in the configured +Git repository (or volume — the `GitService` abstraction covers both). +A separate process — the customer's CI/CD pipeline, a workspace job, a +manual deploy — picks the YAML up and applies it. + +**When to use it.** When the customer has an existing change-management +discipline they don't want Ontos to bypass, when the workspace's SP +doesn't have direct write authority, or when an auditable trail of every +governance change in source control is mandatory. Customers who say +*"maintain everything in Ontos and load it back to the workspace via our +pipeline"* are asking for Indirect. + +**What the user sees.** The change is recorded in Ontos immediately +(databases get updated, the UI reflects the new state). The Git commit +shows up in the configured repo / volume location. The downstream UC +reality lags until the customer's pipeline runs. + +**File layout.** Each entity type has a registered `FileModel` that +controls subdirectory, filename, and YAML schema. The wrapper +(`wrap_as_resource`) gives every record a consistent envelope so the +target loader can identify what kind of resource it is. + +**Current coverage.** Indirect mode covers data contracts, data products, +data domains, roles, and tags — anything that has a `FileModel` in the +registry. Change types that don't have a file model fall back to a +generic timestamped YAML in `changes/` capturing the raw payload. + +#### Manual mode {#manual-mode} + +**What it does.** Creates a notification (and/or logs an entry) asking a +named human or group to perform the change in another system. Used when +there is no programmatic surface to drive the change automatically. + +**When to use it.** When the customer's enforcement system doesn't expose +an API — a legacy data catalog, an offline approval queue, or a downstream +system requiring human inspection. Also useful as a *belt-and-suspenders* +mode alongside Direct, when an admin wants visibility into every change +even if Ontos applies it automatically. + +**What the user sees.** A notification in the bell menu pointing to the +required action. The notification carries the title (e.g., *Grant Access: +DataProduct*) and a body describing principal, privileges, and target. + +**Current coverage.** The notification title/body templates exist for the +common change types. The DB-session-bound notification creation has a +TODO marker in the current implementation; admins should expect the entry +to appear in app logs and treat the in-UI notification surface as +evolving. + +### How concept → UC tag actually flows {#concept-to-uc-tag} + +This is the question customers ask most often, and it deserves an honest +answer because there are two paths and only one of them is fully wired. + +**The intended path (evolving).** A Steward writes a Semantic Link from +a column to a concept. The Delivery Service receives a `TAG_ASSIGN` +change and propagates it via the active modes — Direct (UC tags API), +Indirect (YAML manifest in Git), or Manual (notification). In the +shipping version, this path is partially wired: the change type exists, +the notification templates exist, but `TAG_ASSIGN` does not yet have a +fully-wired Direct handler that calls UC's tag API. Treat this as an +evolving area. + +**The path that ships today.** A Databricks job — the `uc_tag_sync` +workflow — reads `entity_semantic_links` joined with contracts, +products, domains, and assets, computes the desired UC tag set per +table, and issues `ALTER TABLE … SET TAGS (...)` statements via Spark +SQL. The job is installable from Settings → Background Jobs, runs on a +schedule (or on demand), and is the production path for concept-to-UC +sync on customer deployments today. It is independent of the Delivery +Service modes. + +In conversation: yes, semantic links *do* reach UC tags. The mechanism is +a job (not the Delivery Service direct mode). The Delivery Service path +for tag assignment is being filled in; the workflow path is what's +running in customer demos right now. + +### How agreements use Delivery {#agreement-integration} + +Approval workflows produce signed Agreements. The post-approval side +effect — actually granting UC access to the consumer — lives in the +`grant_permissions` step of the workflow definition, **not** in the +Delivery Service path described above. + +`grant_permissions` runs directly inside the workflow executor: it reads +the workflow execution context, uses the service principal's workspace +client, and issues UC `GRANT`s. The two paths converge conceptually +(both are about delivering a grant change to UC) but are wired +independently. The agreement workflow is the producer; the +`grant_permissions` step is the in-line deliverer. The Delivery Service +GRANT path is the alternative route for grants raised outside an +approval workflow. + +See [grant_permissions step](agreement-workflow.md#grant-permissions-step) +for the requirements (notably: the SP needs explicit `MANAGE` on each +securable, not `ALL_PRIVILEGES`). + +### What each persona sees {#per-persona} + +- **Admin** — Configures which modes are active (Settings → Delivery), + the dry-run flag, the Git repo connection, and the notification + targets for Manual mode. Audits the Delivery Service logs when a + customer pipeline gets out of sync. +- **Data Steward / Data Producer** — Sees the result of propagation + rather than the propagation itself. When they make a change in + Ontos, the UI reflects it immediately; the underlying-system + reality follows the mode. +- **Data Consumer** — Doesn't see Delivery directly. Sees the end + effect: their subscription becomes active, their UC `SELECT` + succeeds. + +## Under the hood + +### Change types {#change-types} + +Everything that goes through Delivery is one of these `DeliveryChangeType` +values, grouped by what they govern: + +| Group | Change types | +|---|---| +| **Access** | `GRANT`, `REVOKE` | +| **Tags** | `TAG_ASSIGN`, `TAG_REMOVE`, `TAG_CREATE`, `TAG_UPDATE`, `TAG_DELETE` | +| **Data entities** | `CONTRACT_CREATE` / `CONTRACT_UPDATE` / `CONTRACT_DELETE`, `PRODUCT_CREATE` / `PRODUCT_UPDATE` / `PRODUCT_DELETE`, `DOMAIN_CREATE` / `DOMAIN_UPDATE` / `DOMAIN_DELETE` | +| **Roles** | `ROLE_CREATE`, `ROLE_UPDATE`, `ROLE_DELETE` | + +Each change type is meaningful in each mode independently — `GRANT` in +Direct issues a UC GRANT; `GRANT` in Indirect writes a grant manifest to +Git; `GRANT` in Manual creates a notification reading "grant SELECT to +group-x on catalog.schema.table". + +Not every mode handles every change type yet. Direct currently +implements `GRANT`/`REVOKE` only; the other change types serialize via +Indirect or notify via Manual. Tag-assignment delivery to UC is described +in the next section because it travels via a different path today. + +## Cross-references {#cross-references} + +- [Delivery Method on a Deliverable](data-product-lifecycle.md#delivery-methods) — the *other* delivery concept +- [grant_permissions workflow step](agreement-workflow.md#grant-permissions-step) — agreement-driven UC grants +- [Semantic Link](ontology-and-knowledge-graph.md#three-tier-linking) — the source row a concept-to-UC-tag flow reads +- [Round-trip asymmetry](ontology-and-knowledge-graph.md#round-trip-asymmetry) — current state of concept → UC tag wiring + +## Common questions {#common-questions} + +**"We want to maintain everything in Ontos and load it back to the +workspace via our own deployment pipeline. Can Ontos do that?"** + +Yes. Turn on Indirect mode, point it at your Git repo (or a configured +volume), and Ontos serializes every governance change as YAML there. +Your pipeline reads those YAML files and applies them on its own +schedule. Direct mode can stay off if you want Ontos to be advisory +only, or on (alongside Indirect) if you want the same change to be +applied immediately *and* recorded in Git for audit. + +**"Do these modes conflict? If I enable Direct and Indirect, does the +change happen twice?"** + +The change happens once per mode but they are not duplicates of each +other — Direct calls UC, Indirect writes YAML. Together they give you +"applied" plus "auditable". Conflict only arises if your downstream +pipeline (consuming the Indirect YAML) and Direct mode both try to +write the same UC tag — and even then, idempotency on UC's side makes +the second write a no-op. The aggregated `DeliveryResults` object +captures success per mode independently. + +**"Tags reading from UC is not there — when will the round-trip be +complete?"** + +The forward path from a Semantic Link to a UC tag does ship today via +the `uc_tag_sync` workflow (see [concept → UC tag](#concept-to-uc-tag)). +The Delivery Service Direct mode for `TAG_ASSIGN` is partial in the +current version. The reverse path — pulling existing UC tags into Ontos +as concept assignments — is not yet shipping. Plan demos around the +forward direction with the workflow; flag the reverse direction as +evolving. + +**"Our SP doesn't have MANAGE on the catalog — what happens to Direct +mode grants?"** + +Direct mode raises an error from the underlying UC API. The +`DeliveryResults` records the failure on the Direct entry; if Indirect +is also enabled, the same change is still serialized to YAML there and +the customer's pipeline can apply it under a more-privileged identity. +This is one of the common reasons to enable Direct + Indirect together. + +**"Can I turn Direct off and run only Indirect?"** + +Yes. The active set is a settings toggle. With Direct off, Ontos becomes +a pure system-of-record — it persists what *should* be true, and a +downstream pipeline is responsible for making it true. A common pattern +in regulated environments. + +**"Where do role changes go?"** + +Through the Delivery Service. `ROLE_CREATE` / `ROLE_UPDATE` / +`ROLE_DELETE` are valid change types; Direct mode currently no-ops them +(role state lives in the Ontos DB, not in UC), Indirect serializes them +to Git, Manual notifies. Most customers run roles via Indirect so the +role definitions are auditable in source control alongside their other +governance manifests. + +**"What's the difference between Delivery Method = Table Access and +Delivery Mode = Direct?"** + +They answer different questions about different layers. *Table Access* +is the channel a consumer uses to read a product's data — they query UC +through whatever client. *Direct* is whether Ontos itself wrote the +grant that made that query possible synchronously via UC's API. A +deliverable can be Table Access (channel) while its access grant is +applied Indirect (mode) — those are independent choices. + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/end-to-end-flows.md b/docs/handbook/end-to-end-flows.md new file mode 100644 index 000000000..0fed816f7 --- /dev/null +++ b/docs/handbook/end-to-end-flows.md @@ -0,0 +1,426 @@ +# End-to-End Flows + +Ontos supports two narratives that customers walk in with. Most platforms +force one or the other. Ontos plumbs both, and the join point is the +semantic link table. + +- **Flow A — Bottom-up.** Start from Unity Catalog. Find a table that's + already producing value. Wrap it in a data product. Attach a contract. + Pin meaning to it. Publish. +- **Flow B — Top-down.** Start from a model of the business in your head. + Author an ontology that captures it. Map the concepts to real data + assets. Layer a glossary on top. + +This doc walks each flow step by step, calls out who does what at each +step, and points at the questions Ask Ontos can answer along the way. +Both flows produce semantic links — that's the join. + +## What you see in Ontos + +### Two flows, one bridge {#two-flows-one-bridge} + +The shared mechanism is a row in `entity_semantic_links`. A bottom-up +team adds those rows after they have data and need to express what it +means. A top-down team adds those rows after they have concepts and need +to ground them in real data. Same rows, opposite arrows. + +``` + Bottom-up Top-down + │ │ + ▼ ▼ + UC table / view Ontology / TTL upload + │ │ + ▼ ▼ + Data Product Knowledge graph context + │ │ + ▼ ▼ + Contract bound to Output Port Glossary collection + │ │ + └────────┬─────────────────────┘ + ▼ + entity_semantic_links + (the bridge — same table, + two directions of travel) +``` + +### Flow A — Bottom-up: UC → curated catalog {#flow-a-bottom-up} + +The canonical phrasing customers use: *"bring asset from UC to Ontos, +asset to product, product to contract, assign concepts"*. Eight steps. + +#### Step 0 — Set up domain, team, project {#step-a-0} + +**Who:** Admin or Data Governance Officer. + +**What:** Create a Data Domain (the business area — Sales, Supply Chain, +etc.), a Team (the durable group of users who own the work), and +optionally a Project (a bounded initiative under the team). + +**Why first:** Everything downstream — products, contracts, glossary +collections — scopes to a domain. Some access scoping (`Filtered` level) +keys off domain ownership too. Don't skip this even for small demos. + +**Ask Ontos at this stage:** "What domains exist?" "Show me teams in +domain X." + +#### Step 1 — Bring an asset from UC into Ontos {#step-a-1} + +**Who:** Data Producer or Data Engineer. + +**What:** Identify the UC table, view, or model that already exists and +produces value. Register it as an Asset in Ontos (Asset Explorer → +Create Asset with the appropriate type — PhysicalTable, PhysicalView, +Dataset). The asset carries a pointer back to the UC fully-qualified +name and inherits typing from the ontology-driven asset type system. + +**Why this step exists:** Ontos doesn't own the UC table. It owns a +governance record *about* the UC table. That record can carry custom +properties, tags, semantic links, costs, ratings — context the platform +itself doesn't provide. + +**Ask Ontos at this stage:** "Which UC table does this asset point at?" +"What's the meaning of this column?" + +#### Step 2 — Create the Data Product {#step-a-2} + +**Who:** Data Producer / Data Product Owner. + +**What:** Create a Data Product, give it a name, assign it to the domain +and owner team. Add **Deliverables** (output ports): each Deliverable +gets a name, a Delivery Method (Table Access / Serving Endpoint / File +Export / Streaming), and one or more linked UC assets. Add +**Consumables** (input ports): declared upstream dependencies, each +required to reference a contract version (per ODPS). + +**The vocabulary:** in conversation you'll hear *Deliverable* and +*Consumable*; in the persisted ODPS model these are *output_port* and +*input_port*. They're the same thing — the customer-facing names are the +primary ones. + +**Ask Ontos at this stage:** "What does this delivery method mean?" +"What status do I move the product to next?" + +#### Step 3 — Attach a Data Contract per Deliverable {#step-a-3} + +**Who:** Data Producer (drafting) and Data Steward (reviewing). + +**What:** Decide between **Products-First** (the product exists, +contracts come later per Deliverable) and **Contracts-First** (the +contract is drafted before assets are linked; the product is built to +satisfy the contract). Both supported, neither preferred — it's a +workflow choice driven by team culture, not a value judgment by the +platform. + +A contract is attached to a Deliverable by setting the Deliverable's +`contract_id`. That field may be NULL during early lifecycle stages — +deliberate, not an oversight. Input ports' contract references, by +contrast, are required. + +**Ask Ontos at this stage:** "Which output ports of my product still +have NULL contracts?" "Show me contracts in this domain in +under_review." + +#### Step 4 — Enrich the product {#step-a-4} + +**Who:** Data Producer + Data Steward. + +**What:** Add tags, business owners, custom properties, costs. Define +quality check definitions on the contract — pick dimensions, set +thresholds, choose business impact. Optionally kick off a DQX profiling +run to get *suggested* checks (see [data-quality.md](data-quality.md#dqx-flow)). +Document the product (purpose, usage, limitations, getting-started). + +**Why this matters:** This step is where the product becomes +*discoverable*. A product with no description, no quality, no owner is +indistinguishable from any other UC table. Enrichment is what justifies +the wrapper. + +**Ask Ontos at this stage:** "What quality dimensions am I covering on +this contract — am I missing accuracy on PII fields?" "Which products in +my domain don't have a documented owner?" + +#### Step 5 — Assign semantic terms / concepts {#step-a-5} + +**Who:** Data Steward, Data Engineer, or Business Analyst. + +**What:** Pin meaning. Open the Semantic Links panel on the product, +contract, or column. Search the glossary or the broader ontology. Pick +the concept that fits. The link is written to `entity_semantic_links`. +The three tiers — product, schema, property — are deliberate; pin at the +lowest level you can defend. + +**Why this matters:** This is where the data becomes addressable by +business meaning. Ask Ontos can now answer "products related to customer +churn" by walking the graph; the marketplace search ranks +semantically-linked products higher; agents grounded via MCP can find +the right table by concept, not by name. + +**Ask Ontos at this stage:** "Which products in my domain don't have a +glossary term?" "What concept is this column linked to, and why?" + +#### Step 6 — Steward review + certify + publish {#step-a-6} + +**Who:** Data Steward (certifies) + Data Product Owner (publishes). + +**What:** The producer submits the product for certification. The +Steward reviews against the contract, the quality, the semantic links, +the documentation. If approved, the product moves through `approved` to +`active` status when the owner publishes it. Publication scope +(`domain` / `organization` / `external`) controls who sees it in the +marketplace. + +Approval workflows (see +[agreement-workflow.md](agreement-workflow.md#approval-roles)) may gate +this step — the certification action can trigger an approval workflow +that captures sign-off from named business owners. + +**Ask Ontos at this stage:** "What changed in v2 compared to v1?" "Who +needs to approve this status change?" + +#### Step 7 — Consumer discovers + subscribes {#step-a-7} + +**Who:** Data Consumer. + +**What:** The consumer browses the marketplace or asks Ask Ontos a +natural-language question. They drill into a product, review the +contract, request access via the subscribe wizard. The subscribe wizard +is itself an approval workflow — it captures use case, on-behalf-of +principal (if subscribing for a team or service principal), and routes +to the Data Product Owner for approval. On approval, a +`grant_permissions` step issues real UC grants to the consumer +principals. + +**Ask Ontos at this stage:** "Where can I find a product about customer +churn?" "Who owns the daily-orders product?" "How do I request access?" + +### Flow B — Top-down: ontology → physical assets → UC tags {#flow-b-top-down} + +The mirror narrative. Five steps. The last one — propagating concept +assignments back to UC governance tags — ships today through a workflow +path, with a parallel Delivery Service path being filled in. + +#### Step 1 — Author the ontology externally {#step-b-1} + +**Who:** Knowledge Engineer / Data Architect. + +**What:** Open Protégé, TopBraid, or a text editor. Author an ontology +in OWL / Turtle / RDF/XML / N-Triples. Declare classes, data properties, +object properties, optional SHACL shapes for constraints. The +deliverable is a `.ttl` / `.owl` / `.rdf` / `.nt` file. + +Customers at this skill level talk about RDF, OWL, SHACL the way an +analytics engineer talks about Delta tables. They expect Ontos to handle +those formats natively — and Ontos does, via rdflib. + +**Alternative — LLM-assisted inference.** For teams without an ontology +yet, `OntologyGeneratorManager` (UI: `/concepts/generator`) points at UC +metadata and proposes concepts. Treat the output as a starting point +requiring human curation, not a finished ontology. + +**Ask Ontos at this stage:** Less useful here — the knowledge engineer +is writing TTL outside Ontos. Once it's uploaded, Ask Ontos becomes +useful for exploration. + +#### Step 2 — Upload + enable + visualize {#step-b-2} + +**Who:** Knowledge Engineer or Admin. + +**What:** Settings → Semantic Models → Upload. Pick the file. Enable it. +Refresh the runtime conjunctive graph +(`POST /api/semantic-models/refresh-graph`). Open the Knowledge Graph +view (`/concepts/home` → Graph tab) and explore. The graph is rendered +with Cytoscape; nodes are concepts, edges are object properties. + +**Why visualize:** Customers want to *see* the graph. The visual is what +makes ontology investment legible to non-technical stakeholders. +Identify gaps — clusters with sparse relationships, concepts that don't +connect to anything, missing domains. + +**Ask Ontos at this stage:** "What concepts does our ontology cover for +the Sales domain?" "Which concepts have no data mapping yet?" + +#### Step 3 — Map ontology concepts down to UC assets {#step-b-3} + +**Who:** Data Steward / Data Engineer. + +**What:** Navigate to a concept. Open the Linked Entities panel. +Search for products, contracts, schemas, columns that fit this concept. +Create semantic links. Repeat per concept. + +This is the same `entity_semantic_links` table the bottom-up flow +writes. The difference is direction of approach: top-down starts from +the concept and finds the data; bottom-up starts from the data and +finds the concept. + +Three tiers apply — link at product, contract-schema, or column level. +For ontology grounding, column-level is the gold standard; product-level +is the fallback. + +**Ask Ontos at this stage:** "Show me all products that satisfy concept +X." "What concept does this column embody?" + +#### Step 4 — Layer a Business Glossary on top {#step-b-4} + +**Who:** Data Steward / Business Analyst / Domain Expert. + +**What:** Create a glossary collection. Add concepts to it (these become +the displayable "terms"). Lifecycle each concept through draft → +published → certified. Decide the scope (domain-specific glossary vs +org-wide glossary). + +The relationship to Step 1: the concept IRIs the knowledge engineer +authored can be added to a glossary collection so the steward and +business users have a curated, browse-able view of the parts that matter +to them. The full ontology stays as the long tail; the glossary is the +short list. + +**Ask Ontos at this stage:** "What's the canonical definition of ARR in +our org?" "Show me certified terms in the Finance glossary." + +#### Step 5 — Push concept tags back to UC {#step-b-5} + +**Who:** Data Steward / Admin, with the UC tag-sync job installed. + +**What this does:** Translate concept-to-column semantic links into +Unity Catalog governance tags on the corresponding tables and columns, +so UC search, lineage, and access policies can leverage the concept. +This closes the round-trip from ontology → physical asset → UC. + +**How it ships today.** The `uc_tag_sync` workflow reads +`entity_semantic_links` joined with the contract, product, domain, and +asset metadata, computes the desired UC tag set per table, and issues +`ALTER TABLE … SET TAGS (...)` via Spark SQL. The job is installable +from Settings → Background Jobs and runs on schedule or on demand. It +is the production path for concept-to-UC sync on customer deployments +today. Tags propagate at table and schema granularity in the current +implementation; column-level tag propagation is evolving. + +**The parallel path via Delivery Service.** Separately, +[Delivery Service](delivery-and-propagation.md#concept-to-uc-tag) +defines `TAG_ASSIGN` as a first-class change type, with Direct / +Indirect / Manual modes. The Direct mode handler for tag changes +against UC is partially wired in the current version — the change +type, notification templates, and Indirect (Git) path exist; the +Direct mode call to UC's tag API is being filled in. Customers running +Indirect mode already see concept-driven tag manifests serialized to +Git as a by-product of every link write. + +**What's not yet shipping in either path:** the reverse direction — +reading existing UC tags and reflecting them back into Ontos as concept +assignments. The customer voice tracking this work captures it as +"Tags reading from UC is not there." Forward propagation works; the +backward pull is the part still evolving. + +**Ask Ontos at this stage:** "Has the latest concept-to-column link on +the X product been synced to UC yet?" "When was the uc_tag_sync job +last run?" "Show me UC tables tagged with concept Y." + +### Where the flows meet {#where-they-meet} + +Step 5 of Flow A and Step 3 of Flow B are the same action: writing a +row to `entity_semantic_links`. A team running both flows simultaneously +(common in mid-size enterprises with both a "let's catalog what we +have" effort and a "let's model what we want" effort) will see the same +table grow from both ends. + +The implication: it does not matter which flow you start with. What +matters is that linking happens at the lowest tier the team can +defend, that the same concept IRI is reused across links instead of +being re-created with subtly different IRIs, and that the glossary is +curated as a published subset of the broader ontology rather than as a +separate parallel artifact. + +## Under the hood + +The shared bridge between Flow A and Flow B is the +`entity_semantic_links` table; Step 5 of Flow A and Step 3 of Flow B +both write rows into it. The forward propagation from a semantic link +to a UC governance tag is implemented today by the `uc_tag_sync` +workflow, which joins `entity_semantic_links` against the +contract/product/domain/asset metadata and issues +`ALTER TABLE … SET TAGS (...)` via Spark SQL. The parallel +Delivery Service path defines `TAG_ASSIGN` as a first-class change +type — see [Delivery and Propagation](delivery-and-propagation.md#concept-to-uc-tag) +for the under-the-hood detail. + +## Common questions {#common-questions} + +**"We have UC already populated and no ontology. Where do we start?"** + +Flow A, Steps 0–4. Get a Domain, a Team, one product wrapping one +high-value UC table, a contract with a few quality checks, and one +semantic link to one concept (you can use the bundled ontology or add a +single concept to a glossary collection). Don't start by trying to +model the whole organization. + +**"We have an ontology authored in Protégé and no UC adoption yet. Can +we still use Ontos?"** + +Yes. Upload the ontology (Step B-1, B-2). Use the LLM-assisted +generator to *propose* asset structures from any UC presence you do +have, even partial. Use the marketplace as the place producers see +their concept coverage relative to what data exists. The bottom-up flow +will catch up as UC adoption grows. + +**"Do I have to pick one flow?"** + +No. The two flows are deliberately designed to share the same table. +Mid-size organizations almost always run both. The risk to manage is +glossary drift — two teams creating subtly different concepts for the +same thing. Address this by making the glossary curation a steward +responsibility, not a free-for-all. + +**"What's the difference between an output port and a Deliverable?"** + +Same thing. *Output port* is the persisted ODPS-spec label; *Deliverable* +is the customer-facing name. Same for *input port* and *Consumable*. +Use the customer-facing names in conversation; the ODPS names show up +in exports and in the persisted model. + +**"My consumers are subscribing but not getting access — what's +missing?"** + +Three usual suspects. (1) The subscribe wizard's approval workflow has +not been approved (it sits in `paused` waiting for the Data Product +Owner to respond). (2) The `grant_permissions` step ran but the app's +service principal does not hold `MANAGE` on the target UC securable +(`ALL_PRIVILEGES` is not sufficient). (3) The `consumer_principals` +list on the product points at a workspace-only group; UC accepts only +account-level groups. + +**"Can the consumer see the contract they're signing?"** + +Yes. The subscribe wizard shows the contract details (schema, quality, +SLAs, support channels) before the signer commits. The resulting +agreement record snapshots the contract version at sign-time, so later +contract edits don't retroactively change what the consumer agreed to. + +**"Tags reading from UC is not there. How do concepts in Ontos make it +back to UC governance tags?"** + +Today via the `uc_tag_sync` workflow — a job that reads +`entity_semantic_links` plus the contract/product/domain/asset +metadata and writes UC tags through Spark SQL `ALTER TABLE … SET TAGS`. +The Delivery Service is the parallel emerging path: it defines +`TAG_ASSIGN` as a first-class change type with Direct/Indirect/Manual +modes; the Direct UC handler is being filled in, the Indirect (Git +manifest) path already records every tag assignment. Plan demos around +the forward direction; the reverse direction (UC tags → Ontos concept +assignments) is the part still evolving. See +[Delivery and Propagation](delivery-and-propagation.md#concept-to-uc-tag). + +## Cross-references {#cross-references} + +- [Data Product](data-product-lifecycle.md#what-is-a-data-product) and + [Deliverable](data-product-lifecycle.md#output-port) +- [Data Contract](data-contract-lifecycle.md#what-is-a-contract) and + [Editor of record](data-contract-lifecycle.md#editor-of-record) +- [Ontology and the Knowledge Graph](ontology-and-knowledge-graph.md) +- [Delivery and Propagation](delivery-and-propagation.md) — how + governance changes (grants, tags, entity writes) reach UC and other + external systems +- [Data Quality DQX flow](data-quality.md#dqx-flow) +- [Approval Gate](agreement-workflow.md#approval-gates) and the + [grant_permissions step](agreement-workflow.md#grant-permissions-step) + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/entities-glossary.md b/docs/handbook/entities-glossary.md new file mode 100644 index 000000000..bfc311a3d --- /dev/null +++ b/docs/handbook/entities-glossary.md @@ -0,0 +1,241 @@ +# Entities Glossary + +One-paragraph definitions of every first-class entity in Ontos. Cross-links +go to the appropriate lifecycle, concept, or role document for the longer +story. + +## What you see in Ontos + +### Organizational hierarchy {#organizational} + +- **Domain** {#domain} — A business area that owns data assets (e.g., + "Sales", "Supply Chain"). Domains have an optional parent (`parent_id`) + to express a hierarchy. They are the primary scoping unit for the + `Filtered` access level and the typical anchor for data products. + Persisted in `data_domains`. +- **Team** {#team} — A durable group of users who collectively own work. + Stored in `teams` with `team_members` for membership. Teams can be + the owner of a data product (`owner_team_id`) or contract. A team + member may carry an `app_role_override` that elevates that user's + effective Ontos role for the lifetime of the membership. +- **Project** {#project} — A bounded initiative under a team or domain. + Projects scope data products and contracts (`project_id`). Workflows + can also be scoped to projects via `ScopeType.PROJECT`. + +### Data assets {#data-assets} + +- **Data Product** {#data-product} — A versioned, ODPS-conformant unit + of related assets exposed through Deliverables, owned by a team, + optionally bound to a project and domain. See + [data-product-lifecycle.md](data-product-lifecycle.md#what-is-a-data-product). +- **Data Contract** {#data-contract} — An ODCS v3.1.0 specification of + an interface — schema, quality, SLA, servers, support. A data + contract is attached to a data product via Deliverables (and + referenced by Consumables). See + [data-contract-lifecycle.md](data-contract-lifecycle.md#what-is-a-contract). +- **Deliverable / Output Port** {#output-port} — A consumable surface + of a data product. *Deliverable* is the customer-facing name; + *output port* is the ODPS-spec persisted-model label. May or may not + have a `contract_id` (NULL is allowed). May carry a delivery method + and an SBOM. See + [data-product-lifecycle.md](data-product-lifecycle.md#output-port). +- **Consumable / Input Port** {#input-port} — An upstream dependency + of a data product. *Consumable* is the customer-facing name; + *input port* is the ODPS-spec label. `contract_id` is required (ODPS + rule). See + [data-product-lifecycle.md](data-product-lifecycle.md#input-port). +- **Management Port** {#management-port} — An administrative endpoint + of a data product (discoverability, observability, control, + dictionary). +- **Delivery Method** {#delivery-method} — A configured way of + delivering data from a Deliverable: Table Access (UC SELECT), + Serving Endpoint (HTTP serving), File Export (volume / object store), + Streaming (Kafka / DLT). Configurable from Settings → Delivery + Methods. See + [data-product-lifecycle.md](data-product-lifecycle.md#delivery-methods). +- **Schema Object** {#schema-object} — A table-equivalent entry inside + a contract's schema. Carries logical/physical type, granularity, + tags, quality checks. See + [data-contract-lifecycle.md](data-contract-lifecycle.md#schema-objects). +- **Schema Property** {#schema-property} — A column-equivalent entry + under a schema object. Carries type, flags (`required`, `unique`, + `primary_key`, `partitioned`, `critical_data_element`), + transformation metadata, and may nest via `parent_property_id`. +- **Quality Check** {#quality-check} — A check **definition** attached + to a schema object or property. Not a result. Has a dimension, + severity, business impact, type (`library`/`text`/`sql`/`custom`), + and a family of declarative comparators. See + [data-contract-lifecycle.md](data-contract-lifecycle.md#quality-checks). +- **Quality Item** {#quality-item} — A *measurement* row scoped to an + entity (`data_product`, `data_contract`, `asset`, `data_domain`). + Carries `score_percent`, `checks_passed`, `checks_total`, + `measured_at`, `dimension`, and a `source` enum (`manual` / `dbt` / + `dqx` / `great_expectations` / `soda` / `external`). Rolled up via + `QualityManager.aggregate_for_product`. See + [data-quality.md](data-quality.md#measurements-and-rollup). +- **Asset** {#asset} — A persisted reference to a governed "thing" + (table, view, dashboard, model, etc.) stored in the database with a + typed `asset_type_id` and an optional `domain_id`. Distinct from the + connector-level `UnifiedAssetType` enum, which classifies + cross-platform asset categories at the integration boundary. +- **Asset Type** {#asset-type} — The persisted classification of an + asset, driven by the ontology TTL file at startup. Carries category, + icon, required/optional metadata schemas, allowed relationships. See + [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md#prescriptive-principle). + +### Agreements and workflows {#workflow-entities} + +- **Agreement** {#agreement} — The *immutable* record of a completed + approval workflow, including the workflow snapshot and per-step + results. Never modified after persistence. See + [agreement-workflow.md](agreement-workflow.md#what-is-an-agreement). +- **Approval Gate** {#approval-gate} — A first-class concept: a moment + in an entity's lifecycle where a configured approver must sign off + before the next state. Used at Contract Approval, Sandbox Ready, + Product Certified, Product Active. Implemented as an approval + workflow matched by trigger type. See + [agreement-workflow.md](agreement-workflow.md#approval-gates). +- **Workflow** {#workflow} — A *definition* with a trigger, scope, + type (`process` or `approval`), and an ordered list of steps. Stored + in `process_workflows` with `workflow_steps`. Different from a + Workflow Execution or a Wizard Session. +- **Workflow Execution** {#workflow-execution} — A single *runtime* + invocation of a workflow. Tracks status (`pending` / `running` / + `paused` / `succeeded` / `failed` / `cancelled`) and current step. + See + [agreement-workflow.md](agreement-workflow.md#execution-state-machine). +- **Workflow Step** {#workflow-step} — One node in a workflow, + identified by a slug `step_id`. Has a `step_type` (validation, + approval, user_action, webhook, grant_permissions, etc.), a JSON + `config`, and optional `on_pass` / `on_fail` branching targets. +- **Wizard Session** {#wizard-session} — The user-facing *in-flight* + state of an approval workflow. Holds the snapshotted workflow + definition (so later edits don't change what's being signed) and + per-step user inputs collected so far. Resolves to an Agreement on + completion. See + [agreement-workflow.md](agreement-workflow.md#three-concepts). + +### Semantics and tagging {#semantics-tagging} + +- **Ontology** {#ontology} — The *source artifact*: a `.ttl` / `.owl` + / `.rdf` / `.nt` file authored externally (Protégé, TopBraid, or a + text editor). Declares classes, data properties, object properties, + optional SHACL shapes. Distinct from the runtime knowledge graph + built from it. See + [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md#four-words). +- **Knowledge Graph** {#knowledge-graph} — The *runtime* structure: + an rdflib ConjunctiveGraph rebuilt from the union of enabled + ontologies, stored as triples in the `rdf_triples` table. Queried + via SPARQL at `/api/semantic-models/query`. See + [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md#runtime-graph). +- **Concept / Ontology Concept** {#ontology-concept} — A node in the + knowledge graph, identified by an IRI (e.g., + `https://example.com/ontology/Customer`). RDF-native; not a + separate table. Concepts have a status aligned with the unified + `EntityStatus`. Distinct from a *Glossary Term* (which is a UX + surface for a published concept). +- **Concept IRI** {#concept-iri} — The W3C-style identifier of a + concept. Used as the link target in semantic links. +- **Semantic Link** {#semantic-link} — A row in + `entity_semantic_links` that pins an entity (`data_domain` / + `data_product` / `data_contract` / `data_contract_schema` / + `dataset` / `asset` / `uc_catalog` / `uc_schema` / `uc_table` / + `uc_column`) to a concept IRI with an optional human-readable + label. Three-tier on contracts: contract-level, schema-level, + property-level. See + [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md#three-tier-linking). +- **Business Glossary Term / Glossary Term** {#glossary-term} — The + UX presentation of a published concept that has been added to a + glossary collection (`urn:glossary:` context). There is **no + separate glossary terms table** — a "term" is a concept plus its + glossary-collection membership. Distinct from the underlying + *concept*. See + [ontology-and-knowledge-graph.md](ontology-and-knowledge-graph.md#glossary-as-view). +- **Glossary Collection** {#glossary-collection} — A knowledge + collection with `collection_type=glossary`. Hosts the concepts + presented as terms. Has a name, description, domain scope, and + owner. +- **Tag** {#tag} — A typed label attached to an entity. Tags live + inside a namespace and may carry their own status and permissions. +- **Tag Namespace** {#tag-namespace} — A grouping of tags under a + name (e.g., `pii`, `domain`). Permissions can be configured per + namespace via `tag_namespace_permissions`. + +### Identity and access {#identity-access} + +- **User** {#user} — A caller identified by email and the groups + Ontos resolves for them at request time. See + [roles-and-rbac.md](roles-and-rbac.md#identity-resolution). +- **Role** {#role} — An Ontos role: a named bundle of feature → + access-level mappings, with a list of `assigned_groups`. The seeded + roles are Admin, Data Governance Officer, Data Steward, Data + Producer, Data Consumer, Security Officer. See + [roles-and-rbac.md](roles-and-rbac.md#built-in-roles). +- **Feature** {#feature} — A unit of UI/API surface (e.g., + `data-products`, `data-contracts`, `settings-roles`). Defined in + the `APP_FEATURES` map with a display name, allowed access levels, + and a sidebar group (`Discover`, `Build`, `Govern`, `Deploy`, + `Settings`, `Other`). +- **Permission** {#permission} — A `feature_id : access_level` pair + that gates a user's actions on a feature. See + [roles-and-rbac.md](roles-and-rbac.md#permission-model). +- **Access Level** {#access-level} — One of `None`, `Read-only`, + `Filtered`, `Read/Write`, `Full`, `Admin`. Applied per feature. See + [roles-and-rbac.md](roles-and-rbac.md#access-levels). +- **Consumer Principal** {#consumer-principal} — A typed identity + reference on a data product describing who may consume it. Default + type is `group`; other supported types include `service_principal`. + Resolved into UC grants by the `grant_permissions` step. See + [data-product-lifecycle.md](data-product-lifecycle.md#consumer-principals). +- **Workspace Admin** {#workspace-admin} — A user who is a member of + a Databricks group listed in `APP_ADMIN_DEFAULT_GROUPS`. Granted + admin treatment for cascade-bypass checks **independent of the + Ontos role system**. See + [roles-and-rbac.md](roles-and-rbac.md#workspace-admin-shortcut). +- **Ontos Admin** {#ontos-admin} — A user resolved (via group or + email-as-group fallback) to an Ontos role that holds the `Admin` + level on the `Admin` row (or on the relevant feature). Distinct + from Workspace Admin: a user can be one without the other. +- **Subscription** {#subscription} — A consumer's registration to a + data product, optionally on-behalf-of a group or service principal. + Drives ITSM notifications when the product or its bound contract + changes. + +### People (not seeded roles) {#people-personas} + +- **Knowledge Engineer / Data Architect** {#knowledge-engineer} — + Not a seeded Ontos role; the persona behind ontology authoring. + Authors the ontology in OWL/TTL/SHACL externally and loads it into + Ontos. Decides what the canonical concepts are, how they relate, + and where SHACL constraints live. Ontos uses their ontology to + drive asset types, ground Ask Ontos, and feed agents via MCP. See + [personas-quick-reference.md](personas-quick-reference.md#knowledge-engineer). + +### Other {#other-entities} + +- **Compliance Policy** {#compliance-policy} — A declarative DSL + rule that can be referenced by a workflow's `policy_check` step. + Compliance scoring derives from policy evaluations. +- **Certification Level** {#certification-level} — A configurable + ordinal scale separate from lifecycle status. Both products and + contracts carry a current level, inherited level, certified-by/at, + expiry, and notes. +- **Business Owner** {#business-owner} — A persisted business-side + owner of an asset, separate from the technical team owner. Used by + workflow approval steps that route to "business owners" for sign + off. In the current Ontos version, Business Owners and the + team-based Owner surface are increasingly presented through a + single consolidated Ownership panel on data product and contract + detail pages, with provenance shown for imported (ODCS/ODPS) + contacts and active business-owner records merged into YAML + exports for standards compliance. +- **Business Role** {#business-role} — A configurable business-side + role (e.g., "Head of Sales Analytics") that can be assigned to a + person and referenced by workflows. Distinct from Ontos's + authorization roles. + +## Under the hood + +This file is a flat glossary cross-cutting the rest of the corpus. Persisted-model detail (`*Db` class names, table names, column names) is interleaved with the user-facing label for each entry because the same row backs both the UI label and the underlying data model. Treat the bolded term as the user-facing name (the one to use in answers) and the `inline-code` references as developer-facing pointers into the schema. For schema source-of-truth, see `src/backend/src/db_models/` and the per-entity concept doc linked from each glossary entry. + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/installation-and-troubleshooting.md b/docs/handbook/installation-and-troubleshooting.md new file mode 100644 index 000000000..920c91fc7 --- /dev/null +++ b/docs/handbook/installation-and-troubleshooting.md @@ -0,0 +1,458 @@ +# Installation, Updates, Maintenance, and Common UI Errors + +This doc covers how Ontos gets into a workspace, how it gets updated, the +day-2 maintenance footwork it expects, and the UI errors users surface +most often. Step-by-step setup screenshots live in `docs/Ontos Setup.md`; +this file is the conceptual layer around it — what the choices mean, +where they trade off, and what breaks when something is misconfigured. + +## What you see in Ontos + +### Distribution channels {#distribution-channels} + +Ontos ships through two channels. They install the same application, +but they expect different operator skills and different update +discipline. + +#### Databricks Marketplace {#marketplace-channel} + +The default path. From the workspace's Marketplace UI, an admin searches +for "Ontos", clicks Install, and the workspace's Apps service handles +the rest — container build, OAuth scope acceptance, Lakebase wiring, +and first deploy. Each Marketplace listing version is a semver release +of the OSS repo packaged for distribution. Receiving an update means +re-installing or upgrading through the Marketplace UI; the running +deployment is replaced by the new version. + +Trade-off: governed and versioned. The workspace stays on a known +release; the in-product scope set is fixed in the listing's manifest +and can't be edited from the workspace side. If a customer needs a +different scope set or a pre-release fix, the Marketplace channel is +the wrong choice — go to the Git path. + +#### GitHub repository {#git-channel} + +`databrickslabs/ontos` is the canonical OSS repo. Advanced users clone +or fork it, then deploy via `databricks bundle deploy` / +`databricks sync` plus `databricks apps deploy`. This is the path for +running a feature branch, hot-patching an unmerged PR, or customizing +the app beyond what the Marketplace listing exposes. + +Trade-off: current-tip but unstable. The repo's `main` may carry +in-progress work; pinning to a tag is the operator's responsibility. +Scope changes can be applied directly to the deployed `manifest.yaml`, +but each scope edit forces affected users to re-accept (see +[OAuth scope changes](#oauth-scope-changes)). + +#### When to choose which {#channel-choice} + +For most customer deployments: Marketplace. It gives a clean semver +upgrade story and removes scope-cookie maintenance from the operator's +plate. For partners, OEMs, or any deployment that needs to fork — Git. +For temporary hot-patches against an unmerged upstream PR, Git is the +only option; see [Customer fork hygiene](#customer-fork-hygiene). + +### First-time installation {#first-install} + +`docs/Ontos Setup.md` walks the step-by-step. What matters +conceptually: + +#### Prerequisites {#install-prerequisites} + +- A **Lakebase Postgres** instance the deployment can claim. The + database (`app_ontos` by default) must be created with a permissive + initial grant (`GRANT ALL ON DATABASE app_ontos TO PUBLIC`) because + the app's service principal does not yet exist at create time. The + SP gets created when the app is first deployed and takes ownership + of the schema thereafter. +- A **Unity Catalog Volume** for artifacts (images attached to + entities, the Git-sync repo used by Indirect delivery mode). The SP + gets `READ_WRITE` on the volume after first deploy. +- **On-behalf-of (OBO) authentication enabled** on the workspace's + Apps service. OBO is in Public Preview at time of writing; the + toggle lives in workspace Previews. Without OBO, the app falls back + to SP-only auth and most lifecycle actions fail because they need + the calling user's permissions to grant securely. +- Optionally, a **Marketplace listing** in the workspace's regional + Marketplace feed. + +#### First-admin bootstrap {#first-admin-bootstrap} + +The very first user reaching a freshly seeded Ontos database hits a +chicken-and-egg problem: no role is yet bound to their groups. Two +mechanisms cover this: + +- **`APP_ADMIN_DEFAULT_GROUPS` env var** — read once on first start + by the role-seeding code in `SettingsManager`. The named groups are + written into the Admin role's `assigned_groups` list. This is a + **first-start-only** seeding: subsequent app restarts do not + re-merge changes to the env var. Post-deploy admin changes go + through Settings → RBAC or direct SQL. +- **Email-as-implicit-group fallback** — when SCIM returns an empty + group list for a user (typical on FEVM workspaces, or when the SP + lacks `iam.current-user:read`), the user's email is injected as a + synthetic group name so a direct `Admin.assigned_groups = ["alice@..."]` + entry can still match. The fallback is non-additive — it only + fires when SCIM returned zero groups, never alongside real groups. + Use it as a recovery escape hatch, not as a primary admin path. + +See [Roles and RBAC](roles-and-rbac.md#permission-model) for the full +permission model. + +#### Demo presets {#demo-presets} + +Ontos ships with five self-contained demo packs — `retail` (default), +`hls`, `fsi`, `mfg`, `auto`. Each is delivered as a standalone SQL +file under `src/backend/src/data/demo_data_{preset}.sql` and uses a +per-preset UUID prefix segment (`0000`, `0001`, `0002`, …). Loading is +through `POST /api/settings/demo-data/load?preset=`. Cleanup is +through `DELETE /api/settings/demo-data` and removes records across +all loaded packs by demo UUID prefix. There is no implicit base +overlay: each preset stands alone. + +### Updating to a new version {#updates} + +#### Marketplace upgrade {#marketplace-update} + +Re-install or upgrade the listing through the workspace UI. Databricks +handles the swap; the running deployment is replaced by the new +version. Persistent state (Lakebase tables, Volume artifacts, +configured roles) survives because it's external to the app +container. + +#### Git deploy update {#git-update} + +The expected sequence: + +1. `git pull` (or rebase) on the deployed branch. +2. `databricks sync` the contents of `src/` to the workspace deploy + path. **Sync from `src/`, not from the repo root** — the app's + `app.yaml` lives at `src/app.yaml` and must end up at the + workspace path's root. See + [workspace sync direction](#workspace-sync-direction). +3. If the update touches the handbook corpus + (`docs/handbook/*.md`), upload that tree separately via + `databricks workspace import-dir docs/handbook/ /docs/handbook/`. + `docs/` lives outside `src/` so the regular sync skips it. +4. `databricks apps deploy ` to roll the new deployment. + +#### Migration discipline {#migration-discipline} + +Alembic migrations are **append-only**. Once a revision ID has been +applied to any database, treat its body as frozen — add a new +migration on top rather than editing in place. Editing an already- +applied revision means alembic skips re-running it (because the +version table already records that ID), so the new code never +actually executes against the existing database. + +Revision IDs themselves have a hard ceiling: Postgres +`alembic_version.version_num` is `VARCHAR(32)`. A revision string +longer than 32 characters causes the version-table UPDATE to fail +with `StringDataRightTruncation`, which rolls back the whole +migration and stops the app from starting. Verify length before +upload. + +Single-head invariant: the migration history must have exactly one +head. CI enforces this via `scripts/check-alembic-heads.py`. If two +PRs land sibling revisions, the merging PR must include an explicit +`alembic merge`. + +#### Restart vs database state {#restart-vs-db-state} + +App restarts pick up code changes immediately. Database state +**persists across deploys, branch switches, and git reverts**. This +catches operators off-guard: rolling back a code commit does not +roll back the database row it touched. Use the dedicated reset +endpoint (`DELETE demo-data`) or direct SQL for state changes that +need to mirror a code revert. + +#### Customer fork hygiene {#customer-fork-hygiene} + +When a deployment runs hot-patches against unmerged upstream PRs, it +is functionally a fork. Track the delta explicitly (which PRs are +applied on top of what upstream commit), and reconcile once each +upstream PR merges. The risk to manage: an upstream reviewer asks +for changes during review, and the merged version diverges from what +the deployment already runs. Re-deploys after each upstream PR merge +keep the deployment converging on stock OSS. + +### Maintenance {#maintenance} + +#### Database migrations {#db-migrations-at-startup} + +`alembic upgrade head` runs at startup, before the FastAPI app +finishes initializing. A migration failure stops the app — health +endpoints return 200 (process is up) but `/api/health` JSON reports +`db_ok: false`. The Apps UI may show "running" while the app is +effectively broken; verify via the health JSON, not the badge. + +#### Role re-seeding {#role-reseeding} + +`APP_ADMIN_DEFAULT_GROUPS` seeds only on the first start that +encounters an empty role table. Later restarts do not re-merge. To +add admins post-deploy: edit `Admin.assigned_groups` in Settings → +RBAC, or update the row directly in Postgres. Don't expect env-var +edits to propagate to a running database. + +#### Workspace sync direction {#workspace-sync-direction} + +The sync source directory matters. `app.yaml` lives at `src/app.yaml` +in the repo, but the Apps service expects to find it at the +workspace deploy path's root. Sync from `src/`, not from the repo +root. + +A common failure: an operator runs `databricks sync` from the repo +root, so `app.yaml` lands at `/src/app.yaml` instead of +`/app.yaml`. Apps can't find the manifest, falls back +to defaults, and the deploy reports "App process did not start +within 10 minutes." This is a sync-layout error, not a code error, +and it is the dominant cause of that timeout message. + +`databricks sync` also adds without removing. Re-syncing from the +correct directory does not delete the wrong-layout files left from +a prior bad sync — both sets coexist in the workspace until the +operator deletes the stale paths explicitly. Verify the workspace +layout before deploying. + +#### OAuth scope changes {#oauth-scope-changes} + +When the deployment's required OAuth scopes change — for example, +adding `unity-catalog` to the manifest — already-authorized users +do **not** automatically pick up the new scope set. The first-visit +scope-acceptance is stored in the user's browser cookie for the +app's URL and re-used until the cookie is cleared. The visible +symptom is an error like +`"Provided OAuth token does not have required scopes: unity-catalog"` +even though the manifest lists the scope correctly. + +The fix is per-user: clear the cookie for the app's URL and reload +the page. The user is re-prompted to accept the current scope set, +the new cookie carries the updated scopes, and the error goes away. +There is no admin-side override that can force a re-prompt on every +user — communicate the cookie-clear step when shipping a scope +change. + +#### Customer fork delta {#customer-fork-delta} + +For deployments carrying unmerged upstream patches, maintain a +delta document listing each applied PR and its upstream status. +Rebase once any PR merges into upstream. Don't accumulate divergent +patches without a reconciliation plan; the longer the delta lives, +the more expensive each reconciliation deploy becomes. + +### Common UI errors {#common-ui-errors} + +What follows is the recurring set of user-visible errors with their +root causes. Each entry covers symptom, cause, and fix. + +#### Identity and access errors {#identity-errors} + +##### "Request role" prompt on every page load {#request-role-prompt} + +The user lands on Ontos and is asked to pick a role on every visit, +because the role-resolution step found no matching `assigned_groups` +intersection. + +Two common causes: + +- The app SP can't read the user's groups via SCIM (often because + `iam.current-user:read` is missing from the manifest scope set, or + the workspace returns empty group lists). Fix: verify the SCIM + scope; if the workspace doesn't support SCIM group reads for SPs, + fall back to email-as-implicit-group by adding the user's email + directly to `Admin.assigned_groups` and confirming the SCIM call + returns empty (the fallback only fires when SCIM returns zero + groups). +- The `APP_ADMIN_DEFAULT_GROUPS` env var wasn't set on first start, + so the Admin role was seeded with an empty group list. Fix: edit + Settings → RBAC manually, or rewrite `Admin.assigned_groups` via + SQL. + +##### 403 on data products or contracts the user should be able to see {#unexpected-403} + +The user has the documented feature-level access but a specific +endpoint returns 403. Endpoints can apply stricter sub-gates than +the feature-level access in the matrix; the matrix describes the +front-of-feature expectation, not every endpoint's authorization. +Trust the 403 — read the endpoint's actual `PermissionChecker` +dependency. If the gate is wrong for the user persona, the gate +needs to be lowered; the fix is a code change, not a permission +change on the user's side. See +[Roles and RBAC — permission model](roles-and-rbac.md#permission-model). + +##### "Unity Catalog scope missing" {#uc-scope-missing} + +The browser's cached scope-accept cookie predates a recent scope +change. The new scope is in the manifest, but the user's session +token doesn't carry it. Fix: clear the cookie for the app's URL, +reload, and re-accept the prompt. See +[OAuth scope changes](#oauth-scope-changes). + +#### Workflow and approval errors {#workflow-errors} + +##### "Cannot approve agreement" for a non-Admin reviewer {#cannot-approve} + +A Business Owner or other non-Admin reviewer can't approve an +agreement they're listed on. Two causes recur: + +- **Approver-role filter mismatch.** The agreement's approval + workflow definition filters approvers by role, and the user's + current Ontos role doesn't match. Fix: check the workflow's + approver-role spec and the user's role assignment. +- **Outer permission gate.** The approval-handling endpoint may + require a feature access level the reviewer doesn't have at the + configured level (historically `notifications:READ_WRITE` or + `data-contracts:READ_WRITE`). Fix: align the user's role with the + endpoint's gate, or relax the gate in the deployment's role + config. + +##### "grant_permissions step failed" {#grant-permissions-failed} + +The `grant_permissions` step in an agreement workflow tries to grant +UC permissions and the platform rejects the call with +`"User does not have MANAGE"`. The app SP needs **explicit MANAGE** +on each UC securable it grants on. `ALL_PRIVILEGES` is **not** +sufficient on AWS Unity Catalog — MANAGE must be granted separately. +Fix: `GRANT MANAGE ON CATALOG TO ` for +each catalog the workflow touches. Document this in the deployment +runbook; it bites every customer that uses `grant_permissions` in +production. See +[Agreement workflow — grant_permissions step](agreement-workflow.md#grant-permissions-step). + +#### Database and data errors {#database-errors} + +##### "Alembic version too long" or startup hang {#alembic-version-too-long} + +The app fails to start, logs show `StringDataRightTruncation` on +`alembic_version`. A new migration's revision ID exceeded +`VARCHAR(32)`. Fix: rename the revision in the migration file to a +short identifier (≤32 chars), redeploy. If the database has already +recorded the old long ID, update `alembic_version.version_num` +manually via SQL to the new value before deploying. + +##### Lakebase autoscale not picking up {#lakebase-autoscale-stuck} + +The app reports it can't connect to Lakebase, but the Lakebase +instance shows healthy. The autoscale signal sometimes sticks after +a long idle. Fix: pause and resume the Lakebase instance from the +UI. The app retries on next request. + +##### Stale data in product detail page after a code revert {#stale-data-after-revert} + +The operator reverted a code commit but the bug behavior persists. +Database state is independent of git state — reverting code does +not roll back rows. Fix: either re-apply a corrective migration, or +update the relevant rows manually via SQL. See +[restart vs database state](#restart-vs-db-state). + +#### Deploy and app process errors {#deploy-errors} + +##### "App process did not start within 10 minutes" {#process-did-not-start} + +The most common cause is a workspace sync layout bug: `app.yaml` +landed at the wrong path because the sync was run from the repo +root instead of `src/`. Fix: verify `databricks workspace list` +shows `app.yaml` at the deploy path's root; if it sits one level +deeper, re-sync from `src/`. Also check for missing required env +vars in `app.yaml` — a missing env binding can stall the container +the same way. + +##### Ask Ontos returns "I don't have authoritative information" for everything conceptual {#corpus-not-found} + +The `docs/handbook/` tree wasn't packaged into the deployment. The +`search_ontos_handbook` tool can't find the corpus on disk, so +every conceptual query returns the refusal default. Fix: upload +`docs/handbook/` to the deployment path +(`databricks workspace import-dir docs/handbook/ /docs/handbook/`), +or set `ONTOS_HANDBOOK_DIR` to point at an alternate corpus +location on the container. Restart the app to pick up the new +files. + +### Where to get help {#getting-help} + +- **Bugs and feature requests:** open issues at the + `databrickslabs/ontos` GitHub repository. +- **Contributing:** see `CONTRIBUTING.md` in the repo for the PR + workflow, testing expectations, and code-style conventions. +- **Customer support:** Marketplace deployments are supported + through the workspace's Ontos administrator; Git-channel + deployments take support questions directly to GitHub. + +## Under the hood + +This whole doc speaks to deployment operators, so the line between user-facing and internal is fuzzier than in other concept docs. The deepest internals — the `alembic_version.version_num` `VARCHAR(32)` ceiling, the `alembic upgrade head` startup hook, the workspace sync layout invariant — are called out inline above where they matter. For source-level detail, see `src/backend/alembic/`, `scripts/check-alembic-heads.py`, `src/backend/src/utils/startup_tasks.py`, and the deployment runbooks under `docs/`. + +## Cross-references {#cross-references} + +- [Roles and RBAC — first-admin bootstrap and permission model](roles-and-rbac.md#permission-model) +- [Agreement workflow — grant_permissions step](agreement-workflow.md#grant-permissions-step) +- [Delivery and propagation — Indirect mode requires the Volume-backed Git repo](delivery-and-propagation.md#indirect-mode) +- [MCP and Ask Ontos — what the copilot grounds against, and how the corpus is consumed](mcp-and-ask-ontos.md#grounding-sources) + +## Common questions {#common-questions} + +**"What's the difference between installing Ontos from Marketplace and from the GitHub repo?"** + +Marketplace gives a packaged, versioned, governed install — the +workspace receives a known release, scope acceptance happens once +per user, and upgrades happen through the Marketplace UI. +GitHub-repo install gives current-tip code with the ability to +customize the manifest, fork the code, or run an unmerged feature +branch. Marketplace is the default for production deployments; +Git is the right choice for partner forks, hot-patches, or +deployments that need a custom scope set. See +[Distribution channels](#distribution-channels). + +**"How do I update Ontos to a new version?"** + +If the deployment came from Marketplace, re-install or upgrade +through the workspace's Marketplace UI; the running deployment is +replaced and Lakebase state survives. If the deployment came from +Git, `git pull` the deployed branch, `databricks sync` `src/` to +the workspace, run `databricks apps deploy`, and (if the handbook +corpus changed) upload `docs/handbook/` separately. See +[Updating to a new version](#updates). + +**"The app says my Unity Catalog token is missing a scope, but the manifest looks correct. What's wrong?"** + +The user's scope-accept cookie predates the manifest update. +Clear the cookie for the app's URL, reload, and re-accept the +prompt. Communicate this step to all affected users when shipping +a scope change. See [OAuth scope changes](#oauth-scope-changes). + +**"The app process didn't start within 10 minutes. Is my code broken?"** + +Usually not. The dominant cause of that error is a workspace sync +layout bug — `app.yaml` is at the wrong path because the sync was +run from the repo root instead of `src/`. Verify the workspace +layout with `databricks workspace list` and re-sync from `src/` if +`app.yaml` is one level too deep. See +[workspace sync direction](#workspace-sync-direction). + +**"How do I add a new admin after the app is already deployed?"** + +Don't rely on editing `APP_ADMIN_DEFAULT_GROUPS` — that env var is +read only on the first start that encounters an empty role table. +Edit `Admin.assigned_groups` through Settings → RBAC, or update +the row directly in Postgres. See +[Role re-seeding](#role-reseeding). + +**"I reverted a buggy commit and redeployed, but the bug is still there. Why?"** + +Database state persists across deploys and git reverts. The code +revert removed the buggy code path; it did not undo the database +rows that path wrote. Apply a corrective migration or update the +rows manually. See +[restart vs database state](#restart-vs-db-state). + +**"Why is my `grant_permissions` step failing with 'User does not have MANAGE'?"** + +The app's service principal needs explicit `MANAGE` on each UC +securable the workflow grants on. `ALL_PRIVILEGES` is not +sufficient — MANAGE must be granted separately. Add a UC +pre-flight to the deployment runbook: +`GRANT MANAGE ON CATALOG TO ` for every +catalog the workflow touches. See +[grant_permissions step failed](#grant-permissions-failed). + +_Last verified against codebase: 2026-05-29_ diff --git a/docs/handbook/mcp-and-ask-ontos.md b/docs/handbook/mcp-and-ask-ontos.md new file mode 100644 index 000000000..a74ae392c --- /dev/null +++ b/docs/handbook/mcp-and-ask-ontos.md @@ -0,0 +1,210 @@ +# Ask Ontos and the MCP Server + +Ontos exposes its tool catalog through two surfaces — the **in-product +copilot** and an **MCP server endpoint**. They share the same underlying +tools but are accessed by different clients with different rules. Most +customer questions conflate the two; this doc separates them. + +## What you see in Ontos + +### Ask Ontos — the in-product copilot {#what-is-ask-ontos} + +Ask Ontos is the chat panel you can open from any page. It's a slim LLM +client that lets a logged-in user ask natural-language questions, run +small admin actions where they're allowed, and search the catalog without +having to navigate to the right page first. + +It is meant to feel like talking to a co-worker who knows the platform +well — the user can ask "show me products in my domain that have no +contract yet" or "what does the `arr_usd` column on the daily-revenue +contract mean?" and get a grounded answer. + +### What grounds it {#grounding-sources} + +Two complementary sources keep Ask Ontos honest: + +- **This handbook corpus.** Conceptual questions (lifecycle states, + permissions, what a Deliverable is) ground against the docs in + `docs/handbook/` — the same files this one belongs to. The retrieval + layer finds anchor-tagged sections and gives the model citable + fragments. If a question can't be grounded, the assistant should + decline rather than guess. +- **The tool registry.** Live data (counts, names, IDs, statuses) comes + from calling tools — same tools the MCP server exposes. The + assistant doesn't compose database numbers from the prompt; it asks + the tool and reports back. + +The discipline matters: a copilot that hallucinates lifecycle states or +permission rules erodes trust faster than one that says "I'm not sure". + +### Tools it has access to {#tool-categories} + +The full registry is the source of truth (over forty tools at last +count); a high-level grouping helps: + +- **Search & discovery** — full-text search across products, contracts, + glossary, reviews; semantic-link lookup; concept browsing. +- **Data product CRUD** — read product details, list deliverables, walk + consumer principals. +- **Data contract CRUD** — fetch schemas, quality definitions, version + history. +- **Semantic models** — list concepts, query SPARQL, neighbours, add / + remove semantic links. +- **Tagging** — list tags, list assignments, assign/remove (subject to + permission). +- **Analytics queries** — basic counts and lifecycle breakdowns for + dashboard surfaces. +- **Workflows** — list workflow definitions, look up executions and + agreements. + +If you need the exact list, the registry's `to_mcp_format()` returns the +canonical schema for every tool, and the in-product Settings panel +lists tools the assistant can use. + +### Permissions and impersonation {#permissions} + +Ask Ontos runs **as the calling user**, on-behalf-of. It has exactly the +authority the logged-in caller already has — no more. A Data Consumer +asking it to publish a data product will get the same `403` they'd get +if they hit the publish endpoint directly. + +This is the right default: the assistant is a productivity layer, not a +permission-escalation path. If a user can't do an action manually, the +assistant declines and explains why. See +[Roles and RBAC](roles-and-rbac.md#permission-model). + +### Personalization {#personalization} + +In the current Ontos version, the assistant adapts to two pieces of +caller context: the user's role (so an admin sees admin-specific +suggestions where a consumer sees marketplace-discovery suggestions), +and the page the user is on (so an answer on a contract detail page is +scoped to that contract). Deeper personalization — mode-aware starter +prompts, page-specific tool selection — is evolving. + +### What it deliberately won't do {#wont-do} + +- Run write queries against UC data tables. Read-only by default. +- Delete data products, contracts, or other governed entities through + the chat surface, even when the caller has Admin. Destructive + operations require the explicit UI confirmation flow. +- Operate outside Ontos — no Slack messaging, no email composition, no + Git pushes initiated by the assistant. Those live in other layers. +- Bypass the citation discipline. If a conceptual claim can't be + anchored against the corpus or a tool result, the assistant should + decline. + +Hallucination risk is real and persistent on this kind of surface. The +mitigation is grounding plus a refusal default — when a query can't be +satisfied with confidence, the right behavior is "I don't know" rather +than a fabricated answer. + +### MCP Server — Ontos as a tool catalog for external agents {#mcp-server} + +Separately from the in-product copilot, Ontos exposes a **Model Context +Protocol** (MCP) server at `/api/mcp`. This is a JSON-RPC 2.0 endpoint +that external clients — Claude Code, Cursor, custom agent frameworks — +can connect to. The server publishes the same tool catalog Ask Ontos +uses, but with a different authentication scheme and a different +permission model. + +#### Why two surfaces {#two-surfaces} + +The in-product copilot is convenient for an authenticated user who's +already in Ontos. The MCP server is for cases where an agent running +*outside* Ontos needs to read or act on Ontos's data — a Claude Code +session helping a data engineer write a transformation, a custom agent +generating documentation, a Cursor IDE pulling product metadata into +the editor. + +Same tool registry, different transport, different authentication: + +| | Ask Ontos (in-product) | MCP Server (`/api/mcp`) | +|---|---|---| +| **Caller** | A logged-in Ontos user | An external agent / IDE | +| **Auth** | Browser session + OBO | MCP token (DB-stored, scope-tagged) | +| **Acts as** | The logged-in user (OBO) | The token's bound principal | +| **Permissions** | Inherits the caller's Ontos role | Constrained by the token's scopes | +| **Transport** | App UI calls backend | JSON-RPC 2.0 over HTTP (with SSE option) | + +#### Tokens and scopes {#mcp-tokens} + +External clients authenticate with a token — created from Settings → MCP +Tokens, shown once on creation, then hashed in the DB. Each token +carries a list of scopes (`data-products:read`, `sparql:query`, etc.) +that further restrict what tools it can call. A token with no scope for +a tool can't invoke it even if the bound user has the Ontos permission. + +Admins manage the token store. The MCP feature is gated separately from +ordinary settings access — a user with `settings:READ_WRITE` does not +implicitly get MCP token management. + +#### What MCP exposes {#mcp-exposes} + +The same tool catalog as Ask Ontos, surfaced through the MCP protocol's +`tools/list` and `tools/call` methods. Each tool's input schema is the +canonical JSON-schema definition from `tool.to_mcp_format()`. The MCP +endpoint also implements the protocol's `initialize`, `ping`, and +notification semantics; SSE transport is available for streaming agent +sessions. + +#### What MCP isn't {#mcp-isnt} + +The Ontos MCP server is a *server* (it exposes Ontos's tools so +external agents can call them). It is **not** a *client* of other MCP +servers. Ontos does not currently consume third-party MCP catalogs at +runtime to ground its own answers; for that, the in-product copilot +relies on its concept corpus and tool registry as described above. + +## Under the hood + +Implementation details (JSON-RPC 2.0 transport, `tool.to_mcp_format()` schemas, SSE streaming, token-hashing storage, `SPARQLQueryValidator`) are documented in the MCP routes and Ask Ontos manager source. See `src/backend/src/routes/mcp_routes.py`, `src/backend/src/controller/llm_search_manager.py`, and `src/backend/src/controller/mcp_token_manager.py` for the canonical wiring. + +## Cross-references {#cross-references} + +- [Roles and RBAC — permission model](roles-and-rbac.md#permission-model) +- [Semantic Link — what the assistant reads when answering a "what does this column mean?" question](ontology-and-knowledge-graph.md#three-tier-linking) +- [Personas — what each role typically asks](personas-quick-reference.md) + +## Common questions {#common-questions} + +**"Can Ask Ontos delete a data product or revoke someone's access?"** + +Destructive actions are gated. The assistant won't run a delete or a +revoke from the chat surface even when the caller has Admin permission +— those actions require the explicit confirmation flow in the UI. Read +operations, lifecycle promotions with the caller's normal permissions, +and small CRUD operations are in scope. + +**"Why did it refuse to answer a question I think it should know?"** + +Two common reasons: (1) the conceptual claim couldn't be anchored +against the corpus, and the assistant is configured to decline rather +than guess; (2) the caller doesn't have the permission to read the +underlying data — the assistant won't surface a number the caller +couldn't see directly. Both are intentional. If a refusal feels wrong, +check the user's role and the corpus coverage for the topic. + +**"Is Ask Ontos calling my own LLM provider or a Databricks-hosted +model?"** + +Either, depending on configuration. The model and endpoint live in +Settings → LLM and can point at a Databricks Foundation Model serving +endpoint, an external provider, or a custom deployment. The +configuration is Admin-controlled; the assistant doesn't expose the +endpoint to end users. + +**"Can my Claude Code session in the IDE use Ontos's tools?"** + +Yes — that's what the MCP server is for. Create an MCP token in +Settings → MCP Tokens with the scopes your session needs, register the +Ontos MCP endpoint in your client's config, and your IDE agent will see +the same tools Ask Ontos uses. + +**"If I create an MCP token, does that token bypass Ontos permissions?"** + +No. The token is bound to a principal, and the principal's Ontos +permissions still apply. The token's scopes are an *additional* restrict +— they narrow what the token can do, never widen it. + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/ontology-and-knowledge-graph.md b/docs/handbook/ontology-and-knowledge-graph.md new file mode 100644 index 000000000..08e03c1a2 --- /dev/null +++ b/docs/handbook/ontology-and-knowledge-graph.md @@ -0,0 +1,314 @@ +# Ontology and the Knowledge Graph + +Most platforms that try to do "business glossary" land on a flat list of terms. +Ontos treats meaning as a graph. The pieces are an **ontology** (the source +artifact your knowledge engineer authors), a **knowledge graph** (the runtime +graph Ontos keeps in memory and in the DB), **semantic links** (the pins from +real Ontos objects to graph nodes), and a **business glossary** (a view onto +published concepts that non-technical users actually browse). They are all the +same plumbing under different labels. + +Customers asking these questions tend to be highly skilled — they say "RDF", +"OWL", "SHACL" the way a database engineer says "primary key". This document +is written for that audience first, then translated for everyone else. + +## What you see in Ontos + +### Four words that get confused {#four-words} + +- **Ontology** — the source artifact. A `.ttl` / `.owl` / `.rdf` / `.nt` file + authored externally (Protégé, TopBraid, any text editor). Declares classes, + data properties, object properties, and constraints (SHACL shapes). It is a + file you can put in version control. +- **Knowledge graph** — the runtime structure. An rdflib `ConjunctiveGraph` + rebuilt from the union of enabled ontologies, stored as triples in the + `rdf_triples` table, in memory at request time. You query it with SPARQL. +- **Semantic link** — a pin. One row in `entity_semantic_links` that says + "this Ontos object (a data product, contract, schema, column, …) means this + concept (this IRI)". It is the load-bearing bridge from RDF-land to data + product-land. +- **Business glossary term** — a published concept presented for browsing. + There is **no separate glossary terms table**. A "term" is the UX surface + of a concept that lives in a `urn:glossary:` collection. If you delete the + glossary collection, the underlying concept survives; if you delete the + concept, the term is gone. + +These four words sit on a "semantic maturity ladder" — Controlled +Vocabulary (a flat business glossary) → Taxonomy (a hierarchy) → Ontology +(taxonomy + typed relationships) → Knowledge Graph (ontology + instances). +Each layer adds expressivity on top of the previous one. Ontos is built +for the full ladder but lets customers enter at whichever level their team +is ready for. + +Anti-pattern phrasing: "let me add a term and link it to a column". What you +are actually doing is **adding a concept** to a glossary collection and then +**writing a semantic link** from the column to that concept. The distinction +matters because the same concept can be linked from many places, and a +concept can exist without ever being published as a glossary term. + +### The "ontology is prescriptive" principle {#prescriptive-principle} + +Before the redesign, the ontology was decorative — Ontos loaded the TTL but +nothing in the application changed when you edited it. That is no longer +true. The current Ontos version makes the ontology **the source of truth for +the asset type system**: + +- `ontos-ontology.ttl` is parsed at startup. +- For every class annotated with `ontos:modelTier "asset"`, an `AssetTypeDb` + row is created or updated. That row carries the UI icon, the category, the + persona visibility, the required-fields/optional-fields JSON schema, the + allowed incoming/outgoing relationships. +- The frontend's Asset Explorer reads asset types from the API, not from a + hardcoded list. Adding a new entity type to your knowledge model is an + ontology edit, not a code change. +- The ODCS ontology (`odcs-ontology.ttl`) and the Databricks platform + ontology (`databricks_ontology.ttl`) ship alongside and provide reusable + vocabulary for contract schemas and physical assets. + +If you change the ontology and re-sync, downstream things shift: the form +that renders when you create an asset changes, the relationship types +offered in the relationship panel change, the asset sidebar changes. That +gives the knowledge engineer a real lever over the platform. + +### Bundled taxonomies {#bundled-taxonomies} + +Three ontologies ship out of the box under +`src/backend/src/data/taxonomies/`: + +- **ontos-ontology.ttl** — the application's own vocabulary: `DataDomain`, + `DataProduct`, `Team`, `Project`, `Asset` subclasses (Dataset, + PhysicalTable, PhysicalView, PhysicalColumn, Policy, BusinessTerm, + Dashboard, APIEndpoint, Notebook, MLModel, Stream, System), and the + relationship properties (`hasDataset`, `governedBy`, `hasTable`, + `hasColumn`, `attachedPolicy`, etc.). UI annotations + (`ontos:uiIcon`, `ontos:uiCategory`, `ontos:modelTier`, + `ontos:uiPersonaVisibility`) live here too. +- **odcs-ontology.ttl** — the Open Data Contract Standard vocabulary + (`DataContract`, `SchemaObject`, `SchemaProperty`, `Server`, + `QualityRule`, etc.). Bridges contract semantics with the asset model. +- **databricks_ontology.ttl** — physical-platform vocabulary: + `Catalog`, `Schema`, `Table`, `View`, `Column`. Useful as anchor concepts + when you semantic-link Ontos entities to UC objects. + +These are synced into the DB at startup via `SemanticModelsManager`. You can +disable any of them from Settings → Semantic Models; they will keep their +DB rows but stop contributing triples to the runtime graph. + +### Authoring and uploading an ontology {#authoring} + +The recommended path for a serious knowledge engineering effort: + +1. Author the ontology externally in OWL / Turtle / RDF/XML / N-Triples. + Protégé, TopBraid Composer, or just a text editor will do. Include SHACL + shapes for constraints if you need them. +2. Open Settings → Semantic Models. +3. Upload the file. Ontos accepts `.ttl`, `.owl`, `.rdf`, `.nt`. The format + is auto-detected by rdflib. +4. Ontos parses, validates the triple count is non-zero, and persists the + model to `semantic_models`. Triples are stored under a URN context + (e.g., `urn:semantic-model:my-org-customer-ontology`). +5. Enable the model so its triples join the runtime conjunctive graph. +6. Trigger a graph rebuild (Settings → Semantic Models → Refresh, or + `POST /api/semantic-models/refresh-graph`). +7. The model is now queryable via SPARQL, visible in the graph view, and + available to Ask Ontos and MCP agents. + +### LLM-assisted inference (a starting point, not an ontology) {#inference} + +`OntologyGeneratorManager` is the LLM-assisted alternative for teams that +don't have an ontology yet. It points at UC metadata (table names, column +names, column comments, tags, descriptions) and proposes concepts, +properties, and relationships. Available at `/concepts/generator` and from +inside the contract detail page via the infer-from-catalog dialog. + +Generation runs in the background; the UI shows progress, and the +caller can navigate away and return without losing the run. Recent runs +are persisted so they survive a server restart, and admins can see +runs across users. Cancel mid-flight, delete completed runs, save any +completed run into a collection. + +Set expectations explicitly with the customer: this is **a starting point +that requires human curation, not a finished ontology**. Quality depends on +the richness of the source metadata. The output is OWL/Turtle the user +reviews, edits, and accepts before it is persisted as a semantic model. + +### Semantic links — the three-tier story {#three-tier-linking} + +This is what makes Ontos different from a flat glossary tool. A semantic +link is a row in `entity_semantic_links` that pins one Ontos entity to one +concept IRI, with an optional human-readable label and optional context +notes. + +`SemanticLinksManager` writes rows. The supported `entity_type` values are: + +- `data_domain` +- `data_product` +- `data_contract` +- `data_contract_schema` — a schema object inside a contract +- `dataset` +- `asset` +- `uc_catalog`, `uc_schema`, `uc_table`, `uc_column` — Unity Catalog + objects identified by their three- or four-part name + +The "three tiers" in everyday conversation are: + +- **Product / Contract level** — "this data product is about Customer + Order". Used for marketplace discovery and high-level grounding of + agents. +- **Schema level** — "this schema object in the contract is the + Customer entity". Used for cross-contract joins and concept-level + lineage. +- **Property / Column level** — "this `arr_usd` column is Annual + Recurring Revenue". Used for column-level definitions in Ask Ontos + and for data-driven concept exploration. + +Same table, same manager, three narratives. Discovery, governance, and +agent grounding all benefit from linking at the lowest tier you can +reasonably justify. + +### Business glossary as a published-ontology view {#glossary-as-view} + +When a steward "creates a glossary": + +1. Ontos creates a knowledge collection with `collection_type=glossary`. + Under the hood this lives in the `urn:glossary:` context inside the + conjunctive graph. +2. The steward adds concepts to the collection. These are real RDF + triples — they get an IRI, a label, optional synonyms, optional + broader/narrower relations. +3. To bind a glossary term to a real asset, the steward (or an automated + workflow) writes a semantic link from the asset to the concept IRI. +4. Publication is a lifecycle action on the concept (publish / certify / + deprecate). It controls UX visibility, not the underlying graph + storage. + +This means the same concept can be (a) part of an enterprise ontology +loaded by upload, (b) presented as a glossary term in a domain-scoped +collection, and (c) linked to dozens of contracts and columns — all +through the same triple plumbing. + +### Industry packs {#industry-packs} + +`IndustryOntologyManager` ships pre-built ontologies that you can load as +starting points: FIBO (financial industry), GS1 (supply chain), schema.org +(generic web concepts), and simple OWL packs for common patterns. These +are not opinionated about your specific data — they give the customer a +populated graph to react to rather than a blank canvas. + +### Round-trip asymmetry — be honest about it {#round-trip-asymmetry} + +The top-down flow (ontology → physical assets → UC tags) ships in the +forward direction; the reverse direction is still evolving. Here's the +honest current state. + +**What ships today.** Ontos can: + +- Author and store the ontology. +- Pin concepts to UC objects via semantic links — the read-time + representation works at product, contract, schema, and column levels. +- Surface those links in marketplace, Ask Ontos, and MCP agents. +- **Propagate concept assignments to UC governance tags via the + `uc_tag_sync` workflow.** The job reads `entity_semantic_links` + joined with contract/product/domain/asset metadata and issues UC + `ALTER TABLE … SET TAGS (...)` statements through Spark SQL. Runs + on schedule or on demand. Installed from Settings → Background Jobs. + This is the production path on customer deployments. +- Serialize the same tag changes through the + [Delivery Service](delivery-and-propagation.md#concept-to-uc-tag) in + Indirect mode, so a Git manifest captures every concept-driven tag + assignment for auditable downstream replay. + +**What's still evolving.** + +- The Delivery Service **Direct** mode for `TAG_ASSIGN` against UC's + tag API is partial — the change type, notification templates, and + Indirect path exist; the Direct call to UC's tag API is being filled + in. Today's production path is the workflow described above. +- **Reading existing UC tags back into Ontos as concept assignments** + (the reverse direction) is not shipping. The customer voice tracking + this work captures it as "Tags reading from UC is not there." Plan + demos around the forward path and flag the reverse pull as evolving. + +## Under the hood + +### The runtime knowledge graph {#runtime-graph} + +`SemanticModelsManager` owns the runtime graph: + +- Conjunctive graph: a union of all enabled models, each loaded under its + own URN context (one per model). Contexts make it possible to enable / + disable individual models without rebuilding from scratch. +- Caches concepts, taxonomies, and stats with a five-minute TTL. Refresh + is explicit: changing a semantic model's enable bit invalidates the cache. +- Handles OWL `equivalentClass` parent/child extraction, blank-node + skolemization (so you can address blank nodes by stable URNs after + loading), and RDF list walking. +- Exposes the graph to the rest of the app via `/api/semantic-models/*` — + including `/query` (SPARQL), `/neighbors` (one-hop traversal), + `/statistics` (counts), and `/refresh-graph` (force rebuild). + +### SPARQL and graph navigation {#sparql-and-navigation} + +- `POST /api/semantic-models/query` runs SPARQL against the conjunctive + graph. `SPARQLQueryValidator` does basic input validation. On Unix, a + SIGALRM-based timeout cap protects against runaway queries. +- `GET /api/semantic-models/neighbors` returns one-hop neighbours of a + given IRI — used by the Knowledge Graph view to expand a node on click. +- `GET /api/semantic-models/statistics` returns counts (concepts, + relationships, by-type breakdowns) — used by the graph stats tab. +- `POST /api/semantic-models/refresh-graph` rebuilds the conjunctive + graph from currently-enabled models. Cheap on small ontologies, can be + slow on large industry packs. + +For non-SPARQL users, the same data is reachable through +`/api/semantic-models/concepts` (paged concept list with filters) and +`/api/semantic-models/hierarchy` (subclass tree). + +## Common questions {#common-questions} + +**"What's the best practice to start setting up a knowledge-graph-based +business glossary for our team?"** + +Start small and bottom-up. Pick one domain (say, Customer). Pick five +concepts everyone agrees about. Author them either as a TTL upload or by +adding concepts to a glossary collection. Link them to two or three real +data products you already have. Show the team how the same concept now +surfaces in marketplace search, in Ask Ontos, and in the column-level +glossary panel. Expand from there. Trying to model the whole organization +before linking anything to real data is the usual failure mode. + +**"What is the difference between an ontology and a knowledge graph in +Ontos?"** + +The ontology is the file you upload (or the bundled TTL). The knowledge +graph is the runtime structure built by union'ing all the enabled +ontologies and adding the per-instance triples you generate over time +(glossary collections, instance-level links). One ontology, one graph, +many semantic links sitting on top. + +**"My team uses Protégé and SHACL — does Ontos handle that?"** + +Yes. Ontos uses rdflib to parse uploaded files, so anything rdflib reads +(TTL, OWL, RDF/XML, N-Triples) lands cleanly. SHACL shapes are stored as +triples like everything else. Ontos does not currently run SHACL +validation against your instance data at write time — treat SHACL as +authoritative documentation of constraints rather than as an enforcement +hook in the current version. + +**"Can I have multiple ontologies enabled at the same time?"** + +Yes. Each enabled model contributes its triples to the conjunctive graph +under its own URN context. Ontos handles `owl:imports` lightly — you may +need to upload imported ontologies separately. Watch for concept-IRI +collisions: if two ontologies declare the same IRI, the graph treats them +as the same concept. That's usually what you want. + +## Further reading {#further-reading} + +- [Semantic Link](#semantic-link) and [Concept](#three-tier-linking) in + this file +- [Three-tier linking on contracts](data-contract-lifecycle.md#schema-objects) +- [Bottom-up vs top-down flows](end-to-end-flows.md) +- [Knowledge Engineer persona](personas-quick-reference.md#knowledge-engineer) + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/personas-quick-reference.md b/docs/handbook/personas-quick-reference.md new file mode 100644 index 000000000..28d0f9a23 --- /dev/null +++ b/docs/handbook/personas-quick-reference.md @@ -0,0 +1,221 @@ +# Personas Quick Reference + +This file orients the copilot to "what does this persona typically need from +Ask Ontos?" Each persona maps to one of the seeded roles described in +[roles-and-rbac.md](roles-and-rbac.md#built-in-roles) — except the +Knowledge Engineer, which is a persona without a seeded role (no permission +matrix; described here for completeness). + +The framings below are in the persona's own voice. The questions are +representative, not exhaustive — they shape ranking and disambiguation, not +gating. + +## What you see in Ontos + +### Admin {#admin} + +**What you do.** You own the deployment. Roles, workflows, integrations, +demo data, the MCP token store — when something is broken, you're the +person Ontos expects to fix it. You almost never edit individual +products or contracts; you edit the rules that govern them. + +**Where you spend time.** Settings → RBAC, Settings → Workflows, +Settings → Semantic Models, Settings → MCP, Settings → Connectors, +Settings → Demo Data, the workspace logs page when something is +broken. + +**Typical questions for Ask Ontos:** + +- "Why is user X not seeing feature Y?" — debug a permission gap. +- "Which roles have access to data-contracts at Read/Write?" +- "What workflow steps does the publish flow run in this deployment?" +- "Show me the failed workflow executions in the last 7 days." +- "What does `APP_ADMIN_DEFAULT_GROUPS` do, and is it consulted on + every start?" + +Admins want grounded references to settings, role assignments, +workflow definitions, and the underlying permission matrix. + +### Data Governance Officer {#data-governance-officer} + +**What you do.** You see the whole catalog. Your job is to make sure +products have domains, contracts have quality checks, PII fields are +classified, subscriptions don't outlive the products they depend on. +You don't build; you certify and audit. + +**Where you spend time.** The Data Products list filtered by status, +the Compliance dashboard, Estate Manager, Master Data Management, the +Asset Reviews queue, the Glossary across all collections. + +**Typical questions for Ask Ontos:** + +- "Which data products are missing a domain assignment?" +- "Which contracts have no quality checks for the `accuracy` + dimension?" +- "Are all PII-flagged Deliverables in approved-or-active status?" +- "Which subscriptions are still active on deprecated products?" +- "What's the certification coverage for products in the Finance + domain?" + +A DGO answer often spans multiple entity types and benefits from +links into the compliance and entitlements features. + +### Data Steward {#data-steward} + +**What you do.** You curate a slice — usually a domain. You're the +gatekeeper at two moments: when a contract is proposed for approval, +and when a product is submitted for certification. Outside of those +gates, you're maintaining glossary terms and triaging asset reviews. + +**Where you spend time.** The contract approval queue, the product +certification queue, the Glossary editor for your domain, the Asset +Reviews assigned to you, the Schema tab on contracts (where DQX +suggestions land), Semantic Links panels on products and contracts. + +**Typical questions for Ask Ontos:** + +- "Which contracts in my domain are stuck in `under_review`?" +- "What asset reviews are assigned to me right now?" +- "Who's the business owner of the customer-360 product?" +- "What glossary terms are linked to this contract?" +- "What changed in the latest version of this product compared to + the previous?" +- "I accepted 14 DQX suggestions yesterday — did they make it into + the contract definitions?" + +A Steward wants action-oriented answers: what needs my attention, +who should I escalate to, what's the current state. + +### Data Producer {#data-producer} + +**What you do.** You build products and contracts. You spend your +time on the detail pages — composing Deliverables, drafting schemas, +wiring quality checks, picking delivery methods. Promotion through +lifecycle states (draft → proposed → approved → active) is your day +job; certification is somebody else's. + +**Where you spend time.** Data Products list filtered by your team, +the product detail page (Deliverables, Consumables, semantic links, +quality), the contract detail page (Schema tab, quality, SLAs), +sometimes the Asset Explorer for UC ingestion. + +**Typical questions for Ask Ontos:** + +- "How do I move my product from `draft` to `proposed`?" +- "What does `auto_approve` on a Deliverable do?" +- "Why is my Deliverable's contract showing as NULL — is that + allowed?" (Yes — see + [Deliverable](data-product-lifecycle.md#output-port).) +- "What quality dimensions does ODCS support?" +- "What does the publish workflow do — does it grant permissions + automatically?" (No — publishing controls visibility; access + comes through the subscribe workflow.) +- "What's the difference between a Deliverable and an output port?" + (Same thing — see + [Deliverable](data-product-lifecycle.md#output-port).) + +Producers benefit from concrete answers about lifecycle transitions, +ODPS / ODCS field semantics, and what a workflow will do when +triggered. + +### Data Consumer {#data-consumer} + +**What you do.** You find products in the marketplace and request +access. You don't draft anything — you subscribe, sign agreements, +provide feedback. Notifications tell you when a product you +subscribe to changes or breaks. + +**Where you spend time.** The Marketplace, the home page Discovery +section, individual product detail pages, your "My Subscriptions" +view, the Notification center. + +**Typical questions for Ask Ontos:** + +- "Where can I find a product about customer churn?" +- "Who owns the daily-orders product?" +- "How do I request access to this Deliverable?" +- "What does subscribing to a product do for me?" (Notifications on + deprecation, new versions, compliance violations.) +- "Why don't I see this product in the marketplace?" (Likely + `publication_scope` is `none` or below the consumer's visibility.) + +Consumers want short, action-oriented answers and direct links to +the request-access wizard. + +### Security Officer {#security-officer} + +**What you do.** You configure security features, entitlement sync, +and access classifications. You're consulted on contract approvals +when PII or restricted data is involved. You don't certify products; +you sign off on the security side of the certification. + +**Where you spend time.** Settings → Security Features, Settings → +Entitlements, Settings → Entitlements Sync, the Compliance dashboard, +sometimes contract approval queues for sensitive data. + +**Typical questions for Ask Ontos:** + +- "Which entitlement personas have admin on `security-features`?" +- "What sync jobs are configured for entitlements, and when did they + last run?" +- "Which contracts mark fields with `RESTRICTED` classification?" +- "What does the on_first_access disclaimer workflow look like?" + +Security Officer questions often cross into Admin territory; answers +should respect the persona's `Admin`-level access to +security-features, entitlements, and entitlements-sync, with +`Read-only` on data-asset-reviews. + +### Knowledge Engineer / Data Architect {#knowledge-engineer} + +**What you do.** You author the ontology in OWL/TTL/SHACL externally +and load it into Ontos. You decide what the canonical concepts are, +how they relate, and where SHACL constraints live. Ontos uses your +ontology to drive asset types, drive concept search, ground Ask +Ontos, and feed agents via MCP. + +**Where you spend time.** Outside Ontos in Protégé or TopBraid +authoring TTL. Inside Ontos: Settings → Semantic Models (upload), +the Concept Browser, the Knowledge Graph view, the SPARQL search +tab, the Ontology Generator when starting from UC metadata. + +**Not a seeded role.** The Knowledge Engineer doesn't have a built-in +Ontos role. Most knowledge engineers map to Data Producer or Data +Steward depending on whether they also work on data products. +Permissions to author semantic models come from the +`semantic-models` feature's `Read/Write` or `Admin` level. + +**Typical questions for Ask Ontos:** + +- "What concepts does our ontology cover for the Sales domain?" +- "Which concepts have no data mapping yet?" (Gap analysis.) +- "What's the difference between an ontology and a knowledge graph + in Ontos?" +- "Can I use SHACL shapes for validation?" +- "How do I bring my Protégé file into Ontos?" +- "What does it mean for the ontology to be prescriptive?" + +Knowledge Engineers expect technical, RDF-aware answers — saying +"concept" when you mean concept, "term" when you mean term, +distinguishing the source artifact from the runtime graph. + +## Under the hood + +### The empty-groups persona ("anon") {#anon} + +Used in testing to exercise fully-denied paths. A request from a +user with zero groups should resolve to no Ontos role and hit `None` +on every feature. The copilot should answer such users only on +**public** concept questions (e.g., "what is a data contract?") and +refuse to surface specific entity data. + +## Cross-references {#cross-references} + +- [Roles and RBAC](roles-and-rbac.md) — the seeded permission + matrices behind these personas +- [Demo-mode persona override](roles-and-rbac.md#persona-override) — + how to switch personas at runtime for testing or demos +- [End-to-end flows](end-to-end-flows.md) — who does what at each + step in the canonical bottom-up and top-down journeys + +_Last verified against codebase: 2026-05-28_ diff --git a/docs/handbook/roles-and-rbac.md b/docs/handbook/roles-and-rbac.md new file mode 100644 index 000000000..4f9b05c2f --- /dev/null +++ b/docs/handbook/roles-and-rbac.md @@ -0,0 +1,397 @@ +# Roles and RBAC + +Ontos authorization has two layers that get easily conflated. The **permission +model** says "what is this user allowed to do in this feature?" — it's a +matrix of feature × access level. The **role catalog** says "what bundle of +permissions does this named role carry?" — Admin, Data Steward, Data +Producer, etc. Roles map users (via Databricks groups) to permission rows. + +Customers asking about permissions usually want to know one of three things: +"why can't I see X?", "why can someone else edit Y?", or "what does the new +person need to be able to do?". This doc is structured so you can answer all +three. + +## What you see in Ontos + +### The permission model {#permission-model} + +A **permission** in Ontos is a `feature_id : access_level` pair. The set of +features (the `APP_FEATURES` map) is the source of truth; each feature +declares which access levels are valid for it. + +#### Access levels {#access-levels} + +`FeatureAccessLevel` defines six levels in ascending order: + +| Level | Meaning | +|---|---| +| `None` | No access. Feature is hidden in the UI; API returns 403. | +| `Read-only` | Can view, cannot modify. | +| `Filtered` | Read/Write restricted to a subset (typically by domain). Higher than `Read-only`, lower than `Read/Write`. In the current Ontos version, only the data-products feature implements `Filtered` scoping; other features that list `Filtered` as allowed treat it as `Read/Write` until scoping is wired. | +| `Read/Write` | Can view and modify within the feature. | +| `Full` | All operations within the feature scope, may include feature-level configuration. Used by Catalog Commander and Estate Manager. | +| `Admin` | Everything `Full` does, plus administrative actions (delete glossary, configure feature settings). | + +A given feature does not accept every level. Each feature's +`allowed_levels` list constrains what's assignable. + +> Practical caveat: an individual API endpoint may apply a stricter gate +> than the feature-level access listed in the matrix below. The matrix +> describes the front-of-feature expectation; production endpoints may +> additionally check sub-permissions. If a user with the documented level +> gets a 403, trust the 403 — the endpoint enforces what it actually +> enforces, even when stricter than this table. + +#### Representative features and what each level does {#feature-walkthrough} + +This is not the full list — `APP_FEATURES` is the source of truth — but +these are the features customers ask about most. + +**`data-products`** (sidebar group: Build). `allowed_levels` includes +`None`, `Read-only`, `Filtered`, `Read/Write`, `Admin`. +- `None` — Data Products view is hidden from the sidebar; API returns + 403. +- `Read-only` — Marketplace and detail pages render, but Create / Edit / + Status-change buttons are absent. +- `Filtered` — Edit rights apply only to products in domains the user + owns through team membership. The implementation lives in + DataProductsManager's authorization check. +- `Read/Write` — Standard producer access: create products, edit any + product in your domain, propose for review, transition status up to + active (with the appropriate workflow approvals). +- `Admin` — Add the ability to delete products and to publish even + without going through certification. + +**`data-contracts`** (Build). Same access levels as data-products minus +`Filtered`. +- `Read-only` — Can read schemas, quality checks, SLAs. +- `Read/Write` — Can draft and edit contracts, attach to output ports, + define quality checks. +- `Admin` — Can delete contracts, override certification. + +**`semantic-models`** (Build, displayed as "Concept Browser"). +- `Read-only` — Browse the concept catalog and glossary terms. +- `Read/Write` — Add concepts to glossary collections, create semantic + links, upload ontologies, run SPARQL queries. +- `Admin` — Delete glossaries, manage all semantic-model lifecycle. + +**`process-workflows`** (Govern). Restricted access levels: `None`, +`Read-only`, `Admin`. Mid-tier `Read/Write` is intentionally absent. +- `Read-only` — View workflow definitions, view executions and + agreements. +- `Admin` — Author / edit / delete workflow definitions; only Admin + edits workflow definitions because they have UC-grant-level + consequences via the `grant_permissions` step. + +**`settings`** (Settings layout gate). Read levels gate the visibility +of the Settings sidebar; sub-pages have their own feature IDs (e.g., +`settings-roles`, `settings-workflows`, `settings-semantic-models`). +- `Read-only` — Can see Settings and read the listed configurations. +- `Read/Write` — Can edit most settings. +- `Admin` — Plus dangerous actions: re-seeding roles, demo-data + loading, ITSM connector edits. + +**`marketplace`** (and the marketplace-style discovery features). +- `Read-only` — Browse published products. +- `Read/Write` — Subscribe / unsubscribe; submit access requests. +- `Admin` — Approve subscriptions, configure marketplace policies. + +**`approvals` / `agreements` / `notifications`**. +- `Read-only` — See agreements you signed, notifications addressed to + you. +- `Read/Write` — Approve agreements assigned to you, mark + notifications read. +- `Admin` — See all agreements across the workspace, manage + notification settings centrally. + +If you need the full list, read `APP_FEATURES` in `common/features.py` +directly — it carries the canonical names, sidebar groups, and +allowed-levels lists for every feature including the Govern, Deploy, +and Settings groups. + +#### Why "what level for what feature" actually matters {#why-permissions-matter} + +The level matters because the UI and API behave differently per level +in user-visible ways: + +| Feature | `Read-only` means… | `Read/Write` means… | +|---|---|---| +| settings-roles | You can see role definitions and the assignment matrix. You can't change them. | You can create new roles, edit access-level grants, reassign user groups. | +| data-products | You see products in the marketplace, drill into details, see lineage. You can't create, edit, or publish. | You can create new products, edit any product (subject to domain scoping if `Filtered`), and move them through lifecycle states. | +| data-contracts | You can read schemas, quality definitions, SLAs. | You can author new contracts, add schemas and quality checks, propose for review. | +| process-workflows | You can read workflow definitions and view past executions. | (Not allowed for this feature — only `None`, `Read-only`, `Admin`.) | +| semantic-models | Browse concepts, glossary, view the graph. | Author concepts, write semantic links, upload ontologies, run SPARQL. | +| approvals | See agreements you signed, notifications addressed to you. | Approve agreements assigned to you. | + +A non-obvious pattern: extending a feature to a new persona requires +auditing every endpoint's permission gate, not just the front-of-feature +gate. A wizard that's gated as `settings:READ_ONLY` at the entry point +but `data-contracts:READ_WRITE` inside an inner endpoint will 403 a +consumer-persona user mid-flow — see the +[per-execution authorization](#per-execution-authz) section. + +### Built-in roles {#built-in-roles} + +Ontos seeds six built-in roles on first start when no roles exist yet. +After seeding they are editable like any other role. + +| Role | One-paragraph framing | +|---|---| +| [Admin](#admin) | You own the deployment. Roles, workflows, integrations, demo data, the MCP token store — when something is broken, you're the person Ontos expects to fix it. | +| [Data Governance Officer](#data-governance-officer) | You see the whole catalog. Your job is to make sure products have domains, contracts have quality checks, PII is classified, subscriptions don't outlive their products. You certify and audit; you don't build. | +| [Data Steward](#data-steward) | You curate a slice — usually a domain. You're the gatekeeper at two moments: contract approval and product certification. Outside those gates, you maintain glossary terms and triage reviews. | +| [Data Producer](#data-producer) | You build products and contracts. You spend your time on the detail pages — composing deliverables, drafting schemas, wiring quality checks. Lifecycle promotion is your day job; certification is somebody else's. | +| [Data Consumer](#data-consumer) | You find products and request access. You don't draft anything — you subscribe, sign agreements, provide feedback. | +| [Security Officer](#security-officer) | You configure security features, entitlements, access classifications. You're consulted on contract approvals involving PII or restricted data. You sign off on the security side of certification. | + +#### Admin {#admin} + +Granted `Admin` on every feature by default, including all settings +sub-pages. The Admin role is the canonical carrier of "Ontos admin" +authority: features that require elevated authority (role overrides, +MCP token management, dangerous settings actions) check membership in +*any* role flagged as an admin role rather than treating +`settings:ADMIN` as a proxy. This separation means giving a user write +access to Settings does not implicitly turn on admin-only capabilities +elsewhere. + +Group assignment for Admin is seeded from the `APP_ADMIN_DEFAULT_GROUPS` +environment variable (default: `["admins"]`). This is **only consulted +on first-time seeding** — later restarts do not re-merge env-var values +into the existing role. To add admins after first start, edit the +role's `assigned_groups` from Settings → RBAC. This catches new +deployments often enough that it has its own dedicated section in the +Ontos Setup guide. + +Other seeded roles (Data Governance Officer, Data Steward, Data +Producer, Data Consumer, Security Officer) also ship with default +group bindings matching the conventional group names — see the role +definitions table below and Settings → RBAC for the live values. + +#### Data Governance Officer {#data-governance-officer} + +Cross-cutting governance authority. + +| Feature | Level | +|---|---| +| data-domains | Admin | +| data-products | Admin | +| data-contracts | Admin | +| data-catalog | Admin | +| business-glossary | Admin | +| compliance | Admin | +| estate-manager | Admin | +| master-data | Admin | +| security-features | Admin | +| entitlements | Admin | +| entitlements-sync | Admin | +| data-asset-reviews | Admin | +| catalog-commander | Full | +| process-workflows | Read-only | +| teams | Read-only | +| projects | Read-only | +| comments | Read/Write | + +#### Data Steward {#data-steward} + +Curates domains, contracts, glossary terms; reviews assets. + +| Feature | Level | +|---|---| +| data-domains | Read/Write | +| data-products | Read/Write | +| data-contracts | Read/Write | +| data-catalog | Read/Write | +| business-glossary | Read/Write | +| data-asset-reviews | Read/Write | +| compliance | Read-only | +| process-workflows | Read-only | +| catalog-commander | Read-only | +| teams | Read-only | +| projects | Read-only | +| comments | Read/Write | + +#### Data Producer {#data-producer} + +Creates data products and contracts; manages own teams and projects. + +| Feature | Level | +|---|---| +| data-products | Read/Write | +| data-contracts | Read/Write | +| teams | Read/Write | +| projects | Read/Write | +| data-domains | Read-only | +| data-catalog | Read-only | +| business-glossary | Read-only | +| catalog-commander | Read-only | +| process-workflows | Read-only | +| comments | Read/Write | + +#### Data Consumer {#data-consumer} + +Read-only access for discovery, subscription, and commenting. + +| Feature | Level | +|---|---| +| data-products | Read-only | +| data-contracts | Read-only | +| data-domains | Read-only | +| data-catalog | Read-only | +| business-glossary | Read-only | +| catalog-commander | Read-only | +| process-workflows | Read-only | +| teams | Read-only | +| projects | Read-only | +| comments | Read/Write | + +#### Security Officer {#security-officer} + +Focused on security configuration and entitlements. + +| Feature | Level | +|---|---| +| security-features | Admin | +| entitlements | Admin | +| entitlements-sync | Admin | +| compliance | Read/Write | +| process-workflows | Read-only | +| data-asset-reviews | Read-only | +| comments | Read/Write | + +Features not listed for a given role default to `None`. + +### Filtered (domain-scoped) access {#filtered-access} + +The `Filtered` access level signals "read/write, but only to a subset". +In the current Ontos version it is wired only for the **data-products** +feature, where it restricts visibility and edit rights to products in +domains owned by the caller (resolved via team membership and ownership +ties). Other features list `Filtered` as a permitted level only if the +feature explicitly implements the scoping; without an implementation, +the level behaves like `Read/Write` for that feature. This asymmetry is +evolving — more features may grow scoping in future versions. + +### Demo-mode persona override {#persona-override} + +For local development and customer demos, Ontos supports a runtime +persona switch: + +- Set `TEST_USER_TOKEN` in the backend environment. +- The frontend exposes a persona picker (from + `data/test_personas.yaml`). +- Each request from the frontend carries `X-Test-Token`, + `X-Test-User-Email`, and optional `X-Test-User-Groups` headers. +- The backend resolves the identity from these headers instead of OBO + SCIM for the duration of the request. + +The default persona set covers Admin, Data Governance Officer, Data +Steward, Data Producer, Data Consumer, Security Officer, and an +empty-groups "anon" persona for exercising fully-denied paths. + +Leave `TEST_USER_TOKEN` unset in production. When it is unset, the +persona headers are ignored and normal OBO resolution applies. + +#### Role override (impersonation) {#role-override} + +Independently from the persona-token mechanism, an admin can apply a +role override for a user via the role-switcher UI. The override is +held in-memory for the backend process lifetime and replaces the +user's group-derived role for permission evaluation. Non-admin +callers may only apply overrides to roles whose `assigned_groups` +they actually belong to; admins may apply any override. Clearing the +override returns the user to group-based resolution. + +The role override does not affect the workspace-admin shortcut — a +workspace admin remains a workspace admin even while impersonating a +non-admin role for the rest of Ontos's permission evaluation. + +## Under the hood + +### Identity resolution {#identity-resolution} + +When a request arrives, Ontos resolves the caller through three layers +in order: + +1. **Identity provider (Entra/Okta, etc.).** Ontos never talks to the + IdP directly. It only sees what Databricks SCIM exposes. +2. **Databricks workspace/account groups.** Resolved at request time + via on-behalf-of SCIM lookup. Requires the `iam.current-user:read` + scope to be declared in the app manifest. Workspace-only groups do + **not** appear via the OBO `current_user.me()` path; use + account-level groups for role assignment. +3. **Ontos roles.** A role is a DB row whose `assigned_groups` is a + list of strings. The matcher is a case-insensitive set intersection + between the user's resolved groups and each role's + `assigned_groups`. When multiple roles match, the role with the + highest summed access-level weight wins. + +#### Workspace-admin shortcut {#workspace-admin-shortcut} + +The `is_user_admin` helper checks membership in +`APP_ADMIN_DEFAULT_GROUPS` (default `["admins"]`). This bypass runs +**independently of the Ontos role system** — a workspace admin is +treated as admin for cascade-bypass checks even if they hold no Ontos +role. + +A complementary predicate — the **Ontos-admin** check — resolves +"admin" through the role catalog: a user is an Ontos admin if their +resolved groups intersect the `assigned_groups` of any role flagged +as an admin role. Sensitive endpoints (role overrides, MCP token +management, role-catalog reads) increasingly consult this Ontos-admin +predicate rather than reading `settings:ADMIN` as a proxy. The two +predicates exist side-by-side because the workspace-admin shortcut +is the bootstrap path (it works even before any roles exist), while +the Ontos-admin predicate is the steady-state authority once roles +are configured. + +The practical implication for testing: a workspace admin cannot easily +exercise non-admin code paths from their own account; you need a +non-admin token to test denial branches. Equally, an Ontos admin who +is *not* a workspace admin can still be denied by code that hits the +workspace-admin shortcut — audit both paths when verifying a +permission change. + +#### Email-as-implicit-group fallback {#email-as-group-fallback} + +When SCIM returns an empty group list for a user, Ontos falls back to +using the user's own email as a single implicit "group" name. This is +a recovery mechanism for environments where SCIM is broken (local +dev, certain sandbox setups, service principals that cannot read +SCIM). It is **fallback only, never additive**: if the user has any +resolvable real group, the email is not added. + +Production deployments should not depend on the email fallback. It +exists for bootstrapping the first admin when SCIM is unavailable. +The Ontos Setup guide documents the DB update to fix this case. + +### Per-execution authorization {#per-execution-authz} + +Feature-level checks decide whether a user may enter a feature at all. +Inside a feature, sensitive operations (approving an agreement, +granting permissions, modifying a specific data product) consult +per-entity ownership and per-execution role checks in addition to the +outer feature gate. This is why an Ontos role grant alone may not be +sufficient to approve a specific agreement — the underlying entity's +ownership and the workflow step's configured approver group also gate +the action. + +The pattern matters when extending a feature to a new persona: the +outer feature gate gets the persona in the door, but every inner +sensitive operation may have its own check that was originally written +assuming a different persona. Audit the gates end to end before +declaring a permission change done. + +## Cross-references {#cross-references} + +- [Permission model](#permission-model) and [Filtered access](#filtered-access) +- [Built-in roles](#built-in-roles) — six seeded roles +- [Demo-mode persona override](#persona-override) +- [Per-execution authorization](#per-execution-authz) for sensitive + operations +- [Ontos Setup guide](../Ontos%20Setup.md) — first-admin bootstrap if + SCIM doesn't return the seed group +- [Persona quick reference](personas-quick-reference.md) for the + questions each persona typically asks + +_Last verified against codebase: 2026-05-28_ diff --git a/src/backend/src/controller/llm_search_manager.py b/src/backend/src/controller/llm_search_manager.py index 07b32b95b..48c7b6af4 100644 --- a/src/backend/src/controller/llm_search_manager.py +++ b/src/backend/src/controller/llm_search_manager.py @@ -7,6 +7,7 @@ """ import json +import re import uuid from datetime import datetime from typing import Dict, List, Optional, Any, Tuple @@ -16,6 +17,7 @@ from src.common.config import Settings, get_settings from src.common.logging import get_logger +from src.tools.system_prompts import get_system_prompt from src.models.llm_search import ( ConversationSession, ChatMessage, ChatResponse, MessageRole, ToolCall, ToolName, SessionSummary, LLMSearchStatus, @@ -27,115 +29,46 @@ logger = get_logger(__name__) -# ============================================================================ -# System Prompt -# ============================================================================ - -SYSTEM_PROMPT = """You are Ontos, an intelligent data governance assistant. You help users discover, understand, and analyze data within their organization. - -## Your Capabilities - -You have access to the following tools: - -1. **search_data_products** - Search for data products by name, domain, description, or keywords. Use this to find available datasets. - -2. **search_glossary_terms** - Search the knowledge graph for business concepts, terms, and their definitions from loaded ontologies and taxonomies. - -3. **get_data_product_costs** - Get cost information for data products, including infrastructure, HR, storage, and other costs. - -4. **get_table_schema** - Get the schema (columns and types) of a specific table. Use this before writing analytics queries. - -5. **execute_analytics_query** - Execute a read-only SQL SELECT query against Databricks tables. Use this for aggregations, joins, and data analysis. - -6. **explore_catalog_schema** - List all tables and views in a Unity Catalog schema with their columns. Use this to understand what data assets exist and suggest semantic models or data products. - -7. **create_draft_data_contract** - Create a new draft data contract from schema information. Always create contracts in draft status for user review. - -8. **create_draft_data_product** - Create a new draft data product, optionally linked to a contract. Always create products in draft status for user review. - -9. **update_data_product** - Update an existing data product's domain, description, or status. - -10. **update_data_contract** - Update an existing data contract's domain, description, or status. - -11. **add_semantic_link** - Link a data product or contract to a business term/concept from the knowledge graph. Use search_glossary_terms first to find the concept IRI. - -12. **list_semantic_links** - List semantic links (business term associations) for a data product or contract. - -13. **remove_semantic_link** - Remove a semantic link. Use list_semantic_links first to find the link ID. +# Internal grounding markers — the system prompt instructs the model to emit +# two kinds of markers so reviewers can audit grounding, but neither belongs in +# the user-facing response. Strip server-side as a safety net: +# 1. `` — citation comments. Most markdown +# renderers drop HTML comments, but the chat UI surfaces them as text. +# 2. `[Confirmed]` / `[Documented]` / `[Inferred]` — three-tier confidence +# labels. The model still emits them so the act of stratifying anchors +# the answer in the right source; we capture for audit but hide from +# end users. +# Capture both into debug_info (`internal_citations`, `confidence_labels`). +_CITATION_COMMENT_RE = re.compile(r"") +_CONFIDENCE_LABEL_RE = re.compile(r"\s*\[(Confirmed|Documented|Inferred)\]") -14. **search_tags** - Search for existing tags by name, namespace, or description. -15. **create_tag** - Create a new tag. Tags use the format `namespace/tag_name` (e.g., `import/healthcare` creates tag 'healthcare' in namespace 'import'). If no slash is present, the tag goes into the 'default' namespace. Namespaces are auto-created if they don't exist. +def _strip_internal_citations(text: str) -> Tuple[str, List[str], List[str]]: + """Remove internal grounding markers from the response. -16. **assign_tag_to_entity** - Assign an existing tag to a data product, data contract, domain, team, or project. Use search_tags first to find the tag ID. - -17. **list_entity_tags** - List all tags assigned to a specific entity. - -18. **remove_tag_from_entity** - Remove a tag assignment from an entity. - -## Tag Naming Convention - -Tags are organized using namespaces with a slash (`/`) separator: -- `namespace/tag_name` format (e.g., `import/healthcare`, `compliance/gdpr`, `pii/sensitive`) -- If no slash is present (e.g., just `pii`), the tag uses the 'default' namespace -- Examples: - - `import/healthcare` → namespace='import', tag='healthcare' - - `compliance/gdpr` → namespace='compliance', tag='gdpr' - - `customer-data` → namespace='default', tag='customer-data' - -## Discovery Strategy (IMPORTANT - follow this priority order) - -When users ask about finding, discovering, or locating data, ALWAYS follow this priority order: - -**Tier 1 - Governed Assets (search first, always):** -- search_data_products: Search for curated, governed data products -- global_search: Search across all indexed features (products, contracts, terms) -- search_glossary_terms + find_entities_by_concept: Find products/contracts linked to business concepts - -**Tier 2 - Data Contracts and Semantic Enrichment:** -- search_data_contracts: Search for data contracts that may describe relevant data -- search_glossary_terms: Explore business concepts and definitions - -**Tier 3 - Unity Catalog Direct Exploration (ONLY when appropriate):** -- list_catalogs, explore_catalog_schema, get_table_schema: Browse raw catalog assets -- ONLY use these when: - (a) The user explicitly asks to explore catalogs, schemas, or tables, OR - (b) Tier 1 and Tier 2 returned no results AND you've communicated that to the user - -NEVER skip to Tier 3 directly. Data Products are the primary offering of this platform. - -## Additional Guidelines - -- When users ask about a concept, topic, or term you don't immediately recognize as a data product or catalog item, always try search_glossary_terms first -- it searches loaded ontologies and taxonomies that may contain the concept (e.g., domain-specific terms like "pizza", "customer", "transaction") -- When searching for data products by a business concept or topic, also use search_glossary_terms to find matching ontology concepts, then find_entities_by_concept with the concept IRI to discover products linked semantically. Combine results from both search_data_products and the semantic tool chain for comprehensive discovery. -- When executing analytics queries, first get the table schema to understand available columns -- Explain your reasoning and cite the data sources you used -- If you don't have access to certain data or a query fails, explain why and suggest alternatives -- Format responses with clear sections, tables, and bullet points for readability -- Be concise but thorough - include relevant context without unnecessary verbosity - -## Response Format - -When presenting data: -- Use markdown tables for tabular results. IMPORTANT: Tables must have proper line breaks between each row: - ``` - | Column1 | Column2 | - |---------|---------| - | value1 | value2 | - | value3 | value4 | - ``` - Never put multiple table rows on a single line. -- Use bullet points for lists -- Bold important numbers and findings -- Include units (USD, %, etc.) where applicable + Returns (cleaned_text, citations, confidence_labels). The cleaned text has + both kinds of markers removed and any triple-newlines created by the strip + collapsed back to doubles. + """ + if not text: + return text, [], [] + citations = [m.strip() for m in _CITATION_COMMENT_RE.findall(text)] + confidence_labels = list(_CONFIDENCE_LABEL_RE.findall(text)) + cleaned = _CITATION_COMMENT_RE.sub("", text) + cleaned = _CONFIDENCE_LABEL_RE.sub("", cleaned) + cleaned = re.sub(r"\n{3,}", "\n\n", cleaned).rstrip() + return cleaned, citations, confidence_labels -## Limitations -- You can only execute read-only SELECT queries -- Query results are limited to 1000 rows -- You can only access tables the user has permissions for -- Cost data may not be complete for all products -""" +# ============================================================================ +# System Prompt +# ============================================================================ +# +# The default prompt and the `LLM_SYSTEM_PROMPT` env-override path now live +# in `src.tools.system_prompts.get_system_prompt`. This manager calls +# it once per `_process_with_llm` invocation. Phase 2/3 will start passing +# personalization context (role, page, selected entity, adoption mode) +# through that function — the signature already accepts those. # ============================================================================ @@ -350,7 +283,13 @@ def __init__( self._search_manager = search_manager self._ws_client = workspace_client self._session_store = get_session_store() # Use global singleton - + + # Per-chat-call personalization context (Phase 3). Set by + # ``chat()`` before invoking ``_process_with_llm`` and reset to + # ``None`` afterwards so the manager is safe to reuse across + # requests within a process. + self._chat_context: Optional[Dict[str, Any]] = None + # Initialize tool registry with all default tools self._tool_registry = create_default_registry() @@ -381,8 +320,23 @@ def _get_latest_user_message(self, session: ConversationSession) -> str: # ======================================================================== def get_status(self) -> LLMSearchStatus: - """Get the status of LLM search functionality.""" + """Get the status of LLM search functionality. + + Includes the current ``adoption_mode`` so the frontend can pick + the right starter-prompt set without an extra round-trip. The + snapshot is computed inline; a failure is logged and silently + downgraded to ``adoption_mode=None`` rather than failing the + whole status call. + """ model = self._settings.LLM_ENDPOINT + adoption_mode: Optional[str] = None + try: + from src.tools.app_state import get_adoption_snapshot + snapshot = get_adoption_snapshot(self._db) + adoption_mode = snapshot.get("adoption_mode") + except Exception as e: + logger.warning(f"adoption snapshot in get_status failed: {e}") + return LLMSearchStatus( enabled=self._settings.LLM_ENABLED, endpoint=model, @@ -390,7 +344,8 @@ def get_status(self) -> LLMSearchStatus: disclaimer=self._settings.LLM_DISCLAIMER_TEXT or ( "This feature uses AI to analyze data assets. AI-generated content may contain errors. " "Review all suggestions carefully before taking action." - ) + ), + adoption_mode=adoption_mode, ) def list_sessions(self, user_id: str) -> List[SessionSummary]: @@ -410,23 +365,71 @@ async def chat( user_message: str, user_id: str, session_id: Optional[str] = None, - debug: bool = False + debug: bool = False, + # Phase 3 personalization — frontend sends page / entity from + # the copilot store on every chat request; the route derives + # the user's effective Ontos role(s) and passes them too. All + # optional for backward compatibility — chats from non-UI + # clients (e.g. MCP, tests) still work without context. + role: Optional[str] = None, + page_name: Optional[str] = None, + page_url: Optional[str] = None, + feature_id: Optional[str] = None, + selected_entity: Optional[Dict[str, Any]] = None, ) -> ChatResponse: """ Process a chat message and return the assistant's response. - + Note: The workspace client passed to this manager should already have user credentials (OBO) for proper access control and audit trail. - + Args: user_message: The user's message user_id: ID of the user session_id: Optional session ID to continue conversation debug: When true, include debug info in the response - + role: Effective Ontos role label (Phase 3). Tailoring hint. + page_name: Page the user is on (Phase 3). + page_url: URL of the page (Phase 3). + feature_id: Feature ID the page maps to (Phase 3). Reserved + for future authz / context decisions; not used in the + prompt today. + selected_entity: Entity the user has selected in the UI + (Phase 3). Dict with optional ``type``, ``name``, ``id``. + Returns: ChatResponse with the assistant's message """ + # Stash context for the LLM-processing pass to read. Cleared + # in the ``finally`` below so a long-lived manager instance + # can't leak one request's context into the next. + self._chat_context = { + "role": role, + "page_name": page_name, + "page_url": page_url, + "feature_id": feature_id, + "selected_entity": selected_entity, + } + try: + return await self._chat_inner( + user_message=user_message, + user_id=user_id, + session_id=session_id, + debug=debug, + ) + finally: + self._chat_context = None + + async def _chat_inner( + self, + *, + user_message: str, + user_id: str, + session_id: Optional[str], + debug: bool, + ) -> ChatResponse: + """Body of ``chat()``, split out so ``chat()`` can wrap it in a + try/finally that always clears ``self._chat_context``.""" # Check if LLM is enabled if not self._settings.LLM_ENABLED: logger.warning("LLM chat requested but LLM_ENABLED is False") @@ -460,7 +463,14 @@ async def chat( response_content, tool_calls_executed, sources, debug_info = await self._process_with_llm( session, collect_debug=debug ) - + + # Sanitize orphan asterisk runs. The system prompt forbids + # ``****``/``*****`` as visual markers, but the LLM still + # emits them occasionally. ``****`` (4+) is never valid + # CommonMark, so stripping it is safe. + if response_content: + response_content = re.sub(r"\*{4,}", "", response_content) + # Add assistant response to database assistant_msg = self._session_store.add_message( self._db, session.id, MessageRole.ASSISTANT, content=response_content @@ -564,12 +574,51 @@ async def _process_with_llm( from src.tools.query_classifier import classify_query user_query = self._get_latest_user_message(session) categories = classify_query(user_query) - + # Get filtered tool definitions based on query categories tool_definitions = self._tool_registry.get_openai_definitions_filtered(categories) tool_names = [td["function"]["name"] for td in tool_definitions] logger.info(f"Using {len(tool_definitions)} tools for categories: {categories}") - + + # Pre-fetch the adoption snapshot ONCE per chat call. The same + # snapshot drives (a) the optional ``get_app_state`` tool result + # the LLM may decide to fetch, and (b) the Phase 2 system-prompt + # preamble we inject below so every conceptual answer is + # adoption-mode-aware even when the LLM never invokes the tool. + # Snapshot failures degrade gracefully: we log and proceed with + # ``adoption_mode=None`` so the prompt falls back to the + # Phase 1 default. + adoption_mode: Optional[str] = None + try: + from src.tools.app_state import get_adoption_snapshot + adoption_snapshot = get_adoption_snapshot(self._db) + adoption_mode = adoption_snapshot.get("adoption_mode") + logger.info( + f"Adoption snapshot: mode={adoption_mode} " + f"counts={adoption_snapshot.get('counts')}" + ) + except Exception as snapshot_err: + logger.warning( + f"Adoption snapshot failed; skipping mode preamble: " + f"{snapshot_err}", + exc_info=True, + ) + + # Resolve the system prompt once per chat invocation. The + # `get_system_prompt` helper honors `Settings.LLM_SYSTEM_PROMPT` + # as a verbatim override (previously dead code) and otherwise + # returns the default grounded prompt with optional Phase 2/3 + # preambles. Phase 3 (role / page / entity) is wired in + # ``process_chat`` -> ``chat`` -> ``_process_with_llm``. + system_prompt = get_system_prompt( + settings=self._settings, + role=self._chat_context.get("role") if self._chat_context else None, + page_name=self._chat_context.get("page_name") if self._chat_context else None, + page_url=self._chat_context.get("page_url") if self._chat_context else None, + selected_entity=self._chat_context.get("selected_entity") if self._chat_context else None, + adoption_mode=adoption_mode, + ) + if debug_info is not None: debug_info["query_classification"] = { "user_query": user_query, @@ -578,7 +627,10 @@ async def _process_with_llm( "tools_count": len(tool_definitions), } debug_info["model"] = self._settings.LLM_ENDPOINT - debug_info["system_prompt_length"] = len(SYSTEM_PROMPT) + debug_info["system_prompt_length"] = len(system_prompt) + debug_info["system_prompt_source"] = ( + "env_override" if self._settings.LLM_SYSTEM_PROMPT else "default" + ) # Count prior messages to show if the LLM has conversation context prior_msgs = session.messages[:-1] # exclude the just-added user message prior_tool_msgs = [m for m in prior_msgs if m.role == MessageRole.TOOL] @@ -597,7 +649,7 @@ async def _process_with_llm( raise RuntimeError(f"Session {session_id} not found in cache") # Build messages for LLM - messages = current_session.get_messages_for_llm(SYSTEM_PROMPT) + messages = current_session.get_messages_for_llm(system_prompt) # Call LLM try: @@ -697,13 +749,18 @@ async def _process_with_llm( content=json.dumps(result_dict), tool_call_id=tc.id ) else: + cleaned_content, citations, confidence_labels = _strip_internal_citations( + assistant_message.content or "" + ) if debug_info is not None: if iter_debug is not None: debug_info["iterations"].append(iter_debug) debug_info["total_tool_calls"] = total_tool_calls debug_info["total_iterations"] = iteration + 1 debug_info["total_elapsed_ms"] = int((time.time() - process_start) * 1000) - return assistant_message.content or "", total_tool_calls, sources, debug_info + debug_info["internal_citations"] = citations + debug_info["confidence_labels"] = confidence_labels + return cleaned_content, total_tool_calls, sources, debug_info if debug_info is not None and iter_debug is not None: debug_info["iterations"].append(iter_debug) diff --git a/src/backend/src/models/llm_search.py b/src/backend/src/models/llm_search.py index 7719317c8..2c07f60e4 100644 --- a/src/backend/src/models/llm_search.py +++ b/src/backend/src/models/llm_search.py @@ -102,6 +102,12 @@ class ToolName(str, Enum): # Global search GLOBAL_SEARCH = "global_search" + # Handbook search (grounds the copilot in docs/handbook/) + SEARCH_ONTOS_HANDBOOK = "search_ontos_handbook" + + # App-state introspection (powers adoption-mode preamble too) + GET_APP_STATE = "get_app_state" + # ============================================================================ # Tool Parameter Models @@ -172,11 +178,51 @@ class ChatMessage(BaseModel): model_config = {"from_attributes": True} +class SelectedEntity(BaseModel): + """Small descriptor of the entity currently selected in the UI. + + All three fields are optional so partial payloads (an entity with + only a name, no id yet, etc.) round-trip cleanly. Phase 3 of the + Ask Ontos uplift uses these to render a ``## Current user + context`` block in the system prompt. + """ + type: Optional[str] = Field(None, description="Entity type (e.g., 'data_product', 'data_contract', 'domain').") + name: Optional[str] = Field(None, description="Human-readable entity name as shown in the UI.") + id: Optional[str] = Field(None, description="Entity ID (UUID or natural key, depending on entity type).") + + class ChatMessageCreate(BaseModel): - """Request model for creating a new user message.""" + """Request model for creating a new user message. + + Phase 3 fields (``page_name`` … ``selected_entity``) are all + optional. Non-UI clients (MCP, tests) can keep posting the + original payload shape; the backend simply skips the + ``## Current user context`` preamble when nothing is provided. + """ content: str = Field(..., min_length=1, max_length=4000, description="User's message content") session_id: Optional[str] = Field(None, description="Session ID for continuing a conversation") debug: bool = Field(False, description="When true, include debug info (tool calls, categories, timing) in response") + # Phase 3: page / entity context. The route also derives the + # user's effective Ontos role(s) server-side and injects them into + # the prompt — clients do NOT send role themselves (defense in + # depth: a client can't impersonate a role just by lying in the + # payload). + page_name: Optional[str] = Field( + None, + description="Page the user is on (e.g., 'data-products', 'data-contracts', 'home'). Frontend sources this from the copilot store.", + ) + page_url: Optional[str] = Field( + None, + description="URL of the page (relative path, e.g., '/data-products/abc'). Surfaced as part of the user-context preamble.", + ) + feature_id: Optional[str] = Field( + None, + description="Feature ID the page maps to (e.g., 'data-products'). Reserved for future authz / context decisions; not used in the prompt today.", + ) + selected_entity: Optional[SelectedEntity] = Field( + None, + description="Entity the user is currently viewing in the UI, if any. Rendered as 'Viewing: ' in the user-context preamble.", + ) # ============================================================================ @@ -286,6 +332,15 @@ class LLMSearchStatus(BaseModel): endpoint: Optional[str] = Field(None, description="Configured LLM endpoint") model_name: Optional[str] = Field(None, description="Name of the configured foundation model") disclaimer: str = Field(..., description="Disclaimer text about AI limitations") + # Phase 2: adoption_mode is computed from the live DB snapshot + # whenever ``/api/llm-search/status`` is called. Frontend uses it + # to surface mode-aware starter prompts (blank vs active) without + # an extra round-trip. Stays ``None`` when the snapshot fails so + # the UI degrades to its default starter list. + adoption_mode: Optional[str] = Field( + None, + description="Current workspace adoption mode: 'blank' (no published data products), 'active', or null when undetermined.", + ) # ============================================================================ diff --git a/src/backend/src/routes/llm_search_routes.py b/src/backend/src/routes/llm_search_routes.py index dce4ff2cc..46cb18c91 100644 --- a/src/backend/src/routes/llm_search_routes.py +++ b/src/backend/src/routes/llm_search_routes.py @@ -8,7 +8,7 @@ LLM tools which filter results based on user permissions. """ -from typing import List +from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Request, status from src.common.dependencies import ( @@ -22,12 +22,70 @@ ChatMessageCreate, ChatResponse, ConversationSession, SessionSummary, LLMSearchStatus ) +from src.models.users import UserInfo from src.controller.llm_search_manager import LLMSearchManager from src.common.logging import get_logger logger = get_logger(__name__) + +def _derive_effective_role_label(request: Request, user: UserInfo) -> Optional[str]: + """Derive a single human-readable Ontos role label for the user. + + Strategy: + + 1. Pull the ``AuthorizationManager`` + ``SettingsManager`` off + ``request.app.state`` (the route doesn't take them as + dependencies — keeping the chat endpoint footprint small). + 2. Intersect ``user.groups`` (lowercase, case-insensitive) with + the ``assigned_groups`` of each configured app role. + 3. If multiple roles match, join their names with commas in + declaration order — the prompt is a tone hint, not an authz + gate, so we don't need to pick a "winner". + 4. Any exception logs and returns ``None`` so the chat call + proceeds rather than 500ing because of a context glitch. + """ + try: + settings_manager = getattr(request.app.state, "settings_manager", None) + if settings_manager is None: + return None + all_roles = settings_manager.list_app_roles() + if not all_roles: + return None + + # Honor applied role override first (UI role-switcher). + # ``AppRole.id`` is typed as ``UUID`` but the override is stored as + # a string (JSON has no UUID type), so compare via ``str()``. + try: + override_id = settings_manager.get_applied_role_override_for_user(user.email) + if override_id: + override_id_s = str(override_id) + for role in all_roles: + if str(role.id) == override_id_s: + return role.name + # Override id doesn't match any current role — fall through + # to group intersection rather than erroring. + except Exception as e: + logger.warning(f"Role-override lookup failed; falling back to groups: {e}") + + user_groups_lower = set(g.lower() for g in (user.groups or [])) + if not user_groups_lower: + return None + + matched: List[str] = [] + for role in all_roles: + role_groups_lower = set(g.lower() for g in (role.assigned_groups or [])) + if user_groups_lower & role_groups_lower: + matched.append(role.name) + + if not matched: + return None + return ", ".join(matched) + except Exception as e: + logger.warning(f"Effective-role lookup failed; falling back to role=None: {e}") + return None + router = APIRouter(prefix="/api/llm-search", tags=["LLM Search"]) @@ -99,35 +157,69 @@ async def chat( ) -> ChatResponse: """ Send a chat message and receive the assistant's response. - + The assistant can search for data products, glossary terms, costs, and execute analytics queries to answer your questions. - + Provide a session_id to continue an existing conversation. + + Phase 3 personalization: ``page_name`` / ``page_url`` / ``feature_id`` + / ``selected_entity`` are picked up from the request body and + passed to the manager. The user's effective Ontos role(s) are + derived server-side (NOT from the client) by intersecting the + user's group membership with role assignments — this is the + "what the user can actually do today" view that the copilot + should tailor its tone to. """ success = False details = { "params": { "session_id": message.session_id, - "message_length": len(message.content) + "message_length": len(message.content), + "page_name": message.page_name, } } - + try: - logger.info(f"LLM chat request from user {current_user.email}, session={message.session_id}") - + logger.info( + f"LLM chat request from user {current_user.email}, " + f"session={message.session_id}, page={message.page_name}" + ) + + # Derive a single role label by intersecting the user's groups + # with role assignments via the AuthorizationManager. Multiple + # roles -> comma-separated; none -> None. The label is purely + # a tone-of-voice hint to the LLM (not an authz boundary), so + # we deliberately fail open: any lookup error logs and the + # request proceeds with role=None. + role_label = _derive_effective_role_label(request, current_user) + + # ``selected_entity`` is a Pydantic model on the way in; the + # manager + prompt code expects a plain dict so we dump it + # here (Pydantic v2 ``model_dump`` returns a dict). + selected_entity_dict = ( + message.selected_entity.model_dump(exclude_none=True) + if message.selected_entity is not None + else None + ) + # Note: manager already has OBO workspace client from get_llm_search_manager dependency response = await manager.chat( user_message=message.content, user_id=current_user.email, session_id=message.session_id, - debug=message.debug + debug=message.debug, + role=role_label, + page_name=message.page_name, + page_url=message.page_url, + feature_id=message.feature_id, + selected_entity=selected_entity_dict, ) - + success = True details["session_id"] = response.session_id details["tool_calls"] = response.tool_calls_executed - + return response except Exception as e: diff --git a/src/backend/src/tests/integration/test_llm_search_app_state.py b/src/backend/src/tests/integration/test_llm_search_app_state.py new file mode 100644 index 000000000..9a51930b3 --- /dev/null +++ b/src/backend/src/tests/integration/test_llm_search_app_state.py @@ -0,0 +1,261 @@ +"""Integration tests for the Phase 2 adoption-mode preamble. + +Exercises ``LLMSearchManager._process_with_llm`` end-to-end with a +fake OpenAI client. Verifies: + +1. The new ``get_app_state`` tool is registered AND in the always-on + category set so it appears in the LLM's tool list on every call. +2. The system prompt actually sent to the model contains the + adoption-mode preamble matching the live DB state. +3. The default (no preamble) prompt round-trips byte-identical to + Phase 1 when no Phase 2/3 context is available — protects the + Phase 1 integration tests from a silent regression. +4. ``GET /api/llm-search/status`` exposes ``adoption_mode`` so the + frontend can pick mode-aware starter prompts. + +Patterned after ``test_llm_search_concepts.py``. +""" + +# Set test environment variables BEFORE any app imports +import os +os.environ['TESTING'] = 'true' +os.environ['SKIP_STARTUP_TASKS'] = 'true' + +import json +import uuid +from types import SimpleNamespace +from typing import List +from unittest.mock import MagicMock, patch + +import pytest + +from src.db_models.data_products import DataProductDb + + +# --------------------------------------------------------------------------- +# Fake OpenAI client (same shape as the concepts integration test) +# --------------------------------------------------------------------------- + + +def _make_response(*, content, tool_calls): + message = SimpleNamespace(content=content, tool_calls=tool_calls) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class _ScriptedOpenAIClient: + """Records every ``messages=`` payload it sees so a test can + assert against the actual system message sent on a given call.""" + + def __init__(self, script): + self._script = list(script) + self._call_count = 0 + self.captured_messages: List[List[dict]] = [] + self.chat = SimpleNamespace( + completions=SimpleNamespace(create=self._create) + ) + + def _create(self, *, model, messages, tools, tool_choice, max_tokens): + self.captured_messages.append(messages) + if not self._script: + raise AssertionError("ScriptedOpenAIClient ran out of scripted responses") + resp = self._script.pop(0) + self._call_count += 1 + return resp + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def llm_settings(test_settings): + test_settings.LLM_ENABLED = True + test_settings.LLM_ENDPOINT = "test-endpoint" + test_settings.LLM_SYSTEM_PROMPT = None + return test_settings + + +@pytest.fixture +def llm_manager(db_session, llm_settings, mock_workspace_client): + from src.controller.llm_search_manager import LLMSearchManager + return LLMSearchManager( + db=db_session, + settings=llm_settings, + data_products_manager=MagicMock(), + data_contracts_manager=MagicMock(), + semantic_models_manager=MagicMock(), + costs_manager=MagicMock(), + search_manager=MagicMock(), + workspace_client=mock_workspace_client, + ) + + +def _seed_published_product(db_session): + """Push the workspace from 'blank' to 'active' by inserting one + published product. Returns the row so the caller can roll it back + if needed (the autouse db_session rolls back on teardown anyway).""" + product = DataProductDb( + id=str(uuid.uuid4()), + name="Seed Published", + version="1.0.0", + status="active", + publication_scope="organization", + ) + db_session.add(product) + db_session.commit() + return product + + +# --------------------------------------------------------------------------- +# Registry / always-on category +# --------------------------------------------------------------------------- + + +def test_registry_contains_app_state_tool(llm_manager): + assert "get_app_state" in llm_manager._tool_registry.list_tool_names() + + +def test_app_state_category_is_always_on(): + """``get_app_state`` must be available on EVERY chat call so the + LLM can introspect adoption when relevant; the prompt only carries + a static preamble, the tool gives counts.""" + from src.tools.query_classifier import classify_query, ALWAYS_INCLUDED_CATEGORIES + + assert "app_state" in ALWAYS_INCLUDED_CATEGORIES + # Random unrelated query — `app_state` must still surface. + cats = classify_query("show me cost rollups") + assert "app_state" in cats + + +# --------------------------------------------------------------------------- +# System-prompt injection (blank vs active) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_blank_workspace_injects_blank_preamble(llm_manager, mock_test_user): + """With zero published products in the DB, the assembled system + prompt must contain the blank-mode preamble AND the new + ``## Current workspace state`` H2 above the default body.""" + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + first_call_messages = fake_client.captured_messages[0] + sys_text = first_call_messages[0]["content"] + + assert "## Current workspace state" in sys_text, ( + "blank-mode preamble heading is missing — adoption-mode " + "injection probably regressed" + ) + assert "no data products are published yet" in sys_text + # Phase 1 body must still be present below the preamble. + assert "## Tool-first policy for conceptual questions" in sys_text + + +@pytest.mark.asyncio +async def test_active_workspace_injects_active_preamble( + llm_manager, mock_test_user, db_session +): + _seed_published_product(db_session) + + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + sys_text = fake_client.captured_messages[0][0]["content"] + assert "## Current workspace state" in sys_text + assert "has published data products" in sys_text + + +# --------------------------------------------------------------------------- +# Override path still wins +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_llm_system_prompt_override_skips_adoption_preamble( + llm_manager, mock_test_user +): + """``LLM_SYSTEM_PROMPT`` is a full replacement — the adoption-mode + preamble must NOT be prepended on top, otherwise the override + promise breaks.""" + sentinel = "OVERRIDE PROMPT — exact bytes only." + llm_manager._settings.LLM_SYSTEM_PROMPT = sentinel + + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + sys_text = fake_client.captured_messages[0][0]["content"] + assert sys_text == sentinel + + +# --------------------------------------------------------------------------- +# get_status surfaces the mode +# --------------------------------------------------------------------------- + + +def test_get_status_returns_adoption_mode(llm_manager): + """``adoption_mode`` must be on the status payload so the frontend + can switch starter prompts without a separate round-trip.""" + status = llm_manager.get_status() + assert status.adoption_mode in ("blank", "active") + # No published products seeded -> blank. + assert status.adoption_mode == "blank" + + +def test_get_status_returns_active_after_publish(llm_manager, db_session): + _seed_published_product(db_session) + status = llm_manager.get_status() + assert status.adoption_mode == "active" + + +# --------------------------------------------------------------------------- +# Snapshot failure degrades gracefully +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_snapshot_failure_falls_back_to_default_prompt( + llm_manager, mock_test_user +): + """If the snapshot raises, the manager must log and proceed with + ``adoption_mode=None`` — the chat must still succeed and the + prompt sent must be the Phase 1 default (no preamble heading).""" + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch( + "src.tools.app_state.get_adoption_snapshot", + side_effect=RuntimeError("snapshot broke"), + ): + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + response = await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + assert response.message.content + sys_text = fake_client.captured_messages[0][0]["content"] + # No preamble when the snapshot failed. + assert "## Current workspace state" not in sys_text + # But the Phase 1 body must still be present. + assert "## Tool-first policy for conceptual questions" in sys_text diff --git a/src/backend/src/tests/integration/test_llm_search_handbook.py b/src/backend/src/tests/integration/test_llm_search_handbook.py new file mode 100644 index 000000000..a9f113447 --- /dev/null +++ b/src/backend/src/tests/integration/test_llm_search_handbook.py @@ -0,0 +1,264 @@ +"""Integration tests for the Ask Ontos copilot's handbook-grounding path. + +These tests exercise ``LLMSearchManager._process_with_llm`` end-to-end +with a fake OpenAI client. Why not hit ``POST /api/llm-search/chat`` +through the TestClient? Two reasons: + +1. The chat route depends on ``get_obo_workspace_client`` and the audit + manager pipeline — neither is interesting for verifying that the + new tool is wired into the registry and that the new prompt is in + the message stream. +2. Patching at the manager level lets us script the LLM's tool-call + sequence deterministically (call ``search_ontos_handbook`` -> read + the result -> emit a final text response). That sequence is what + we're actually trying to certify. + +The fake client returns whatever its ``script`` says next, so a single +test can simulate multiple LLM iterations. +""" + +# Set test environment variables BEFORE any app imports +import os +os.environ['TESTING'] = 'true' +os.environ['SKIP_STARTUP_TASKS'] = 'true' + +import json +from types import SimpleNamespace +from typing import Any, List +from unittest.mock import MagicMock, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Fake OpenAI client +# --------------------------------------------------------------------------- + + +def _make_tool_call(call_id: str, name: str, arguments: dict) -> SimpleNamespace: + """Build an object shaped like the OpenAI SDK's tool_call attribute + tree (id, function.name, function.arguments). The manager only + reads those four attributes.""" + function = SimpleNamespace( + name=name, + arguments=json.dumps(arguments), + ) + return SimpleNamespace(id=call_id, function=function) + + +def _make_response(*, content: str | None, tool_calls: List | None) -> SimpleNamespace: + """Build an object shaped like the OpenAI ``ChatCompletion`` result + that the manager consumes — only ``choices[0].message.{content, + tool_calls}`` matter.""" + message = SimpleNamespace(content=content, tool_calls=tool_calls) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class _ScriptedOpenAIClient: + """Returns scripted responses in order. Records every ``messages=`` + payload it sees so the test can assert against the system prompt + that was actually sent.""" + + def __init__(self, script: List[SimpleNamespace]): + self._script = list(script) + self._call_count = 0 + self.captured_messages: List[List[dict]] = [] + # Mirror the OpenAI SDK shape: client.chat.completions.create(...) + self.chat = SimpleNamespace( + completions=SimpleNamespace(create=self._create) + ) + + def _create(self, *, model, messages, tools, tool_choice, max_tokens): + self.captured_messages.append(messages) + if not self._script: + raise AssertionError( + f"_ScriptedOpenAIClient ran out of scripted responses on " + f"call #{self._call_count + 1}" + ) + resp = self._script.pop(0) + self._call_count += 1 + return resp + + +# --------------------------------------------------------------------------- +# Manager fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture +def llm_settings(test_settings): + """Override the standard test_settings to enable LLM features and + leave LLM_SYSTEM_PROMPT unset so we exercise the default-prompt + branch.""" + test_settings.LLM_ENABLED = True + test_settings.LLM_ENDPOINT = "test-endpoint" + test_settings.LLM_SYSTEM_PROMPT = None + return test_settings + + +@pytest.fixture +def llm_manager(db_session, llm_settings, mock_workspace_client): + """Build a real LLMSearchManager with mocked downstream managers. + + The integration boundary we care about is: + manager -> tool registry -> SearchOntosHandbookTool -> filesystem + + Nothing else needs to be real. + """ + from src.controller.llm_search_manager import LLMSearchManager + + return LLMSearchManager( + db=db_session, + settings=llm_settings, + data_products_manager=MagicMock(), + data_contracts_manager=MagicMock(), + semantic_models_manager=MagicMock(), + costs_manager=MagicMock(), + search_manager=MagicMock(), + workspace_client=mock_workspace_client, + ) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_registry_contains_handbook_search_tool(llm_manager): + """First-line check: the new tool actually got registered. If this + fails, the rest of the integration is moot.""" + tool_names = llm_manager._tool_registry.list_tool_names() + assert "search_ontos_handbook" in tool_names + + +def test_handbook_category_visible_for_conceptual_query(): + """The query classifier must surface the ``handbook`` category for + a 'what is X' question so the new tool is in the LLM's tool list.""" + from src.tools.query_classifier import classify_query + + cats = classify_query("what is a data steward?") + assert "handbook" in cats + + +def test_handbook_category_visible_for_default_path(): + """Vague queries fall back to DEFAULT_CATEGORIES — ``handbook`` + must be in there so the tool is always at least nominally visible.""" + from src.tools.query_classifier import classify_query, DEFAULT_CATEGORIES + + assert "handbook" in DEFAULT_CATEGORIES + # An empty query takes the default path explicitly. + cats = classify_query("") + assert "handbook" in cats + + +@pytest.mark.asyncio +async def test_chat_calls_handbook_search_tool_for_conceptual_question( + llm_manager, mock_test_user +): + """Script the LLM to ask for ``search_ontos_handbook`` and then + emit a final text answer. Verify that the manager executed the + tool, the tool returned real corpus matches, and the final answer + reached the caller.""" + # The full chat() entry point manages session creation + persistence. + # We script two LLM iterations: first a tool call, then a final reply. + scripted = [ + _make_response( + content=None, + tool_calls=[ + _make_tool_call( + call_id="call_1", + name="search_ontos_handbook", + arguments={"query": "data steward"}, + ) + ], + ), + _make_response( + content=( + "A Data Steward in Ontos is a built-in role that... " + "[Documented]\n" + ), + tool_calls=None, + ), + ] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + response = await llm_manager.chat( + user_message="What is a Data Steward?", + user_id=mock_test_user.email, + debug=True, + ) + + # The manager executed exactly one tool call — our new one. + assert response.tool_calls_executed == 1 + assert response.sources, "expected at least one source recorded" + assert response.sources[0]["tool"] == "search_ontos_handbook" + assert response.sources[0]["success"] is True + + # The final answer reached us verbatim. + assert response.message.content.startswith("A Data Steward in Ontos") + + # Debug payload records the new tool was offered. + assert response.debug is not None + assert "search_ontos_handbook" in response.debug["query_classification"]["tools_provided"] + + +@pytest.mark.asyncio +async def test_chat_sends_grounded_system_prompt(llm_manager, mock_test_user): + """The first message sent to the LLM must be a system message + containing the new tool-first policy. If the prompt source + regresses to the old constant, this fails.""" + scripted = [ + _make_response(content="Brief answer.", tool_calls=None), + ] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="What is a Data Steward?", + user_id=mock_test_user.email, + debug=False, + ) + + # Inspect the messages that were actually sent on the first LLM call. + assert fake_client.captured_messages, "fake client received no calls" + first_call_messages = fake_client.captured_messages[0] + system_msg = first_call_messages[0] + assert system_msg["role"] == "system" + sys_text = system_msg["content"] + + # Phrasing markers from the new prompt — load-bearing changes that + # we want to lock in. + assert "search_ontos_handbook" in sys_text, ( + "system prompt does not mention the new tool — the new prompt is " + "probably not being assembled" + ) + assert "[Documented]" in sys_text, ( + "system prompt missing the three-tier confidence label scheme" + ) + assert "Tier 0" in sys_text, ( + "system prompt missing the new Tier 0 = handbook-corpus framing" + ) + + +@pytest.mark.asyncio +async def test_chat_honors_llm_system_prompt_override(llm_manager, mock_test_user): + """Settings-level override path: ``LLM_SYSTEM_PROMPT`` was previously + dead code. With the new get_system_prompt() helper it must take + precedence and be sent verbatim.""" + sentinel = "OVERRIDE PROMPT — Ontos copilot in override mode." + llm_manager._settings.LLM_SYSTEM_PROMPT = sentinel + + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + first_call_messages = fake_client.captured_messages[0] + assert first_call_messages[0]["role"] == "system" + assert first_call_messages[0]["content"] == sentinel diff --git a/src/backend/src/tests/integration/test_llm_search_user_context.py b/src/backend/src/tests/integration/test_llm_search_user_context.py new file mode 100644 index 000000000..0dad745d0 --- /dev/null +++ b/src/backend/src/tests/integration/test_llm_search_user_context.py @@ -0,0 +1,386 @@ +"""Integration tests for the Phase 3 user-context preamble. + +Exercises the role + page + selected-entity injection through both +``LLMSearchManager.chat`` (directly) and ``get_system_prompt`` (the +assembly point). Verifies: + +1. When the chat request includes ``page_name`` / ``selected_entity``, + the system prompt sent to the model contains a + ``## Current user context`` H2 block with the matching fields. +2. The role label propagates verbatim from the chat call. +3. Phase 3 fields are all optional — a payload without any of them + still hits the same code paths without raising and without + emitting the preamble. + +We don't reach through ``POST /api/llm-search/chat`` for these tests +— the route's role-derivation helper (``_derive_effective_role_label``) +is exercised separately in unit tests; here we focus on what the +manager actually sends to the LLM. +""" + +# Set test environment variables BEFORE any app imports +import os +os.environ['TESTING'] = 'true' +os.environ['SKIP_STARTUP_TASKS'] = 'true' + +from types import SimpleNamespace +from typing import List +from unittest.mock import MagicMock, patch + +import pytest + + +def _make_response(*, content, tool_calls): + message = SimpleNamespace(content=content, tool_calls=tool_calls) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class _ScriptedOpenAIClient: + """Records every ``messages=`` payload it sees so a test can + assert against the actual system message sent.""" + + def __init__(self, script): + self._script = list(script) + self.captured_messages: List[List[dict]] = [] + self.chat = SimpleNamespace( + completions=SimpleNamespace(create=self._create) + ) + + def _create(self, *, model, messages, tools, tool_choice, max_tokens): + self.captured_messages.append(messages) + if not self._script: + raise AssertionError("ScriptedOpenAIClient ran out of scripted responses") + return self._script.pop(0) + + +@pytest.fixture +def llm_settings(test_settings): + test_settings.LLM_ENABLED = True + test_settings.LLM_ENDPOINT = "test-endpoint" + test_settings.LLM_SYSTEM_PROMPT = None + return test_settings + + +@pytest.fixture +def llm_manager(db_session, llm_settings, mock_workspace_client): + from src.controller.llm_search_manager import LLMSearchManager + return LLMSearchManager( + db=db_session, + settings=llm_settings, + data_products_manager=MagicMock(), + data_contracts_manager=MagicMock(), + semantic_models_manager=MagicMock(), + costs_manager=MagicMock(), + search_manager=MagicMock(), + workspace_client=mock_workspace_client, + ) + + +# --------------------------------------------------------------------------- +# Direct prompt-assembly tests +# --------------------------------------------------------------------------- + + +def test_get_system_prompt_renders_full_user_context_block(llm_settings): + """All three Phase 3 inputs should produce a fully-populated + ``## Current user context`` block above the default body.""" + from src.tools.system_prompts import get_system_prompt + + prompt = get_system_prompt( + settings=llm_settings, + role="Data Consumer", + page_name="data-products", + page_url="/data-products/abc", + selected_entity={ + "type": "data_product", + "name": "Customer 360", + "id": "uuid-abc", + }, + ) + assert "## Current user context" in prompt + assert "**Role**: Data Consumer" in prompt + assert "**Currently on**: data-products (/data-products/abc)" in prompt + assert '"Customer 360"' in prompt + assert "uuid-abc" in prompt + # Default body still present below the preamble. + assert "## Tool-first policy for conceptual questions" in prompt + + +def test_get_system_prompt_omits_user_context_when_all_inputs_empty(llm_settings): + """Phase 1 byte-identity contract — when nothing is passed we must + return the default prompt unchanged so existing tests don't see + spurious whitespace / preamble drift.""" + from src.tools.system_prompts import ( + _DEFAULT_SYSTEM_PROMPT, + get_system_prompt, + ) + + prompt = get_system_prompt(settings=llm_settings) + assert prompt == _DEFAULT_SYSTEM_PROMPT + + +def test_get_system_prompt_partial_user_context_still_renders(llm_settings): + """Role alone (no page, no entity) is enough to trigger the + block — partial payloads are common in practice (e.g., home page + where the user has a role but no entity selected).""" + from src.tools.system_prompts import get_system_prompt + + prompt = get_system_prompt(settings=llm_settings, role="Admin") + assert "## Current user context" in prompt + assert "**Role**: Admin" in prompt + # No "Currently on" line because page_name is None. + assert "**Currently on**" not in prompt + assert "**Viewing**" not in prompt + + +def test_user_context_handles_entity_without_id(llm_settings): + """When ``selected_entity`` lacks an id (e.g., user is creating a + new draft), the preamble must still render the type + name + without crashing or emitting a 'None' literal.""" + from src.tools.system_prompts import get_system_prompt + + prompt = get_system_prompt( + settings=llm_settings, + role="Data Producer", + page_name="data-products", + selected_entity={"type": "data_product", "name": "Draft X"}, + ) + assert '"Draft X"' in prompt + assert "id:" not in prompt # no id => no "id: …" suffix + assert "None" not in prompt # never leak a Python None + + +# --------------------------------------------------------------------------- +# End-to-end through LLMSearchManager.chat +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_chat_propagates_user_context_to_system_prompt( + llm_manager, mock_test_user +): + """The chat() entry point must pass role / page / entity through + to the LLM call, which means the system prompt captured by the + fake client should contain matching strings.""" + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + role="Data Consumer", + page_name="data-products", + page_url="/data-products/abc", + selected_entity={ + "type": "data_product", + "name": "Customer 360", + "id": "uuid-abc", + }, + ) + + sys_text = fake_client.captured_messages[0][0]["content"] + assert "## Current user context" in sys_text + assert "Data Consumer" in sys_text + assert "data-products" in sys_text + assert "Customer 360" in sys_text + + +@pytest.mark.asyncio +async def test_chat_without_context_still_works(llm_manager, mock_test_user): + """Backward compatibility: payload with no Phase 3 fields must + still chat successfully and must NOT emit the preamble heading.""" + scripted = [_make_response(content="ok", tool_calls=None)] + fake_client = _ScriptedOpenAIClient(scripted) + + with patch.object(llm_manager, "_get_openai_client", return_value=fake_client): + response = await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + assert response.message.content + sys_text = fake_client.captured_messages[0][0]["content"] + assert "## Current user context" not in sys_text + + +@pytest.mark.asyncio +async def test_chat_context_is_cleared_between_calls(llm_manager, mock_test_user): + """``self._chat_context`` is set by ``chat()`` and must be reset + in a finally block so the next call without context doesn't + inherit the previous call's role / page.""" + # First call WITH context. + scripted_a = [_make_response(content="ok", tool_calls=None)] + fake_a = _ScriptedOpenAIClient(scripted_a) + with patch.object(llm_manager, "_get_openai_client", return_value=fake_a): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + role="Admin", + page_name="settings", + ) + assert llm_manager._chat_context is None + + # Second call WITHOUT context — must not leak "Admin" / "settings". + scripted_b = [_make_response(content="ok", tool_calls=None)] + fake_b = _ScriptedOpenAIClient(scripted_b) + with patch.object(llm_manager, "_get_openai_client", return_value=fake_b): + await llm_manager.chat( + user_message="hello", + user_id=mock_test_user.email, + ) + + sys_text_b = fake_b.captured_messages[0][0]["content"] + assert "Admin" not in sys_text_b or "## Current user context" not in sys_text_b + + +# --------------------------------------------------------------------------- +# Route role-derivation helper +# --------------------------------------------------------------------------- + + +def test_derive_effective_role_label_returns_none_when_no_settings_manager(): + """Defense-in-depth: a missing settings_manager on app.state + (unusual but possible during boot) must not 500 the chat call.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + request = MagicMock() + request.app.state = SimpleNamespace() # no settings_manager attr + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=["admins"], + ) + assert _derive_effective_role_label(request, user) is None + + +def test_derive_effective_role_label_returns_none_for_empty_groups(): + """Anonymous-style call (no groups). Helper should short-circuit.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + request = MagicMock() + request.app.state.settings_manager = MagicMock() + request.app.state.settings_manager.list_app_roles.return_value = [ + SimpleNamespace(name="Admin", assigned_groups=["admins"]), + ] + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=[], + ) + assert _derive_effective_role_label(request, user) is None + + +def test_derive_effective_role_label_intersects_groups(): + """Group intersection (case-insensitive) -> role name string.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + request = MagicMock() + request.app.state.settings_manager = MagicMock() + request.app.state.settings_manager.list_app_roles.return_value = [ + SimpleNamespace(name="Admin", assigned_groups=["admins"]), + SimpleNamespace(name="Data Producer", assigned_groups=["data-producers"]), + SimpleNamespace(name="Data Consumer", assigned_groups=["data-consumers"]), + ] + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=["Data-Producers"], # mixed case — must still match + ) + assert _derive_effective_role_label(request, user) == "Data Producer" + + +def test_derive_effective_role_label_joins_multiple_roles(): + """A user in multiple role groups returns a comma-joined label.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + request = MagicMock() + request.app.state.settings_manager = MagicMock() + request.app.state.settings_manager.list_app_roles.return_value = [ + SimpleNamespace(name="Admin", assigned_groups=["admins"]), + SimpleNamespace(name="Data Producer", assigned_groups=["data-producers"]), + ] + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=["admins", "data-producers"], + ) + label = _derive_effective_role_label(request, user) + assert "Admin" in label + assert "Data Producer" in label + assert ", " in label + + +def test_derive_effective_role_label_honors_applied_role_override(): + """When the user has clicked an "apply role" override in the UI, + the override role wins over the group-intersection role — even + if the user's groups would otherwise resolve to Admin.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + admin_role = SimpleNamespace( + id="role-admin", name="Admin", assigned_groups=["admins"] + ) + producer_role = SimpleNamespace( + id="role-producer", name="Data Producer", assigned_groups=["data-producers"] + ) + + request = MagicMock() + request.app.state.settings_manager = MagicMock() + request.app.state.settings_manager.list_app_roles.return_value = [ + admin_role, + producer_role, + ] + request.app.state.settings_manager.get_applied_role_override_for_user.return_value = ( + "role-producer" + ) + + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=["admins"], # would normally match Admin + ) + assert _derive_effective_role_label(request, user) == "Data Producer" + + +def test_derive_effective_role_label_falls_back_when_override_id_unknown(): + """If the persisted override id doesn't match any current role + (e.g., the role was deleted), fall back to group intersection + rather than erroring or returning None.""" + from src.routes.llm_search_routes import _derive_effective_role_label + from src.models.users import UserInfo + + admin_role = SimpleNamespace( + id="role-admin", name="Admin", assigned_groups=["admins"] + ) + + request = MagicMock() + request.app.state.settings_manager = MagicMock() + request.app.state.settings_manager.list_app_roles.return_value = [admin_role] + request.app.state.settings_manager.get_applied_role_override_for_user.return_value = ( + "role-deleted-long-ago" + ) + + user = UserInfo( + email="u@example.com", + username="u", + user="u", + ip="127.0.0.1", + groups=["admins"], + ) + assert _derive_effective_role_label(request, user) == "Admin" diff --git a/src/backend/src/tests/unit/test_get_app_state_tool.py b/src/backend/src/tests/unit/test_get_app_state_tool.py new file mode 100644 index 000000000..b60981aa1 --- /dev/null +++ b/src/backend/src/tests/unit/test_get_app_state_tool.py @@ -0,0 +1,250 @@ +"""Unit tests for ``GetAppStateTool`` and the shared adoption snapshot. + +Two layers under test: + +1. ``get_adoption_snapshot`` — the pure-function helper that reads + counts via SQLAlchemy and derives a binary ``adoption_mode``. Same + function powers the tool result and the system-prompt preamble, so + we test it directly to make sure both paths agree. +2. ``GetAppStateTool.execute`` — the LLM-callable wrapper. Covers + metadata, the no-param call shape, and graceful degrade on a DB + error so the model still gets a usable refusal rather than a 500. + +The tests use the in-memory SQLite fixture wired by ``conftest.py`` +(``db_session``) so counts are real, not mocked. ``adoption_mode`` +hinges on *published* products (``publication_scope`` non-null AND +not the literal string 'none'), so the "active" fixture sets that +column explicitly. +""" + +# Set test environment variables BEFORE any app imports +import os +os.environ['TESTING'] = 'true' +os.environ['SKIP_STARTUP_TASKS'] = 'true' + +import uuid +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy.orm import Session + +from src.db_models.data_contracts import DataContractDb +from src.db_models.data_domains import DataDomain +from src.db_models.data_products import DataProductDb +from src.tools.app_state import ( + ADOPTION_MODE_ACTIVE, + ADOPTION_MODE_BLANK, + GetAppStateTool, + get_adoption_snapshot, +) +from src.tools.base import ToolContext + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_product( + *, + name: str = "Test Product", + publication_scope=None, +) -> DataProductDb: + """Build a minimal DataProductDb row. Only the fields exercised by + the snapshot helper are populated; everything else relies on + column defaults. ``DataProductDb.description`` is a relationship, + not a column — don't set it here.""" + kwargs = dict( + id=str(uuid.uuid4()), + name=name, + version="1.0.0", + status="draft", + ) + if publication_scope is not None: + kwargs["publication_scope"] = publication_scope + return DataProductDb(**kwargs) + + +def _make_ctx(db: Session) -> ToolContext: + return ToolContext(db=db, settings=MagicMock()) + + +# --------------------------------------------------------------------------- +# Tool metadata +# --------------------------------------------------------------------------- + + +def test_tool_metadata(): + """Lock down the name / category / no-scope-gate contract. The + registry dispatches on name and the query classifier dispatches on + category, so both are load-bearing.""" + tool = GetAppStateTool() + assert tool.name == "get_app_state" + assert tool.category == "app_state" + # Counts aren't sensitive, so we deliberately drop the default + # admin-only scope inherited from BaseTool. + assert tool.required_scope is None + assert tool.required_params == [] + # The tool takes no parameters; an empty parameters dict is what + # surfaces in the OpenAI function schema. + assert tool.parameters == {} + + +# --------------------------------------------------------------------------- +# get_adoption_snapshot — blank workspace +# --------------------------------------------------------------------------- + + +def test_blank_workspace_yields_blank_mode(db_session: Session): + """A workspace with zero rows in every entity table is the + archetypal 'blank' state. ``adoption_mode`` must be 'blank' and + every count must be a non-negative int.""" + snapshot = get_adoption_snapshot(db_session) + assert snapshot["adoption_mode"] == ADOPTION_MODE_BLANK + + counts = snapshot["counts"] + for key in ( + "data_products_total", + "data_products_published", + "data_contracts_total", + "domains_total", + "teams_total", + "projects_total", + "roles_total", + ): + assert key in counts + assert isinstance(counts[key], int) + assert counts[key] >= 0 + + # ISO-8601 string with timezone offset (the function uses UTC). + assert "T" in snapshot["computed_at"] + + +def test_draft_only_workspace_still_blank(db_session: Session): + """A workspace with DRAFT products only is still 'blank' — the + distinction is about *published* assets, not whether anyone has + started any work.""" + db_session.add(_make_product(name="Draft A")) + db_session.add(_make_product(name="Draft B", publication_scope="none")) + db_session.commit() + + snapshot = get_adoption_snapshot(db_session) + assert snapshot["adoption_mode"] == ADOPTION_MODE_BLANK + assert snapshot["counts"]["data_products_total"] == 2 + assert snapshot["counts"]["data_products_published"] == 0 + + +def test_publication_scope_literal_none_treated_as_unpublished(db_session: Session): + """Belt-and-suspenders: ``publication_scope='none'`` (literal + string, mirrors the marketplace filter) must NOT flip mode to + active. Some legacy rows have the string form rather than NULL.""" + db_session.add(_make_product(name="Legacy", publication_scope="none")) + db_session.add(_make_product(name="Legacy Upper", publication_scope="NONE")) + db_session.commit() + + snapshot = get_adoption_snapshot(db_session) + assert snapshot["adoption_mode"] == ADOPTION_MODE_BLANK + assert snapshot["counts"]["data_products_published"] == 0 + + +# --------------------------------------------------------------------------- +# get_adoption_snapshot — active workspace +# --------------------------------------------------------------------------- + + +def test_published_product_flips_mode_to_active(db_session: Session): + """A single published product is enough to flip mode to 'active'.""" + db_session.add(_make_product(name="Published A", publication_scope="organization")) + db_session.add(_make_product(name="Draft", publication_scope=None)) + db_session.commit() + + snapshot = get_adoption_snapshot(db_session) + assert snapshot["adoption_mode"] == ADOPTION_MODE_ACTIVE + assert snapshot["counts"]["data_products_total"] == 2 + assert snapshot["counts"]["data_products_published"] == 1 + + +def test_counts_cover_multiple_entity_types(db_session: Session): + """Spot-check that the snapshot picks up domain + contract counts + in addition to products, so we don't silently lose entities if + schema changes drop a relationship.""" + db_session.add(_make_product(publication_scope="organization")) + db_session.add( + DataDomain( + id=str(uuid.uuid4()), + name="Finance", + description="finance domain", + created_by="owner@example.com", + ) + ) + db_session.add( + DataContractDb( + id=str(uuid.uuid4()), + name="Test Contract", + version="1.0.0", + status="draft", + ) + ) + db_session.commit() + + snapshot = get_adoption_snapshot(db_session) + assert snapshot["adoption_mode"] == ADOPTION_MODE_ACTIVE + assert snapshot["counts"]["domains_total"] >= 1 + assert snapshot["counts"]["data_contracts_total"] >= 1 + + +# --------------------------------------------------------------------------- +# GetAppStateTool.execute +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_execute_returns_success_with_mode(db_session: Session): + """End-to-end through the BaseTool surface — should return the + snapshot under ``data`` with ``success=True``.""" + tool = GetAppStateTool() + result = await tool.execute(_make_ctx(db_session)) + assert result.success is True + assert result.data["adoption_mode"] in (ADOPTION_MODE_BLANK, ADOPTION_MODE_ACTIVE) + assert "counts" in result.data + assert "computed_at" in result.data + + +@pytest.mark.asyncio +async def test_execute_ignores_unexpected_kwargs(db_session: Session): + """LLMs sometimes pass empty dicts or stray fields. The tool takes + no params; it must not raise when handed extras.""" + tool = GetAppStateTool() + result = await tool.execute(_make_ctx(db_session), random_field="ignored") + assert result.success is True + + +@pytest.mark.asyncio +async def test_execute_handles_snapshot_failure_gracefully(): + """If the underlying snapshot raises, ``execute`` must convert to + a structured tool error rather than letting the exception escape + (the LLM loop would otherwise abort the whole chat).""" + tool = GetAppStateTool() + with patch( + "src.tools.app_state.get_adoption_snapshot", + side_effect=RuntimeError("db down"), + ): + ctx = ToolContext(db=MagicMock(), settings=MagicMock()) + result = await tool.execute(ctx) + + assert result.success is False + assert "db down" in (result.error or "") + + +# --------------------------------------------------------------------------- +# Snapshot contract — used by LlmSearchManager pre-fetch +# --------------------------------------------------------------------------- + + +def test_snapshot_shape_is_stable(db_session: Session): + """Lock the top-level keys returned by ``get_adoption_snapshot`` — + the manager pre-fetch path destructures ``adoption_mode`` and + ``counts`` directly, so a key rename would silently disable mode + awareness.""" + snapshot = get_adoption_snapshot(db_session) + assert set(snapshot.keys()) == {"adoption_mode", "counts", "computed_at"} diff --git a/src/backend/src/tests/unit/test_search_ontos_handbook_tool.py b/src/backend/src/tests/unit/test_search_ontos_handbook_tool.py new file mode 100644 index 000000000..eeaac653e --- /dev/null +++ b/src/backend/src/tests/unit/test_search_ontos_handbook_tool.py @@ -0,0 +1,255 @@ +"""Unit tests for ``SearchOntosHandbookTool``. + +Exercises: + +- Empty query => failure. +- Real corpus queries (the tool resolves ``docs/handbook/`` relative to + the repo root; that directory ships with the branch under test). +- Known-handbook lookups land in the right file. +- No-match queries return ``success=True`` with an empty match list and + a friendly ``message`` field. +- Anchor extraction handles the ``{#kebab-case}`` syntax used across + the corpus, with a slugified fallback when a section omits an anchor. +- Graceful degrade when ``docs/handbook/`` is absent (deployed-app case). + +We don't mock the corpus contents — the docs are read-only inputs to +this branch and exercising them directly is the most realistic test we +can write without bringing in fixtures that drift from the real files. +""" + +# Set test environment variables BEFORE any app imports +import os +os.environ['TESTING'] = 'true' +os.environ['SKIP_STARTUP_TASKS'] = 'true' + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from src.tools.base import ToolContext +from src.tools.handbook import ( + SearchOntosHandbookTool, + _parse_sections, + _slugify_fallback_anchor, + _resolve_handbook_dir, +) + + +def _make_ctx() -> ToolContext: + """Handbook search ignores ``ctx`` entirely — it reads the filesystem + directly. A skeletal mock is enough.""" + return ToolContext(db=MagicMock(), settings=MagicMock()) + + +@pytest.fixture +def tool() -> SearchOntosHandbookTool: + return SearchOntosHandbookTool() + + +# --------------------------------------------------------------------------- +# Smoke / preconditions +# --------------------------------------------------------------------------- + + +def test_corpus_is_resolvable_in_this_test_run(): + """Sanity check: the test environment can locate the corpus. If + this fails, every other test in this module is meaningless — flag + it loudly rather than hide behind silent skips.""" + handbook_dir = _resolve_handbook_dir() + assert handbook_dir is not None, ( + "docs/handbook/ not found at the expected location. The path " + "resolution math in src/backend/src/tools/handbook.py may have " + "drifted relative to the repo layout." + ) + assert handbook_dir.is_dir() + + +def test_tool_metadata(tool: SearchOntosHandbookTool): + """The registered tool name and category are load-bearing — the + query classifier dispatches on the category and the registry + surfaces the tool by name. Lock both down.""" + assert tool.name == "search_ontos_handbook" + assert tool.category == "handbook" + assert "query" in tool.required_params + # No scope gate — handbook docs are public grounding material. + assert tool.required_scope is None + + +# --------------------------------------------------------------------------- +# Empty / invalid input +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_empty_query_returns_error(tool: SearchOntosHandbookTool): + result = await tool.execute(_make_ctx(), query="") + assert result.success is False + assert "non-empty" in (result.error or "").lower() + + +@pytest.mark.asyncio +async def test_whitespace_only_query_returns_error(tool: SearchOntosHandbookTool): + result = await tool.execute(_make_ctx(), query=" \t \n") + assert result.success is False + + +# --------------------------------------------------------------------------- +# Known-topic queries (corpus-backed) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_data_steward_query_hits_roles_doc(tool: SearchOntosHandbookTool): + """'data steward' is a built-in role; it must land in roles-and-rbac.md.""" + result = await tool.execute(_make_ctx(), query="data steward") + assert result.success is True + matches = result.data["matches"] + assert matches, "expected at least one match for 'data steward'" + + files_returned = {m["file"] for m in matches} + assert "roles-and-rbac.md" in files_returned, ( + f"'data steward' did not match anything in roles-and-rbac.md " + f"(got files: {files_returned})" + ) + + # Every match must carry a citable source_uri of the form file.md#anchor + for m in matches: + assert m["source_uri"].endswith( + f"#{m['anchor']}" + ), f"source_uri shape wrong: {m['source_uri']}" + assert m["source_uri"].startswith(m["file"]) + + +@pytest.mark.asyncio +async def test_delivery_mode_query_hits_delivery_doc(tool: SearchOntosHandbookTool): + """'delivery mode' is the topic of delivery-and-propagation.md.""" + result = await tool.execute(_make_ctx(), query="delivery mode") + assert result.success is True + matches = result.data["matches"] + assert matches, "expected matches for 'delivery mode'" + + # Top-scored result should come from delivery-and-propagation.md. + top_file = matches[0]["file"] + assert top_file == "delivery-and-propagation.md", ( + f"expected delivery-and-propagation.md as top match, got {top_file}" + ) + + +@pytest.mark.asyncio +async def test_what_is_agreement_query_hits_agreement_workflow( + tool: SearchOntosHandbookTool, +): + """A common conceptual question — exercise the title-match path.""" + result = await tool.execute(_make_ctx(), query="agreement workflow") + assert result.success is True + matches = result.data["matches"] + assert matches + files_returned = {m["file"] for m in matches} + assert "agreement-workflow.md" in files_returned + + +@pytest.mark.asyncio +async def test_max_results_caps_returned_matches(tool: SearchOntosHandbookTool): + """A broad query against the corpus must respect max_results.""" + result = await tool.execute(_make_ctx(), query="role", max_results=3) + assert result.success is True + assert len(result.data["matches"]) <= 3 + + +@pytest.mark.asyncio +async def test_max_results_is_clamped(tool: SearchOntosHandbookTool): + """Excessive max_results values are clamped (not rejected).""" + result = await tool.execute(_make_ctx(), query="role", max_results=999) + assert result.success is True + assert len(result.data["matches"]) <= 10 # implementation cap + + +# --------------------------------------------------------------------------- +# No-match queries +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_no_match_query_returns_success_with_empty_list( + tool: SearchOntosHandbookTool, +): + """A query that can't possibly match the corpus (a wholly unrelated + SRE topic) must return success=True with an empty match list and a + friendly message — NOT an error. The LLM uses this signal to fall + back to its refusal template.""" + # Truly off-topic tokens — picked to avoid words that appear in the + # corpus (the docs do mention "deployment", "chart", "step", etc., + # which would yield spurious low-score matches). + result = await tool.execute( + _make_ctx(), + query="zygomorphic platypus rutabaga marshmallow", + ) + assert result.success is True + assert result.data["matches"] == [] + assert "No matching handbook entries found" in result.data.get("message", "") + assert result.data["total_files_searched"] > 0 + + +# --------------------------------------------------------------------------- +# Anchor extraction +# --------------------------------------------------------------------------- + + +def test_kebab_anchor_extracted_from_explicit_syntax(tmp_path: Path): + """Sections with ``## Title {#anchor-id}`` headings must surface + 'anchor-id' as the anchor.""" + md = tmp_path / "sample.md" + md.write_text( + "# Doc\n\n" + "Intro paragraph.\n\n" + "## The permission model {#permission-model}\n\n" + "Body of the permission-model section.\n\n" + "### Access levels {#access-levels}\n\n" + "Body of the access-levels section.\n", + encoding="utf-8", + ) + sections = _parse_sections(md) + anchors = {s.title: s.anchor for s in sections} + assert anchors["The permission model"] == "permission-model" + assert anchors["Access levels"] == "access-levels" + + +def test_section_without_explicit_anchor_falls_back_to_slug(tmp_path: Path): + md = tmp_path / "noanchor.md" + md.write_text( + "# Doc\n\n" + "Intro.\n\n" + "## A Section Without An Anchor\n\n" + "Some body text here.\n", + encoding="utf-8", + ) + sections = _parse_sections(md) + bare = next(s for s in sections if s.title == "A Section Without An Anchor") + # Anchor is empty in the parsed Section dataclass... + assert bare.anchor == "" + # ...but the tool slugifies it for source_uri rendering. + assert _slugify_fallback_anchor(bare.title) == "a-section-without-an-anchor" + + +# --------------------------------------------------------------------------- +# Graceful degrade when corpus is missing +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_missing_corpus_returns_empty_matches_not_an_error( + tool: SearchOntosHandbookTool, +): + """Deployed Apps may not package docs/handbook/. The tool must + still return ``success=True`` so the LLM can fall back to its + refusal template gracefully.""" + with patch( + "src.tools.handbook._resolve_handbook_dir", + return_value=None, + ): + result = await tool.execute(_make_ctx(), query="data steward") + assert result.success is True + assert result.data["matches"] == [] + assert result.data["total_files_searched"] == 0 + assert "not available" in result.data.get("message", "").lower() diff --git a/src/backend/src/tools/app_state.py b/src/backend/src/tools/app_state.py new file mode 100644 index 000000000..9d9761d39 --- /dev/null +++ b/src/backend/src/tools/app_state.py @@ -0,0 +1,180 @@ +""" +App-state snapshot tool for the Ask Ontos copilot. + +Computes a small, cheap "is this workspace empty or active?" snapshot +that drives two things at runtime: + +1. **Tool-callable introspection.** The LLM can invoke + ``get_app_state`` when a user asks "how many data products do we + have?", "is this a fresh install?", "what's our adoption?" etc. +2. **System-prompt adoption-mode preamble.** ``LlmSearchManager`` + calls :func:`get_adoption_snapshot` ONCE per chat request and feeds + the derived ``adoption_mode`` into ``get_system_prompt``. That makes + every conceptual answer mode-aware (onboarding tone vs operational + tone) without requiring the LLM to call the tool explicitly. + +Both call sites share :func:`get_adoption_snapshot` so the tool result +and the prompt preamble can never disagree. The snapshot is built with +direct SQLAlchemy ``count()`` queries against the entity tables — no +manager indirection (managers fetch full rows, expand relationships, +and apply permission cascades; we just want counts). + +The binary `adoption_mode` (`blank` vs `active`) intentionally hinges +on *published* data products, not draft ones. A workspace with 30 +draft contracts and zero published products is still "blank" from the +end-user perspective: nothing's been released to consumers yet. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict + +from sqlalchemy import func +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from src.common.logging import get_logger +from src.db_models.data_contracts import DataContractDb +from src.db_models.data_domains import DataDomain +from src.db_models.data_products import DataProductDb +from src.db_models.projects import ProjectDb +from src.db_models.settings import AppRoleDb +from src.db_models.teams import TeamDb +from src.tools.base import BaseTool, ToolContext, ToolResult + +logger = get_logger(__name__) + + +# Adoption-mode literals — string constants rather than an Enum so the +# values flow straight through Pydantic / JSON without conversion. +ADOPTION_MODE_BLANK = "blank" +ADOPTION_MODE_ACTIVE = "active" + + +def _count(db: Session, model_cls) -> int: + """Run ``SELECT count(*) FROM `` and return an int. + + Wrapped so a single table's count error doesn't bring down the whole + snapshot — adoption mode is best-effort, not load-bearing for auth. + """ + try: + # ``func.count()`` against the PK column is the dialect-portable + # cheap form (vs ``SELECT *`` which fetches rows). + pk_col = list(model_cls.__table__.primary_key.columns)[0] + return int(db.query(func.count(pk_col)).scalar() or 0) + except SQLAlchemyError as e: + logger.warning( + f"[get_app_state] count({model_cls.__tablename__}) failed: {e}; " + "returning 0" + ) + return 0 + + +def _count_published_products(db: Session) -> int: + """Count data products that are visible to consumers. + + Publication state is encoded in ``DataProductDb.publication_scope`` + — a string column where ``None`` / ``'none'`` means "not published" + and any other value (e.g. ``'organization'``, ``'public'``) means + "published with that scope". This mirrors the marketplace filter in + ``DataProductsManager.get_published_products``. + """ + try: + return int( + db.query(func.count(DataProductDb.id)) + .filter( + DataProductDb.publication_scope.isnot(None), + func.lower(DataProductDb.publication_scope) != "none", + ) + .scalar() + or 0 + ) + except SQLAlchemyError as e: + logger.warning( + f"[get_app_state] count(published data_products) failed: {e}; " + "returning 0" + ) + return 0 + + +def get_adoption_snapshot(db: Session) -> Dict[str, Any]: + """Compute the shared app-state snapshot. + + Returns a dict with two top-level keys: + + - ``adoption_mode`` (str) — ``"blank"`` when no products are + published, ``"active"`` otherwise. + - ``counts`` (dict) — per-entity counts. Counts are non-negative + ints (best-effort: a query error yields 0 for that one entity, + not a raised exception). + - ``computed_at`` (str) — ISO-8601 UTC timestamp for cache-busting + and audit. Not used by the prompt path; surfaced via the tool. + """ + counts: Dict[str, int] = { + "data_products_total": _count(db, DataProductDb), + "data_products_published": _count_published_products(db), + "data_contracts_total": _count(db, DataContractDb), + "domains_total": _count(db, DataDomain), + "teams_total": _count(db, TeamDb), + "projects_total": _count(db, ProjectDb), + "roles_total": _count(db, AppRoleDb), + } + + adoption_mode = ( + ADOPTION_MODE_BLANK + if counts["data_products_published"] == 0 + else ADOPTION_MODE_ACTIVE + ) + + return { + "adoption_mode": adoption_mode, + "counts": counts, + "computed_at": datetime.now(timezone.utc).isoformat(), + } + + +class GetAppStateTool(BaseTool): + """Introspect the current workspace's adoption state. + + Returns total / published counts for the headline entities and a + binary ``adoption_mode`` flag (``blank`` if no products are + published, otherwise ``active``). Useful for questions like + "how many data products do we have?", "is anyone using Ontos + yet?", "what's our adoption?". + + No parameters. No scope gate — counts are not sensitive. + """ + + name = "get_app_state" + category = "app_state" + description = ( + "Get a snapshot of the current Ontos workspace's adoption " + "state: total and published counts of data products, data " + "contracts, domains, teams, projects, and roles. Also returns " + "a binary 'adoption_mode' ('blank' if no data products are " + "published, 'active' otherwise) — use this to tailor your " + "framing (onboarding vs operational). Takes no parameters." + ) + parameters: Dict[str, Any] = {} + required_params: list = [] + required_scope = None # type: ignore[assignment] + + async def execute(self, ctx: ToolContext, **kwargs) -> ToolResult: + # No params; ignore kwargs (the LLM occasionally passes empty + # dicts or unrelated keys, which we want to tolerate silently). + logger.info("[get_app_state] computing adoption snapshot") + try: + snapshot = get_adoption_snapshot(ctx.db) + except Exception as e: + logger.error(f"[get_app_state] snapshot failed: {e}", exc_info=True) + return ToolResult( + success=False, + error=f"failed to compute app-state snapshot: {type(e).__name__}: {e}", + ) + + logger.info( + f"[get_app_state] mode={snapshot['adoption_mode']} " + f"counts={snapshot['counts']}" + ) + return ToolResult(success=True, data=snapshot) diff --git a/src/backend/src/tools/handbook.py b/src/backend/src/tools/handbook.py new file mode 100644 index 000000000..06fd1bd0e --- /dev/null +++ b/src/backend/src/tools/handbook.py @@ -0,0 +1,411 @@ +""" +Handbook-search tool for the Ask Ontos copilot. + +Grounds the LLM in the curated `docs/handbook/` corpus so it can answer +"what is X?" / "how does Y work?" / "what's the difference between A and B?" +questions from authoritative project documentation rather than training +knowledge. Each result is a section excerpt with a stable +`file.md#anchor` source URI the model can cite. + +The handbook is treated as read-only at runtime; the tool walks the +directory on every call (it's small — 13 files, ~100KB) and tokenizes +the query for a simple title/anchor/body-frequency match. Intentionally +no embeddings, no index — keeps the deployment surface zero. + +Naming note: this corpus used to be called "concepts", but "Concept" is +already an Ontos ontology entity (an RDFS class / SKOS concept in the +knowledge graph). To avoid overloading the noun in code, API surface, +and docs, the LLM-grounding markdown corpus is now "handbook". +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from src.common.logging import get_logger +from src.tools.base import BaseTool, ToolContext, ToolResult + +logger = get_logger(__name__) + + +# --------------------------------------------------------------------------- +# Path resolution +# --------------------------------------------------------------------------- + +# The corpus lives at `docs/handbook/` relative to the repository root. +# Depending on how the app is laid out at runtime, the repo root is at a +# different number of parents above this file: +# - Local dev: /src/backend/src/tools/handbook.py (5 parents up) +# - Deployed app: /backend/src/tools/handbook.py (4 parents up) +# We walk a small range and pick the first parent that has `docs/handbook/`. +# An explicit env var `ONTOS_HANDBOOK_DIR` overrides the search for +# deployments that ship the corpus to a non-standard location. + +_THIS_FILE = Path(__file__).resolve() +_HANDBOOK_DIR_ENV_VAR = "ONTOS_HANDBOOK_DIR" + + +def _resolve_handbook_dir() -> Optional[Path]: + """Return the handbook directory if present on disk, else None. + + Resolution order: + 1. ``ONTOS_HANDBOOK_DIR`` env var (explicit override). + 2. Walk parents 2..6 above this file looking for ``docs/handbook/``. + 3. Return ``None`` — the tool degrades gracefully (no matches). + """ + override = os.environ.get(_HANDBOOK_DIR_ENV_VAR) + if override: + candidate = Path(override).expanduser().resolve() + if candidate.is_dir(): + return candidate + logger.warning( + "%s=%s but the directory does not exist; falling back to search.", + _HANDBOOK_DIR_ENV_VAR, + override, + ) + + for depth in range(2, 7): + try: + base = _THIS_FILE.parents[depth] + except IndexError: + break + candidate = base / "docs" / "handbook" + if candidate.is_dir(): + return candidate + return None + + +# --------------------------------------------------------------------------- +# Section parsing +# --------------------------------------------------------------------------- + +# Match a markdown heading line of level 2-4, optionally with an explicit +# {#anchor-id} suffix. Example matches: +# "## The permission model {#permission-model}" +# "### Access levels {#access-levels}" +# "#### Data Steward {#data-steward}" +# "## Some heading without an anchor" +_HEADING_RE = re.compile( + r"^(?P#{2,4})\s+" + r"(?P.+?)" + r"(?:\s+\{#(?P<anchor>[a-z0-9][a-z0-9\-]*)\})?\s*$" +) + + +@dataclass +class _Section: + file: str # relative filename, e.g. "roles-and-rbac.md" + title: str # heading text without the {#anchor} suffix + anchor: str # explicit anchor id, or "" if none was given + body: str # body text between this heading and the next h2/h3 + + +def _slugify_fallback_anchor(title: str) -> str: + """Build a kebab-case anchor from a title when none was declared. + + The corpus convention is to use explicit ``{#anchor}`` syntax, but a + few sections may omit it. We fall back to a slug so the citation + still resolves. + """ + slug = re.sub(r"[^a-z0-9\s\-]", "", title.lower()) + slug = re.sub(r"\s+", "-", slug).strip("-") + return slug or "section" + + +def _parse_sections(file_path: Path) -> List[_Section]: + """Split a markdown file into h2/h3/h4 sections. + + Each section starts at an h2, h3, or h4 heading and runs until the + next h2/h3/h4 (or EOF). Content before the first heading (typically + the h1 + intro paragraph) is captured as a synthetic intro section + using the filename stem as title and "" as anchor — this is what + lets queries like "what is mcp" match doc-level intros even when + there isn't an explicit h2 for it. + + H4 inclusion matters after the corpus restructure into + "What you see in Ontos" / "Under the hood": named entities like + individual roles (e.g. `#### Data Steward`) are nested under + common parent h3s and would otherwise dilute into a long mixed + section body. + """ + try: + text = file_path.read_text(encoding="utf-8") + except OSError as e: + logger.warning(f"[search_ontos_handbook] Could not read {file_path}: {e}") + return [] + + sections: List[_Section] = [] + current_title: Optional[str] = None + current_anchor: str = "" + current_body: List[str] = [] + intro_body: List[str] = [] + intro_title: Optional[str] = None + + rel_name = file_path.name + + for line in text.splitlines(): + # Capture H1 as the intro section title (skip the "# " prefix). + if intro_title is None and line.startswith("# ") and not line.startswith("## "): + intro_title = line[2:].strip() + continue + + m = _HEADING_RE.match(line) + if m and m.group("hashes") in ("##", "###", "####"): + # Flush previous section + if current_title is not None: + sections.append(_Section( + file=rel_name, + title=current_title, + anchor=current_anchor, + body="\n".join(current_body).strip(), + )) + elif intro_title is not None and intro_body: + # We're transitioning from the intro into the first h2 — + # emit the intro as a synthetic section. + sections.append(_Section( + file=rel_name, + title=intro_title, + anchor="", + body="\n".join(intro_body).strip(), + )) + intro_body = [] + + current_title = m.group("title").strip() + current_anchor = (m.group("anchor") or "").strip() + current_body = [] + continue + + if current_title is None: + intro_body.append(line) + else: + current_body.append(line) + + # Flush final section (or intro-only file) + if current_title is not None: + sections.append(_Section( + file=rel_name, + title=current_title, + anchor=current_anchor, + body="\n".join(current_body).strip(), + )) + elif intro_title is not None and intro_body: + sections.append(_Section( + file=rel_name, + title=intro_title, + anchor="", + body="\n".join(intro_body).strip(), + )) + + return sections + + +# --------------------------------------------------------------------------- +# Scoring +# --------------------------------------------------------------------------- + +_TOKEN_RE = re.compile(r"[a-zA-Z0-9]+") +_STOPWORDS = frozenset({ + "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", + "how", "i", "in", "is", "it", "of", "on", "or", "that", "the", + "to", "was", "what", "where", "which", "who", "why", "with", + "do", "does", "did", "can", "could", "should", "would", +}) + + +def _tokenize(text: str) -> List[str]: + return [t.lower() for t in _TOKEN_RE.findall(text) if t.lower() not in _STOPWORDS] + + +def _score_section(section: _Section, query: str, tokens: List[str]) -> float: + """Score a section against a query. Higher is better.""" + if not tokens: + return 0.0 + + title_lower = section.title.lower() + anchor_lower = section.anchor.lower() + body_lower = section.body.lower() + query_lower = query.lower().strip() + + score = 0.0 + + # Strongest signal: whole-query substring in title + if query_lower and query_lower in title_lower: + score += 20.0 + + # Per-token title hits + for tok in tokens: + if tok in title_lower: + score += 6.0 + + # Anchor hits + if query_lower and query_lower.replace(" ", "-") in anchor_lower: + score += 8.0 + for tok in tokens: + if tok in anchor_lower: + score += 3.0 + + # Body-keyword frequency (capped so a single mega-section can't dominate) + for tok in tokens: + body_hits = body_lower.count(tok) + if body_hits: + score += min(body_hits, 5) * 1.0 + + return score + + +def _truncate_excerpt(body: str, max_chars: int = 400) -> str: + """Truncate a section body to roughly ``max_chars`` characters on a + word boundary.""" + if len(body) <= max_chars: + return body + # Cut on the nearest space before the limit, fall back to hard cut + cut = body.rfind(" ", 0, max_chars) + if cut < max_chars - 80: # if the last space is too far back, just hard-cut + cut = max_chars + return body[:cut].rstrip() + "..." + + +# --------------------------------------------------------------------------- +# Tool +# --------------------------------------------------------------------------- + + +class SearchOntosHandbookTool(BaseTool): + """Search the Ontos handbook for definitions and explanations. + + Returns ranked excerpts from the `docs/handbook/` corpus — the curated + grounding source for the Ask Ontos copilot. Use this for any + conceptual question (definitions, lifecycle states, role + responsibilities, the agreement workflow, the ontology + knowledge + graph model, data quality, delivery modes, MCP vs Ask Ontos, etc.) + BEFORE answering from training knowledge. + """ + + name = "search_ontos_handbook" + # New category — see query_classifier.CATEGORY_KEYWORDS["handbook"]. + # Also added to DEFAULT_CATEGORIES so vague / generic questions still + # see this tool. + category = "handbook" + description = ( + "Search the Ontos handbook for definitions, role " + "responsibilities, lifecycle states (data product / data contract " + "/ agreement / workflow execution), the approval workflow, the " + "ontology + knowledge graph model, the data quality model, " + "delivery modes (Direct / Indirect / Manual), the MCP server vs " + "the Ask Ontos copilot, and other platform concepts. " + "USE THIS TOOL FIRST for any 'what is X?' / 'how does Y work?' / " + "'what's the difference between A and B?' question — do not " + "answer conceptual questions from training knowledge before " + "checking the handbook. Each result includes a `source_uri` " + "(file.md#anchor) suitable for citation." + ) + parameters = { + "query": { + "type": "string", + "description": ( + "Natural-language question or keyword(s). Examples: " + "'data steward', 'what is a delivery mode', " + "'agreement workflow vs execution', 'ODCS quality items'." + ), + }, + "max_results": { + "type": "integer", + "description": ( + "Maximum number of section excerpts to return (default 5, " + "max 10)." + ), + }, + } + required_params = ["query"] + # Handbook docs are public grounding material; no scope gate. + required_scope = None # type: ignore[assignment] + + async def execute( + self, + ctx: ToolContext, + query: str, + max_results: int = 5, + ) -> ToolResult: + logger.info(f"[search_ontos_handbook] query='{query}' max_results={max_results}") + + if not query or not query.strip(): + return ToolResult( + success=False, + error="query must be non-empty", + ) + + max_results = max(1, min(int(max_results or 5), 10)) + + handbook_dir = _resolve_handbook_dir() + if handbook_dir is None: + logger.warning( + "[search_ontos_handbook] docs/handbook/ not found " + f"(checked ${_HANDBOOK_DIR_ENV_VAR} and parent walk) " + "— returning empty matches" + ) + return ToolResult( + success=True, + data={ + "matches": [], + "total_files_searched": 0, + "message": ( + "Handbook not available in this deployment." + ), + }, + ) + + # Walk the corpus + md_files = sorted(p for p in handbook_dir.iterdir() if p.suffix == ".md") + # Exclude README.md — it's a meta-index, not a handbook doc. + md_files = [p for p in md_files if p.name.lower() != "readme.md"] + + tokens = _tokenize(query) + scored: List[Tuple[float, _Section]] = [] + + for md_file in md_files: + for section in _parse_sections(md_file): + score = _score_section(section, query, tokens) + if score > 0: + scored.append((score, section)) + + if not scored: + return ToolResult( + success=True, + data={ + "matches": [], + "total_files_searched": len(md_files), + "message": "No matching handbook entries found.", + }, + ) + + # Sort: score desc, then file alphabetical (stable tiebreak) + scored.sort(key=lambda x: (-x[0], x[1].file, x[1].title)) + + results: List[Dict[str, Any]] = [] + for score, section in scored[:max_results]: + anchor = section.anchor or _slugify_fallback_anchor(section.title) + source_uri = f"{section.file}#{anchor}" + results.append({ + "file": section.file, + "anchor": anchor, + "title": section.title, + "excerpt": _truncate_excerpt(section.body), + "source_uri": source_uri, + "score": round(score, 2), + }) + + logger.info( + f"[search_ontos_handbook] SUCCESS: {len(results)} matches " + f"(searched {len(md_files)} files, {len(scored)} candidates scored)" + ) + return ToolResult( + success=True, + data={ + "matches": results, + "total_files_searched": len(md_files), + }, + ) diff --git a/src/backend/src/tools/query_classifier.py b/src/backend/src/tools/query_classifier.py index d4b50f044..e4de26060 100644 --- a/src/backend/src/tools/query_classifier.py +++ b/src/backend/src/tools/query_classifier.py @@ -14,11 +14,24 @@ # Category definitions with their trigger keywords CATEGORY_KEYWORDS = { "unity_catalog": [ - "catalog", "catalogs", "schema", "schemas", "table", "tables", + "catalog", "catalogs", "schema", "schemas", "table", "tables", "view", "views", "database", "databases", "sql", "query", "column", "columns", "unity", "uc", "explore", "browse", "list catalog", "my catalogs", "own catalog", "owner" ], + # Ontos-side governance handle for UC resources (tables, views, + # models, dashboards, etc.). The system prompt teaches the LLM + # that UC tables become Assets when they enter Ontos via a + # Deliverable — these keywords ensure the classifier flags Asset + # intent alongside the raw `unity_catalog` browse intent. No tool + # currently registers `category = "assets"`, but the matched + # category is wired into the per-request category list so future + # asset tools and the integration test suite can rely on it. + "assets": [ + "asset", "assets", "table", "tables", "view", "views", + "unity catalog", "uc", "catalog", "schema", "delta table", + "publish", "govern", "expose", + ], "data_products": [ "data product", "product", "products", "output port", "output table", "create product", "draft product", "publish", @@ -50,13 +63,44 @@ "analyze", "analysis", "aggregate", "sum", "count", "average", "statistics", "metrics", "measure", "calculate" ], + # Conceptual / "what is X" / "how does Y work" questions about the + # platform itself (roles, lifecycles, the agreement workflow, the + # ontology + knowledge graph model, delivery modes, MCP, etc.). + # The matching tool — search_ontos_handbook — grounds the LLM in + # the curated docs/handbook/ corpus. + "handbook": [ + "what is", "what's", "what are", "how does", "how do", + "how is", "how are", "explain", "definition", "define", + "difference between", "vs", "versus", "lifecycle", + "rbac", "role", "roles", "permission", "permissions", + "workflow", "approval", "agreement", "ontology", + "knowledge graph", "delivery mode", "mcp", + "data steward", "data producer", "data consumer", + "data owner", "business owner", + ], + # App-state / adoption questions. Surfaces ``get_app_state`` for + # questions like "how many data products do we have?", "is this a + # fresh install?", "what's our adoption?". Also always-on (see + # ``ALWAYS_INCLUDED_CATEGORIES``) because the same snapshot drives + # the system-prompt adoption-mode preamble. + "app_state": [ + "how many", "how much", "adoption", "empty", "new install", + "fresh install", "getting started", "onboarding", + "total number", "count of", "current state", "anyone using", + ], } -# Categories that are always included for general discovery -ALWAYS_INCLUDED_CATEGORIES = ["discovery"] +# Categories that are always included for general discovery. +# `handbook` is always-on so the LLM can ground any vague question in +# the corpus — the cost of carrying one extra tool definition is low and +# the safety upside (fewer hallucinated platform concepts) is large. +# `app_state` is always-on for the same reason: it's a single +# parameter-less tool whose result is cheap and is sometimes the right +# answer to a vague "how are we doing?" question. +ALWAYS_INCLUDED_CATEGORIES = ["discovery", "handbook", "app_state"] # Default categories when no specific match is found -DEFAULT_CATEGORIES = ["discovery", "data_products", "data_contracts", "semantic"] +DEFAULT_CATEGORIES = ["discovery", "handbook", "app_state", "data_products", "data_contracts", "semantic"] def classify_query(query: str) -> List[str]: diff --git a/src/backend/src/tools/registry.py b/src/backend/src/tools/registry.py index 6ca4ed134..b59b22fcc 100644 --- a/src/backend/src/tools/registry.py +++ b/src/backend/src/tools/registry.py @@ -214,6 +214,8 @@ def create_default_registry() -> ToolRegistry: GetConceptNeighborsTool ) from src.tools.search import GlobalSearchTool + from src.tools.handbook import SearchOntosHandbookTool + from src.tools.app_state import GetAppStateTool from src.tools.analytics import ( GetTableSchemaTool, ExecuteAnalyticsQueryTool, @@ -307,6 +309,19 @@ def create_default_registry() -> ToolRegistry: # Search tools registry.register(GlobalSearchTool()) + + # Handbook search — grounds the LLM in docs/handbook/ for any + # "what is X" / "how does Y work" question. New `handbook` category; + # see query_classifier.CATEGORY_KEYWORDS["handbook"] and + # ALWAYS_INCLUDED_CATEGORIES. + registry.register(SearchOntosHandbookTool()) + + # App-state introspection — surfaces total/published counts and a + # binary adoption mode (blank vs active). New `app_state` category + # added to ALWAYS_INCLUDED_CATEGORIES; the same snapshot is + # pre-fetched by LlmSearchManager to inject a mode preamble into + # the system prompt. + registry.register(GetAppStateTool()) # Analytics tools registry.register(GetTableSchemaTool()) diff --git a/src/backend/src/tools/system_prompts.py b/src/backend/src/tools/system_prompts.py new file mode 100644 index 000000000..41e48eaee --- /dev/null +++ b/src/backend/src/tools/system_prompts.py @@ -0,0 +1,346 @@ +""" +System-prompt assembly for the Ask Ontos copilot. + +Phase 1 of the Ask Ontos uplift extracts the system prompt out of +`llm_search_manager.py` (where it lived as a hardcoded constant) into a +function so that: + +1. The `LLM_SYSTEM_PROMPT` env override — defined in `Settings` but + never consumed — finally takes effect as a verbatim replacement of + the default prompt. +2. Phase 2/3 can inject per-page / per-role / per-entity / adoption-mode + personalization without touching the manager. + +The new default prompt is grounded-first: it instructs the model to +call the `search_ontos_handbook` tool for any "what is X" / "how does +Y work" question BEFORE answering from training knowledge, and to +attach hidden `<!-- ref: file.md#anchor -->` citations to claims that +came from the handbook corpus. + +Phase 2 ("adoption mode") and Phase 3 ("role + page + entity") layer +two short preambles ABOVE the default prompt: + +- ``## Current workspace state`` — derived from + ``tools.app_state.get_adoption_snapshot``. The LLM gets onboarding + vs operational framing on every call without having to invoke the + introspection tool itself. +- ``## Current user context`` — derived from the chat request payload + (page name, page URL, selected entity) plus the user's effective + Ontos role. + +When neither preamble is applicable (e.g. the override is set, or the +caller didn't pass context), the default prompt is returned verbatim +so we don't drift from the Phase 1 behavior captured by existing +integration tests. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from src.common.config import Settings + + +# --------------------------------------------------------------------------- +# Default system prompt +# --------------------------------------------------------------------------- + +_DEFAULT_SYSTEM_PROMPT = """You are Ontos, the in-product copilot for the Ontos data governance and data products platform. You help users discover, understand, and analyze data assets, and answer questions about how the platform itself works. You have two grounding sources: + +1. **The curated handbook corpus** (`docs/handbook/`), reached via the `search_ontos_handbook` tool. This is the authoritative source for "what is X" / "how does Y work" questions about Ontos itself. +2. **Live data via tools** — data products, data contracts, the knowledge graph, Unity Catalog, costs, tags, search. + +## Audience and tone + +You are speaking to an **Ontos end user** — an admin, data producer, data consumer, data steward, or governance officer using the Ontos web app. You are NOT speaking to a developer, DBA, or the team building Ontos itself. + +**Forbidden vocabulary** — these exist in the corpus to anchor your reasoning, but they MUST NOT appear in user-visible output: +- SQLAlchemy model class names (anything ending in `Db`, e.g. `DataQualityCheckDb`, `QualityItemDb`, `AssetDb`, `DataProductDb`). +- Pydantic / API model class names (`AppRole`, `AssetCreate`, etc.). +- Raw database column names (`score_percent`, `checks_passed`, `measured_at`, `publication_scope`, `entity_data`). +- Internal workflow IDs (`dqx_profile_datasets`), source filter strings (`source='dqx'`), or table names. + +When you reference an Ontos concept, use the **UI label** the user sees: "Data Product", "Deliverable", "Quality panel", "Asset", "Profile dataset action", "Settings → Workflows". If the corpus gives you a `Db` name or column name, translate it: `DataQualityCheckDb` → "quality check definitions you configure on a contract"; `QualityItemDb` → "execution results shown in the Quality panel"; `score_percent` → "overall quality score". + +**Exception:** if the user explicitly asks about implementation / schema / internals / "how is this stored", you may descend into developer-facing detail. Default behavior is end-user. + +## World model (vocabulary primer) + +**Organizational scope** + +- **Domain** — top-level business-area scope (e.g., Finance, Supply Chain). Every data product, contract, and glossary collection lives under a domain. +- **Team** — durable ownership unit inside a domain. Governs edit rights on data products during draft/development. +- **Project** — optional bounded initiative under a team that groups related work items. + +**Core artifacts** + +- **Data product** — a versioned, governed unit that packages one or more Databricks assets through Deliverables (output ports), optionally depending on Consumables (input ports), owned by a team, optionally bound to data contracts. Follows the Open Data Product Standard (ODPS v1.0.0). "Published" is a separate dimension (`publication_scope`), not part of the definition. +- **Data contract** — the technical and semantic agreement bound to a Deliverable: schema, quality checks, SLAs, servers, support, pricing. Implements the Open Data Contract Standard (ODCS) v3.1.0. Ontos is the editor of record; the workspace (volume / repo) is the deployment surface. +- **Asset** — the Ontos-side handle for a governed UC resource (table, view, model, dashboard, notebook, job) or any other "thing" you want to apply governance to. Created automatically when a UC resource is linked into a Deliverable, or manually via the Assets section. Each Asset carries name, type (ontology-driven), optional domain, owner, lifecycle status, and persona-aware visibility. UC tables become Assets when they enter Ontos — they do not stay as raw catalog references. + +**Product surfaces** + +- **Deliverable** (ODPS *output port*) — a consumable surface of a data product, shipped through one Delivery Method. Optionally bound to a data contract. "Deliverable" is the customer-facing name; "output port" is the ODPS-spec label. +- **Consumable** (ODPS *input port*) — declares an upstream data product this product depends on. Usually omitted for first-time products that just expose existing UC tables; only needed when this product reads from another Ontos-governed data product (not just raw UC tables). Per ODPS, every Consumable references a contract version of the upstream product. +- **Delivery Method** — the configured *how* of a Deliverable: Table Access (UC SELECT), Serving Endpoint (HTTP serving), File Export (volume/object store), or Streaming (Kafka/DLT). Configurable under Settings → Delivery Methods. Distinct from **Delivery Mode** (Direct vs Indirect — a separate governance-propagation axis). + +**Semantic layer** + +- **Ontology** — the *source artifact*: an OWL/RDFS/SKOS file (`.ttl` / `.owl` / `.rdf` / `.nt`) authored externally (Protégé, TopBraid, text editor) that declares classes, data properties, object properties, and optional SHACL shapes. The ontology is *prescriptive* in Ontos: edits to `ontos-ontology.ttl` reshape the asset-type system at startup. +- **Knowledge graph** — the *runtime* structure: an rdflib `ConjunctiveGraph` built from the union of enabled ontologies plus instance-level triples (semantic links, glossary collections). Stored as triples in `rdf_triples`, queried via SPARQL. The ontology is the TBox (terminology); the runtime graph adds the ABox (assertions about real data). +- **Business glossary** — a curated, browsable *view* over published concepts. A glossary term is a concept living in a `urn:glossary:` collection — there is no separate glossary-terms table. Glossary sits at the lowest-expressivity end of the semantic-maturity ladder (Controlled Vocabulary → Taxonomy → Ontology → Knowledge Graph); it is *layered on top of* the same RDF plumbing, not a parallel system. +- **Concept** — a node in the knowledge graph identified by an IRI (typically an RDFS class or SKOS concept). The same concept can be referenced as an ontology class, surfaced as a glossary term inside a `urn:glossary:` collection, *and* pinned to data via semantic links — these are different presentations of one underlying RDF node, not separate entities. +- **Semantic link** — an explicit pin (a row in `entity_semantic_links`) from an Ontos entity (data product, contract, schema object, column, UC table/column, asset, domain) to a concept IRI. The pinned concept may be sourced from an uploaded ontology *and/or* surfaced as a glossary term — the link itself targets the IRI, not a vocabulary surface. On contracts: three-tier (product/contract-level, schema-level, property-level). + +**Physical layer** + +- **Asset** — a governed thing (table, view, dataset, ML model, dashboard, function, etc.) persisted in Ontos with a typed `asset_type` driven by the ontology. The ontology is *prescriptive*: editing `ontos-ontology.ttl` reshapes the asset-type system at startup. + +**Governance machinery** + +- **Workflow** — a *definition*: trigger, scope, ordered steps. The reusable template for an approval / propagation flow. +- **Workflow Execution** — a single *runtime* invocation of a Workflow, tracking status (`pending` / `running` / `paused` / `succeeded` / `failed` / `cancelled`) and current step. +- **Agreement** — the *immutable* record of a completed approval Workflow Execution: snapshotted workflow definition plus per-step results. The audit trail for gated transitions (contract approval, product certification, access grants, tag propagation). + +**Identity** + +- **Role** — an Ontos authorization role: a named bundle of feature × access-level permissions (Admin, Data Governance Officer, Data Steward, Data Producer, Data Consumer, Security Officer). Mapped to users via Databricks groups. +- **Persona** — an audience label (Knowledge Engineer, Data Architect, AI Engineer, Business Analyst, etc.) used in docs and onboarding. *Not* the same as a Role; one person can play multiple personas under one Role. +- **Business Role** — an organizational role label (e.g., "Head of Sales Analytics", "Data Owner", "Technical Owner") referenced inside contracts and approval workflows. Distinct from authorization Roles. + +## Language + +The Ontos handbook and the UI labels in the app are written in English. Users may write to you in any of the supported UI locales — English, German, Spanish, French, Italian, Japanese, Dutch. + +- **Answer in the user's language.** If the user writes in German, answer in German. If the user writes in Japanese, answer in Japanese. Default to the language of the user's most recent message. +- **Keep Ontos terms and UI labels in English, exactly as they appear in the app.** Do not translate: **Data Product**, **Data Contract**, **Deliverable**, **Consumable**, **Delivery Method**, **Asset**, **Domain**, **Team**, **Project**, **Quality Rules**, **Quality panel**, **Profile with DQX**, **Settings → Workflows**, **Marketplace**, **Concept**, **Concept** (ontology term), **Knowledge Graph**, **Glossary**. These are product nouns and appear in English in every UI locale. +- **Handbook excerpts are English; that's fine.** Translate the meaning into the answer's language but keep proper-noun UI labels unchanged. + +## Tool-first policy for conceptual questions (CRITICAL) + +For ANY question of the form "what is X?", "how does Y work?", "what's the difference between A and B?", or "explain Z" — where X/Y/Z/A/B is an Ontos platform concept (a role, a lifecycle state, a workflow, an entity, a delivery mode, a permission, MCP, the knowledge graph, etc.) — your FIRST action is to call `search_ontos_handbook(query=...)`. Do NOT answer conceptual questions from training knowledge before checking the handbook. If the handbook has nothing relevant, fall back to the refusal template below. + +## Three-tier confidence labels (internal — stripped from the user response) + +Annotate each substantive claim in your answer with exactly one of: + +- `[Confirmed]` — the claim comes from a live-data tool result (e.g., a row from `search_data_products`, a schema returned by `get_table_schema`). +- `[Documented]` — the claim comes from a `search_ontos_handbook` excerpt. +- `[Inferred]` — the claim comes from training knowledge or general reasoning. Use sparingly and flag explicitly. + +These labels are stripped from the user-facing response (alongside the `<!-- ref: ... -->` citations below). They exist so reviewers can audit grounding via the debug payload, AND so the act of writing them forces you to stratify confidence — which prevents you from passing off inferred claims as documented ones. Emit one label per substantive claim; the strip is server-side, do not skip them and do not write any user-facing prose treating them as visible. + +## Hidden citations + +When you cite a handbook entry, attach the source URI in this hidden HTML-comment format at the end of your answer, one per line: + + <!-- ref: file.md#anchor --> + +These markers are stripped before the user sees the answer. They exist so reviewers can audit grounding. In v1 we do NOT surface citations to the user — do not write `[source: ...]` or any inline citation; only the hidden comment form. + +## Refusal template + +If no tool result and no handbook excerpt supports the answer, say: + +> "I don't have authoritative information about this in the Ontos documentation or live data. <plain-language alternative or follow-up suggestion>." + +Do not infer beyond what the tools and corpus provide. It is always better to refuse than to fabricate. + +## Tool catalog (strategy only — full schemas are provided separately) + +- `search_ontos_handbook` — Tier 0: handbook corpus. Always tried first for conceptual questions. +- `search_data_products`, `get_data_product`, `search_data_contracts`, `get_data_contract` — Tier 1: governed assets. +- `global_search`, `search_glossary_terms`, `find_entities_by_concept` — Tier 1 / 2: cross-feature search and semantic linking. +- `search_domains`, `search_teams`, `search_projects` — organizational structure. +- `search_tags`, `assign_tag_to_entity`, `list_entity_tags` — tags (namespace/tag_name format; missing namespace defaults to `default`). +- `add_semantic_link`, `list_semantic_links`, `remove_semantic_link` — wire products/contracts to glossary concepts. +- `execute_sparql_query`, `get_concept_hierarchy`, `get_concept_neighbors` — knowledge-graph traversal. +- `list_catalogs`, `get_catalog_details`, `list_schemas`, `explore_catalog_schema`, `get_table_schema` — Tier 3: raw Unity Catalog browsing. +- `execute_analytics_query` — read-only SELECT against Databricks tables. +- `get_data_product_costs` — cost rollups. +- `create_draft_data_contract`, `create_draft_data_product`, `update_data_contract`, `update_data_product` — write operations; always create in `draft` status for user review. + +## Discovery strategy (priority order) + +When users ask about finding, discovering, or locating data, follow this priority: + +- **Tier 0 — Handbook.** Any "what / how / why" question about the platform itself: `search_ontos_handbook` first. +- **Tier 1 — Governed assets.** Curated data products and contracts: `search_data_products`, `search_data_contracts`, `global_search`, `search_glossary_terms` + `find_entities_by_concept`. +- **When a user mentions UC tables, views, models, or other Databricks resources they want to publish, govern, or expose:** ground in the Asset model — those resources become Ontos Assets when linked into a Deliverable. Don't treat them as raw catalog objects. +- **Tier 2 — Semantic enrichment.** Explore concepts and their links to assets when the user asks by topic rather than by name. +- **Tier 3 — Unity Catalog direct browsing.** Use `list_catalogs` / `explore_catalog_schema` / `get_table_schema` ONLY when the user explicitly asks to browse the catalog, OR when Tiers 1 and 2 returned nothing AND you have told the user that. + +Never skip directly to Tier 3 — data products are the primary offering of this platform. + +## Out-of-scope deflection + +If the user asks about something unrelated to Ontos, data governance, the data products on this platform, or general data engineering questions about Databricks / Unity Catalog: politely deflect and offer to redirect to in-scope topics. + +## Response format + +- **Do not restate, echo, or rephrase the user's question** at the start of your response. Do NOT open with a bolded header of the question (e.g., `**What is a Team?**`), and do NOT use fillers like "Great question!" or "Let me explain…". Begin with the answer directly. The user can see their own question above in the chat thread — repeating it is noise. +- **Markdown rules — strict.** Every `*` must pair correctly: + - `## Header` for section headers (never `**Header**` on its own line). + - `**word**` for emphasis — exactly one pair of two asterisks per emphasis. The closing `**` must immediately follow the emphasised text, on the same line. + - `-` for bullets. + - Do NOT use three or more consecutive asterisks anywhere. Sequences like `***`, `****`, `*****` are never valid here — they render as literal characters. If you want a divider, use a blank line or a `##` heading. If you started emphasis with `**`, close it with `**` on the same line and then continue plain text. +- **For "how do I…" or "where is X…" questions, structure the answer as:** + - **Action** — one concrete UI step (e.g. "On this contract page, click **Profile with DQX**"). + - **What happens** — outcome in business terms. + - **Where to see it** — the user-visible surface (panel, page, badge), not a storage table. + - **Next** (optional) — one natural follow-up action. +- Use markdown tables for tabular results. Each row on its own line: + + | Column1 | Column2 | + |---------|---------| + | value1 | value2 | + | value3 | value4 | + + Never put multiple table rows on a single line. +- Use bullet points for lists. +- Bold important numbers and findings. +- Include units (USD, %, rows) where applicable. +- Be concise but thorough. + +## Limitations + +- You execute read-only SELECT queries only. +- Query results are capped at 1000 rows. +- You can only access tables the user has permissions for. +- Cost data may not be complete for all products. +- Handbook citations point to internal grounding material; in v1 they are hidden from the user (HTML comments only). +""" + + +# --------------------------------------------------------------------------- +# Preamble assembly +# --------------------------------------------------------------------------- + + +# Adoption-mode preamble text. Kept short on purpose — the model +# context is precious and these strings ship on every chat call. The +# wording is calibrated to nudge the tone of the answer (suggestion +# style, default examples, what NOT to spend tokens on) without +# overriding the substantive grounded-first / refusal-template policy. +_ADOPTION_PREAMBLE_BLANK = ( + "This workspace is new to Ontos — no data products are published yet. " + "Lean toward onboarding-style suggestions and 'getting started' " + "framings. Avoid optimization advice that assumes existing assets." +) + +_ADOPTION_PREAMBLE_ACTIVE = ( + "This workspace has published data products. Operational and " + "optimization-oriented questions are appropriate; do not over-explain " + "basics unless asked." +) + + +def _adoption_preamble(adoption_mode: Optional[str]) -> Optional[str]: + """Map an ``adoption_mode`` string to its preamble body, or + ``None`` when the mode is missing / unrecognized so the caller can + omit the section entirely (rather than emit a placeholder).""" + if adoption_mode == "blank": + return _ADOPTION_PREAMBLE_BLANK + if adoption_mode == "active": + return _ADOPTION_PREAMBLE_ACTIVE + return None + + +def _user_context_block( + role: Optional[str], + page_name: Optional[str], + page_url: Optional[str], + selected_entity: Optional[Dict[str, Any]], +) -> Optional[str]: + """Render the Phase 3 ``## Current user context`` section. + + Returns ``None`` when every input is empty — the caller drops the + whole H2 in that case, so the default Phase 1 prompt still + round-trips byte-identical for the no-context path. Otherwise we + emit a small bullet list with a one-line tailoring instruction. + """ + if not any([role, page_name, page_url, selected_entity]): + return None + + lines = ["## Current user context", ""] + lines.append(f"- **Role**: {role or 'unknown'}") + + if page_name or page_url: + location = page_name or "unknown" + if page_url: + location += f" ({page_url})" + lines.append(f"- **Currently on**: {location}") + + if selected_entity: + # ``selected_entity`` is a small dict with `type`, `name`, `id` + # — render only the fields that are actually present so the + # block looks clean for partial payloads (e.g. an entity with + # no id yet). + entity_type = selected_entity.get("type") or "entity" + entity_name = selected_entity.get("name") or "(unnamed)" + entity_id = selected_entity.get("id") + viewing = f'- **Viewing**: {entity_type} "{entity_name}"' + if entity_id: + viewing += f" (id: {entity_id})" + lines.append(viewing) + + lines.append("") + lines.append( + "Tailor answers to this role and page. A Data Consumer needs " + "task-completion help; an Admin needs configuration depth." + ) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def get_system_prompt( + *, + settings: Settings, + role: Optional[str] = None, + page_name: Optional[str] = None, + page_url: Optional[str] = None, + selected_entity: Optional[Dict[str, Any]] = None, + adoption_mode: Optional[str] = None, +) -> str: + """Return the system prompt for the Ask Ontos copilot. + + Precedence: + 1. ``settings.LLM_SYSTEM_PROMPT`` (env override) — returned + verbatim when set. This unblocks the previously-dead override + path; the env override is treated as a full replacement, so + Phase 2/3 preambles do NOT get prepended on top. + 2. Otherwise, the default grounded prompt with optional Phase 2 + (adoption-mode) and Phase 3 (user-context) preambles prepended. + + Section order in the assembled prompt: + + 1. ``## Current workspace state`` (Phase 2 — adoption mode) + 2. ``## Current user context`` (Phase 3 — role + page + entity) + 3. The original Phase 1 default prompt (``You are Ontos, ...``) + + When every Phase 2/3 input is ``None`` (or unrecognized), the + function returns the Phase 1 default prompt byte-identically so + existing tests don't regress. + """ + override = getattr(settings, "LLM_SYSTEM_PROMPT", None) + if override: + return override + + sections: list[str] = [] + + adoption_body = _adoption_preamble(adoption_mode) + if adoption_body: + sections.append("## Current workspace state\n\n" + adoption_body) + + user_block = _user_context_block(role, page_name, page_url, selected_entity) + if user_block: + sections.append(user_block) + + if not sections: + return _DEFAULT_SYSTEM_PROMPT + + return "\n\n".join(sections) + "\n\n" + _DEFAULT_SYSTEM_PROMPT diff --git a/src/frontend/src/components/copilot/copilot-panel.tsx b/src/frontend/src/components/copilot/copilot-panel.tsx index f58b24699..fd58f5c7f 100644 --- a/src/frontend/src/components/copilot/copilot-panel.tsx +++ b/src/frontend/src/components/copilot/copilot-panel.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useRef, useCallback } from 'react'; import { useTranslation } from 'react-i18next'; -import { Send, Loader2, Sparkles, X, MessageSquare, Plus, Trash2 } from 'lucide-react'; +import { Send, Loader2, Sparkles, X, MessageSquare, Plus, Trash2, Check, ChevronDown } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { ScrollArea } from '@/components/ui/scroll-area'; import { Separator } from '@/components/ui/separator'; @@ -16,21 +16,17 @@ import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import LLMConsentDialog, { hasLLMConsent } from '@/components/common/llm-consent-dialog'; import { fetchLLMStatus, fetchSessions, sendMessage, deleteSession } from '@/components/search/llm-search-api'; -import { useCopilotStore, type CopilotPageContext } from '@/stores/copilot-store'; +import { useCopilotStore } from '@/stores/copilot-store'; import { useCopilotQuestions } from '@/hooks/use-copilot-questions'; import type { LLMConfig } from '@/types/llm'; import type { ChatMessage, LLMSearchStatus, SessionSummary } from '@/types/llm-search'; const WELCOME_DISMISSED_KEY = 'copilot-welcome-dismissed'; -function buildContextPrefix(ctx: CopilotPageContext): string { - let prefix = `[Context: User is on the "${ctx.pageName}" page at ${ctx.pageUrl}`; - if (ctx.selectedEntity) { - prefix += `, viewing ${ctx.selectedEntity.type} "${ctx.selectedEntity.name}" (id: ${ctx.selectedEntity.id})`; - } - prefix += '. Consider this context when answering.]'; - return prefix; -} +// Page / role / entity context is now sent in the chat-request payload +// (see `llm-search-api.ts` + `ChatMessageCreate`) and rendered server-side +// as a structured `## Current user context` preamble in the system prompt. +// No need to prefix the user's message client-side anymore. function CopilotMessage({ message }: { message: ChatMessage }) { const isUser = message.role === 'user'; @@ -90,10 +86,16 @@ export default function CopilotPanel() { const { t } = useTranslation(['search', 'common']); const isOpen = useCopilotStore((s) => s.isOpen); const pageContext = useCopilotStore((s) => s.pageContext); - const { closePanel } = useCopilotStore((s) => s.actions); - const questionGroups = useCopilotQuestions(); + const contextScope = useCopilotStore((s) => s.contextScope); + const { closePanel, setContextScope } = useCopilotStore((s) => s.actions); const [status, setStatus] = useState<LLMSearchStatus | null>(null); + // ``status?.adoption_mode`` is forwarded into the question hook so a + // blank workspace gets the onboarding starter prompts and an active + // workspace gets the regular catalog. The hook handles `null` + // (snapshot unavailable) by hiding mode-tagged questions only. + const questionGroups = useCopilotQuestions(status?.adoption_mode ?? null); + const [sessions, setSessions] = useState<SessionSummary[]>([]); const [currentSessionId, setCurrentSessionId] = useState<string | undefined>(); const [messages, setMessages] = useState<ChatMessage[]>([]); @@ -152,11 +154,6 @@ export default function CopilotPanel() { return; } - let contextualMessage = messageContent; - if (pageContext) { - contextualMessage = buildContextPrefix(pageContext) + '\n\n' + messageContent; - } - const userMessage: ChatMessage = { id: `temp-${Date.now()}`, role: 'user', @@ -169,7 +166,7 @@ export default function CopilotPanel() { setIsLoading(true); try { - const response = await sendMessage(contextualMessage, currentSessionId); + const response = await sendMessage(messageContent, currentSessionId); setCurrentSessionId(response.session_id); setMessages((prev) => [...prev, response.message]); const updatedSessions = await fetchSessions(); @@ -319,20 +316,66 @@ export default function CopilotPanel() { </div> </div> - {/* Context badge */} - {pageContext?.selectedEntity && ( - <div className="px-4 py-2 border-b bg-muted/30 shrink-0"> - <div className="flex items-center gap-2 text-xs text-muted-foreground"> - <span>{t('search:copilot.askingAbout')}</span> - <Badge variant="outline" className="text-xs font-medium"> - {pageContext.selectedEntity.name} - </Badge> - <Badge variant="secondary" className="text-xs"> - {pageContext.selectedEntity.type} - </Badge> - </div> + {/* Context badge — dropdown lets the user flip between page- + scoped ("Asking about <entity>" or "<page name>") and a + scope-free "Ontos (general)" mode. */} + <div className="px-4 py-2 border-b bg-muted/30 shrink-0"> + <div className="flex items-center gap-2 text-xs text-muted-foreground"> + <span>{t('search:copilot.askingAbout')}</span> + <DropdownMenu> + <DropdownMenuTrigger asChild> + <button + type="button" + className="flex items-center gap-1.5 rounded-md hover:bg-accent/60 px-1 py-0.5 transition-colors" + > + {contextScope === 'general' ? ( + <Badge variant="outline" className="text-xs font-medium"> + {t('search:copilot.scopeGeneral')} + </Badge> + ) : pageContext?.selectedEntity ? ( + <> + <Badge variant="outline" className="text-xs font-medium"> + {pageContext.selectedEntity.name} + </Badge> + <Badge variant="secondary" className="text-xs"> + {pageContext.selectedEntity.type} + </Badge> + </> + ) : ( + <Badge variant="outline" className="text-xs font-medium"> + {pageContext?.pageName || t('search:copilot.scopeThisPage')} + </Badge> + )} + <ChevronDown className="w-3 h-3 text-muted-foreground" /> + </button> + </DropdownMenuTrigger> + <DropdownMenuContent align="start" className="w-48"> + <DropdownMenuItem + onClick={() => setContextScope('page')} + className="gap-2" + > + <Check + className={`w-3.5 h-3.5 ${ + contextScope === 'page' ? 'opacity-100' : 'opacity-0' + }`} + /> + <span>{t('search:copilot.scopePageSpecific')}</span> + </DropdownMenuItem> + <DropdownMenuItem + onClick={() => setContextScope('general')} + className="gap-2" + > + <Check + className={`w-3.5 h-3.5 ${ + contextScope === 'general' ? 'opacity-100' : 'opacity-0' + }`} + /> + <span>{t('search:copilot.scopeGeneral')}</span> + </DropdownMenuItem> + </DropdownMenuContent> + </DropdownMenu> </div> - )} + </div> {/* Messages / Welcome */} <ScrollArea className="flex-1 min-h-0"> diff --git a/src/frontend/src/components/search/llm-search-api.ts b/src/frontend/src/components/search/llm-search-api.ts index e49112f68..2930951c3 100644 --- a/src/frontend/src/components/search/llm-search-api.ts +++ b/src/frontend/src/components/search/llm-search-api.ts @@ -7,6 +7,22 @@ import type { LLMSearchStatus, SessionSummary, } from '@/types/llm-search'; +import { useCopilotStore, type CopilotEntity } from '@/stores/copilot-store'; + +/** + * Wire-shape for ``POST /api/llm-search/chat``. Mirrors the + * ``ChatMessageCreate`` Pydantic model on the backend (Phase 3 fields + * are optional — pre-Phase 3 clients still work). + */ +interface ChatRequestBody { + content: string; + session_id?: string; + debug?: boolean; + page_name?: string; + page_url?: string; + feature_id?: string; + selected_entity?: CopilotEntity; +} export async function fetchLLMStatus(): Promise<LLMSearchStatus> { const response = await fetch('/api/llm-search/status'); @@ -21,10 +37,33 @@ export async function fetchSessions(): Promise<SessionSummary[]> { } export async function sendMessage(content: string, sessionId?: string, debug?: boolean): Promise<ChatResponse> { + // Read page context directly from the Zustand store. ``getState()`` + // is the supported escape hatch for non-component callers (this is + // a plain async function, not a hook). Pulling here keeps both the + // copilot panel and the LLMSearch page on the same payload shape + // without forcing them to pass it explicitly. + const { pageContext, contextScope } = useCopilotStore.getState(); + + const body: ChatRequestBody = { + content, + session_id: sessionId, + debug: debug || false, + }; + // When the user flips the chip to "Ontos (general)", strip every + // page-context field so the backend treats this as a scope-free + // question — the absence of `page_name`/`feature_id`/etc. matches + // the pre-context-aware behavior. + if (pageContext && contextScope !== 'general') { + if (pageContext.pageName) body.page_name = pageContext.pageName; + if (pageContext.pageUrl) body.page_url = pageContext.pageUrl; + if (pageContext.featureId) body.feature_id = pageContext.featureId; + if (pageContext.selectedEntity) body.selected_entity = pageContext.selectedEntity; + } + const response = await fetch('/api/llm-search/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ content, session_id: sessionId, debug: debug || false }), + body: JSON.stringify(body), }); if (!response.ok) { const error = await response.json().catch(() => ({ detail: 'Chat request failed' })); diff --git a/src/frontend/src/components/search/llm-search.tsx b/src/frontend/src/components/search/llm-search.tsx index c7ae75722..4bc305916 100644 --- a/src/frontend/src/components/search/llm-search.tsx +++ b/src/frontend/src/components/search/llm-search.tsx @@ -28,6 +28,7 @@ import remarkGfm from 'remark-gfm'; import LLMConsentDialog, { hasLLMConsent } from '@/components/common/llm-consent-dialog'; import type { LLMConfig } from '@/types/llm'; import type { + AdoptionMode, ChatMessage, DebugInfo, LLMSearchStatus, @@ -389,17 +390,30 @@ function SessionList({ interface ExampleQuestionsProps { onSelectQuestion: (question: string) => void; + adoptionMode?: AdoptionMode | null; } -function ExampleQuestions({ onSelectQuestion }: ExampleQuestionsProps) { +function ExampleQuestions({ onSelectQuestion, adoptionMode }: ExampleQuestionsProps) { const { t } = useTranslation(['search']); - const examples = [ - t('search:llm.examples.findCustomerData'), - t('search:llm.examples.dataProductsCost'), - t('search:llm.examples.businessTermsSales'), - t('search:llm.examples.showDataProducts'), - ]; + // Mode-aware example set: a blank workspace wants onboarding + // questions, not "show me failing checks". When the snapshot is + // unavailable (`adoptionMode == null`) we keep the historical + // 'active' list as the safe default. + const examples = + adoptionMode === 'blank' + ? [ + t('search:llm.examples.howToCreateProduct'), + t('search:llm.examples.howToSetupDomains'), + t('search:llm.examples.whatIsOntos'), + t('search:llm.examples.coreConcepts'), + ] + : [ + t('search:llm.examples.findCustomerData'), + t('search:llm.examples.dataProductsCost'), + t('search:llm.examples.businessTermsSales'), + t('search:llm.examples.showDataProducts'), + ]; return ( <div className="space-y-2"> @@ -753,7 +767,10 @@ export default function LLMSearch() { {t('search:llm.welcomeMessage')} </p> </div> - <ExampleQuestions onSelectQuestion={handleSelectQuestion} /> + <ExampleQuestions + onSelectQuestion={handleSelectQuestion} + adoptionMode={status?.adoption_mode ?? null} + /> </div> ) : ( <div className="space-y-4"> diff --git a/src/frontend/src/config/copilot-questions.ts b/src/frontend/src/config/copilot-questions.ts index 53077d3cc..6dd32d535 100644 --- a/src/frontend/src/config/copilot-questions.ts +++ b/src/frontend/src/config/copilot-questions.ts @@ -6,9 +6,30 @@ export interface CopilotQuestionDef { contexts: string[]; featureId: string; minAccess: FeatureAccessLevel; + /** + * Optional adoption-mode filter. When set, the question is ONLY + * surfaced if the current workspace adoption_mode matches. + * + * - `'blank'`: workspace has no published data products yet — + * onboarding-style "how do I get started" questions. + * - `'active'`: workspace has published products — operational + * "show me failing checks", "low quality scores" style. + * - omitted: question is mode-agnostic and shown regardless. + */ + adoptionMode?: 'blank' | 'active'; + /** + * When `true`, the question is only surfaced on detail pages + * where a `selectedEntity` is present in `pageContext`. The + * localized text MUST use the `{{entityName}}` placeholder, which + * the hook substitutes with the current entity's name (e.g. + * `Customer 360`). Without an entity selected, the question is + * hidden — list pages stay focused on list-level prompts. + */ + requiresEntity?: boolean; } export const COPILOT_CATEGORIES = [ + 'getting_started', 'explore', 'build', 'govern', @@ -18,6 +39,16 @@ export const COPILOT_CATEGORIES = [ export type CopilotCategory = (typeof COPILOT_CATEGORIES)[number]; export const COPILOT_QUESTIONS: CopilotQuestionDef[] = [ + // ── Getting Started (blank workspace) ───────────────────────────── + // Surface ONLY when the backend reports `adoption_mode='blank'`. + // Wording is neutral / generic — no customer specifics. Order in + // this list = display order within the group. + { key: 'gs_create_first_product', category: 'getting_started', contexts: [], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, adoptionMode: 'blank' }, + { key: 'gs_setup_domains', category: 'getting_started', contexts: [], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, adoptionMode: 'blank' }, + { key: 'gs_assign_roles', category: 'getting_started', contexts: [], featureId: 'settings', minAccess: FeatureAccessLevel.READ_ONLY, adoptionMode: 'blank' }, + { key: 'gs_what_is_ontos', category: 'getting_started', contexts: [], featureId: 'search', minAccess: FeatureAccessLevel.READ_ONLY, adoptionMode: 'blank' }, + { key: 'gs_concepts_overview', category: 'getting_started', contexts: [], featureId: 'search', minAccess: FeatureAccessLevel.READ_ONLY, adoptionMode: 'blank' }, + // ── Explore & Discover ──────────────────────────────────────────── // Global / Home – any authenticated user @@ -41,18 +72,31 @@ export const COPILOT_QUESTIONS: CopilotQuestionDef[] = [ // Data Products – read-only { key: 'dp_list_domain', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY }, - { key: 'dp_show_contracts', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY }, + { key: 'dp_show_contracts', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + // Data Products – entity-templated (detail-page only) + { key: 'dp_quality_score', category: 'explore', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dp_owner', category: 'explore', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dp_schema', category: 'explore', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dp_last_updated', category: 'explore', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dp_subscribe', category: 'explore', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dp_consumers', category: 'govern', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, // Data Products – contributor { key: 'dp_draft_product', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_WRITE }, { key: 'dp_package_tables', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_WRITE }, { key: 'dp_add_output_port', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_WRITE }, + { key: 'dp_publication_status', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_WRITE, requiresEntity: true }, + { key: 'dp_publish_blockers', category: 'build', contexts: ['data-products'], featureId: 'data-products', minAccess: FeatureAccessLevel.READ_WRITE, requiresEntity: true }, // Data Contracts – read-only { key: 'ct_show_failing', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_ONLY }, - { key: 'ct_explain_quality', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_ONLY }, + // Data Contracts – entity-templated (detail-page only) + { key: 'ct_what_covers', category: 'explore', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'ct_used_by', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'ct_owner', category: 'govern', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, // Data Contracts – contributor { key: 'ct_create_contract', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_WRITE }, - { key: 'ct_add_quality_check', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_WRITE }, + { key: 'ct_add_quality_check', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_WRITE, requiresEntity: true }, + { key: 'ct_version_impact', category: 'build', contexts: ['data-contracts'], featureId: 'data-contracts', minAccess: FeatureAccessLevel.READ_WRITE, requiresEntity: true }, // Concepts / Semantic Models – read-only { key: 'sm_explain_concept_property', category: 'build', contexts: ['concepts'], featureId: 'semantic-models', minAccess: FeatureAccessLevel.READ_ONLY }, @@ -64,7 +108,12 @@ export const COPILOT_QUESTIONS: CopilotQuestionDef[] = [ // Assets { key: 'asset_find_unmapped', category: 'build', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY }, { key: 'asset_map_columns', category: 'build', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_WRITE }, - { key: 'asset_show_lineage', category: 'build', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY }, + { key: 'asset_show_lineage', category: 'build', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + // Assets – entity-templated (detail-page only) + { key: 'asset_built_on', category: 'explore', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'asset_freshness', category: 'explore', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'asset_quality', category: 'explore', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'asset_consumers', category: 'govern', contexts: ['assets'], featureId: 'assets', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, // ── Govern & Comply ────────────────────────────────────────────── @@ -76,8 +125,16 @@ export const COPILOT_QUESTIONS: CopilotQuestionDef[] = [ // Data Domains { key: 'dom_list_domains', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY }, - { key: 'dom_domain_health', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY }, + // `dom_domain_health` is kept as-is for backward compat — its localized + // text still works for detail pages (the placeholder substitution simply + // applies). `dom_health_detail` is the new explicit-entity variant. + { key: 'dom_domain_health', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, { key: 'dom_create_domain', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_WRITE }, + // Data Domains – entity-templated (detail-page only) + { key: 'dom_products_in', category: 'explore', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dom_business_terms', category: 'explore', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dom_owner', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, + { key: 'dom_health_detail', category: 'govern', contexts: ['data-domains'], featureId: 'data-domains', minAccess: FeatureAccessLevel.READ_ONLY, requiresEntity: true }, // Asset Reviews { key: 'rev_pending_reviews', category: 'govern', contexts: ['data-asset-reviews'], featureId: 'data-asset-reviews', minAccess: FeatureAccessLevel.READ_ONLY }, diff --git a/src/frontend/src/hooks/use-copilot-questions.ts b/src/frontend/src/hooks/use-copilot-questions.ts index f5f1598d3..f400419ee 100644 --- a/src/frontend/src/hooks/use-copilot-questions.ts +++ b/src/frontend/src/hooks/use-copilot-questions.ts @@ -9,6 +9,7 @@ import { COPILOT_CATEGORIES, type CopilotQuestionDef, } from '@/config/copilot-questions'; +import type { AdoptionMode } from '@/types/llm-search'; export interface CopilotQuestionGroup { category: string; @@ -29,10 +30,23 @@ function resolveFeatureId(pathname: string, contextFeatureId?: string): string | return null; } -export function useCopilotQuestions(): CopilotQuestionGroup[] { +// Specificity ranking: lower numbers sort first. Entity-templated +// questions (`requiresEntity`) are the most specific because they +// embed the current detail-page entity in the prompt; page-scoped +// questions come next; global ones are last. +function specificityRank(q: CopilotQuestionDef): number { + if (q.requiresEntity === true) return 0; + if (q.contexts.length > 0) return 1; + return 2; +} + +export function useCopilotQuestions( + adoptionMode?: AdoptionMode | null, +): CopilotQuestionGroup[] { const { t } = useTranslation(['copilot-questions']); const { pathname } = useLocation(); const pageContext = useCopilotStore((s) => s.pageContext); + const contextScope = useCopilotStore((s) => s.contextScope); const { hasPermission, isLoading: permissionsLoading } = usePermissions(); const currentFeatureId = useMemo( @@ -40,10 +54,40 @@ export function useCopilotQuestions(): CopilotQuestionGroup[] { [pathname, pageContext?.featureId], ); + const selectedEntityName = pageContext?.selectedEntity?.name; + return useMemo(() => { if (permissionsLoading) return []; + // When the user flips the chip to "Ontos (general)" we drop every + // page-scoped or entity-templated question and keep only globals. + // The cap also widens to the main-page bucket (15) so the panel + // doesn't shrink unexpectedly. + const generalScope = contextScope === 'general'; + const matching: CopilotQuestionDef[] = COPILOT_QUESTIONS.filter((q) => { + // Adoption-mode filter: questions tagged with a specific + // adoption mode are only shown when it matches. When the + // backend snapshot is unavailable (`adoptionMode` is null/ + // undefined) we hide blank-mode onboarding prompts and keep + // the regular catalog visible — same as the pre-PR behavior. + if (q.adoptionMode && q.adoptionMode !== adoptionMode) return false; + + if (generalScope) { + // General scope: only globals (no contexts, no entity binding). + if (q.requiresEntity === true) return false; + if (q.contexts.length > 0) return false; + return hasPermission(q.featureId, q.minAccess); + } + + // Entity-aware filter: questions tagged `requiresEntity` are + // only surfaced on detail pages where a `selectedEntity` lives + // in `pageContext`. The localized text uses `{{entityName}}` — + // see substitution below. + if (q.requiresEntity === true && !pageContext?.selectedEntity) { + return false; + } + const contextMatch = q.contexts.length === 0 || (currentFeatureId !== null && q.contexts.includes(currentFeatureId)); @@ -52,15 +96,30 @@ export function useCopilotQuestions(): CopilotQuestionGroup[] { return hasPermission(q.featureId, q.minAccess); }); + // Sort by specificity so the most-context-bound questions surface + // first: entity-templated → page-scoped → global. Keeps relative + // order stable within a tier (definition order in COPILOT_QUESTIONS). + matching.sort((a, b) => specificityRank(a) - specificityRank(b)); + + // Cap by page scope. Detail pages (selectedEntity present) and + // type-scoped list pages get a tight cap so the most specific + // questions dominate; main/marketplace/search and general-scope + // get a wider cap. + const onDetailPage = !generalScope && !!pageContext?.selectedEntity; + const onListScope = !generalScope && !onDetailPage && currentFeatureId !== null; + const cap = onDetailPage ? 6 : onListScope ? 6 : 15; + const capped = matching.slice(0, cap); + const groups: CopilotQuestionGroup[] = []; for (const cat of COPILOT_CATEGORIES) { - const catQuestions = matching + const catQuestions = capped .filter((q) => q.category === cat) - .map((q) => ({ - key: q.key, - text: t(`copilot-questions:questions.${q.key}`), - })); + .map((q) => { + const raw = t(`copilot-questions:questions.${q.key}`); + const text = raw.replace(/\{\{entityName\}\}/g, selectedEntityName ?? ''); + return { key: q.key, text }; + }); if (catQuestions.length > 0) { groups.push({ @@ -72,5 +131,14 @@ export function useCopilotQuestions(): CopilotQuestionGroup[] { } return groups; - }, [currentFeatureId, permissionsLoading, hasPermission, t]); + }, [ + currentFeatureId, + permissionsLoading, + hasPermission, + t, + adoptionMode, + pageContext?.selectedEntity, + selectedEntityName, + contextScope, + ]); } diff --git a/src/frontend/src/i18n/locales/de/copilot-questions.json b/src/frontend/src/i18n/locales/de/copilot-questions.json index 8e8736f38..e6c9908a4 100644 --- a/src/frontend/src/i18n/locales/de/copilot-questions.json +++ b/src/frontend/src/i18n/locales/de/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "Wo finde ich Kundendaten?", "global_business_terms_sales": "Welche Geschäftsbegriffe beziehen sich auf den Vertrieb?", "global_what_domains_exist": "Welche Datendomänen sind in der Organisation definiert?", - "mp_browse_products": "Zeige mir Datenprodukte in der Kundendomäne.", "mp_product_cost": "Was kosten alle Datenprodukte?", "mp_subscribe_product": "Wie abonniere ich ein Datenprodukt?", - "dc_search_tables": "Suche nach Tabellen mit Kundeninformationen.", "dc_find_columns": "Finde Spalten mit E-Mail-Adressen in allen Tabellen.", - "search_across_all": "Suche über alle Datenprodukte, Verträge und Begriffe.", - "dp_list_domain": "Welche Datenprodukte sind in meiner Domäne verfügbar?", - "dp_show_contracts": "Welche Verträge sind mit diesem Datenprodukt verknüpft?", + "dp_show_contracts": "Welche Verträge sind mit {{entityName}} verknüpft?", + "dp_quality_score": "Wie ist der Datenqualitätswert von {{entityName}}?", + "dp_owner": "Wem gehört {{entityName}}?", + "dp_schema": "Zeige mir das Schema von {{entityName}}.", + "dp_last_updated": "Wann wurde {{entityName}} zuletzt aktualisiert?", + "dp_subscribe": "Wie abonniere ich {{entityName}}?", + "dp_consumers": "Wer nutzt {{entityName}}?", "dp_draft_product": "Hilf mir, ein neues Datenprodukt aus bestehenden Tabellen zu entwerfen.", "dp_package_tables": "Hilf mir, Analysetabellen als Datenprodukt zu bündeln.", "dp_add_output_port": "Wie füge ich einem Datenprodukt einen Output-Port hinzu?", - + "dp_publication_status": "Wie ist der Veröffentlichungsstatus von {{entityName}}?", + "dp_publish_blockers": "Was verhindert die Veröffentlichung von {{entityName}}?", "ct_show_failing": "Zeige mir Verträge mit fehlgeschlagenen Qualitätsprüfungen.", - "ct_explain_quality": "Erkläre die Qualitätsprüfungen in diesem Vertrag.", + "ct_what_covers": "Was deckt {{entityName}} ab?", + "ct_used_by": "Welche Datenprodukte nutzen {{entityName}}?", + "ct_owner": "Wem gehört {{entityName}}?", "ct_create_contract": "Erstelle einen neuen Datenvertrag für mein Vertriebsschema.", - "ct_add_quality_check": "Hilf mir, eine Qualitätsprüfung zu diesem Vertrag hinzuzufügen.", - + "ct_add_quality_check": "Hilf mir, eine Qualitätsprüfung zu {{entityName}} hinzuzufügen.", + "ct_version_impact": "Was ändert sich, wenn ich die Version von {{entityName}} erhöhe?", "sm_explain_concept_property": "Erkläre den Unterschied zwischen einem Konzept und einer Eigenschaft.", "sm_browse_collections": "Welche Konzeptsammlungen sind verfügbar?", "sm_define_vocabulary": "Hilf mir, ein Vokabular für unsere Kundendomäne zu definieren.", "sm_suggest_concepts": "Welche Konzepte sollte ich für ein Supply-Chain-Modell erstellen?", - "asset_find_unmapped": "Finde Tabellen, die semantische Zuordnungen benötigen.", "asset_map_columns": "Wie ordne ich Spalten semantischen Eigenschaften zu?", - "asset_show_lineage": "Zeige mir die Herkunft dieses Assets.", - + "asset_show_lineage": "Zeige mir die Herkunft von {{entityName}}.", + "asset_built_on": "Welche Datenprodukte basieren auf {{entityName}}?", + "asset_freshness": "Wie aktuell ist {{entityName}}?", + "asset_quality": "Wie ist der Qualitätsstatus von {{entityName}}?", + "asset_consumers": "Wer nutzt {{entityName}}?", "comp_low_scores": "Zeige Datenprodukte mit niedrigen Compliance-Werten.", "comp_failing_checks": "Welche Compliance-Prüfungen schlagen derzeit fehl?", "comp_create_policy": "Erstelle eine neue Compliance-Richtlinie für personenbezogene Daten.", - "dom_list_domains": "Liste alle Datendomänen und ihre Verantwortlichen auf.", - "dom_domain_health": "Wie ist der Gesundheitsstatus dieser Domäne?", + "dom_domain_health": "Wie ist der Gesundheitsstatus von {{entityName}}?", "dom_create_domain": "Hilf mir, eine neue Datendomäne zu erstellen.", - + "dom_products_in": "Welche Datenprodukte gehören zu {{entityName}}?", + "dom_business_terms": "Welche Geschäftsbegriffe sind mit {{entityName}} verknüpft?", + "dom_owner": "Wer verwaltet {{entityName}}?", + "dom_health_detail": "Wie ist die Gesundheit von {{entityName}}?", "rev_pending_reviews": "Zeige mir ausstehende Asset-Überprüfungen.", "rev_start_review": "Starte eine neue Überprüfung für eine Tabelle.", - "gov_semantic_coverage": "Warum ist die semantische Abdeckung letzte Woche gesunken?", "gov_domains_ready": "Welche Domänen sind bereit für den Agenteneinsatz?", - "cc_table_columns": "Welche Spalten hat diese Tabelle?", "cc_table_owner": "Wem gehört diese Tabelle?", "cc_table_usage": "Wie wird diese Tabelle verwendet?", - "settings_manage_roles": "Wie verwalte ich Benutzerrollen und Berechtigungen?", "settings_configure_jobs": "Hilf mir, Hintergrundaufträge zu konfigurieren." } diff --git a/src/frontend/src/i18n/locales/de/search.json b/src/frontend/src/i18n/locales/de/search.json index 680322b23..e27e0d1b0 100644 --- a/src/frontend/src/i18n/locales/de/search.json +++ b/src/frontend/src/i18n/locales/de/search.json @@ -169,5 +169,10 @@ "deleteSessionFailed": "Sitzung konnte nicht gelöscht werden", "loadFailed": "LLM-Suche konnte nicht geladen werden. Bitte versuchen Sie es später erneut." } + }, + "copilot": { + "scopePageSpecific": "Seitenspezifisch", + "scopeGeneral": "Ontos (allgemein)", + "scopeThisPage": "Diese Seite" } } diff --git a/src/frontend/src/i18n/locales/en/copilot-questions.json b/src/frontend/src/i18n/locales/en/copilot-questions.json index b42cf5aab..b833ed9b2 100644 --- a/src/frontend/src/i18n/locales/en/copilot-questions.json +++ b/src/frontend/src/i18n/locales/en/copilot-questions.json @@ -1,62 +1,74 @@ { "categories": { + "getting_started": "Getting Started", "explore": "Explore & Discover", "build": "Build & Create", "govern": "Govern & Comply", "operate": "Operate & Deploy" }, "questions": { + "gs_create_first_product": "How do I create my first data product?", + "gs_setup_domains": "How do I set up data domains?", + "gs_assign_roles": "How do I assign roles to my team?", + "gs_what_is_ontos": "What is Ontos and what can I do with it?", + "gs_concepts_overview": "Explain the core concepts: data products, contracts, and domains.", "global_find_customer_data": "Where can I find customer data?", "global_business_terms_sales": "What business terms are related to sales?", "global_what_domains_exist": "What data domains are defined in the organization?", - "mp_browse_products": "Show me data products in the Customer domain.", "mp_product_cost": "How much do all data products cost?", "mp_subscribe_product": "How do I subscribe to a data product?", - "dc_search_tables": "Search for tables containing customer information.", "dc_find_columns": "Find columns related to email addresses across all tables.", - "search_across_all": "Search across all data products, contracts, and terms.", - "dp_list_domain": "What data products are available in my domain?", - "dp_show_contracts": "Which contracts are linked to this data product?", + "dp_show_contracts": "Which contracts are linked to {{entityName}}?", + "dp_quality_score": "What is the data quality score of {{entityName}}?", + "dp_owner": "Who owns {{entityName}}?", + "dp_schema": "Show me the schema of {{entityName}}.", + "dp_last_updated": "When was {{entityName}} last updated?", + "dp_subscribe": "How do I subscribe to {{entityName}}?", + "dp_consumers": "Who consumes {{entityName}}?", "dp_draft_product": "Help me draft a new data product from existing tables.", "dp_package_tables": "Help me package analytics tables as a data product.", "dp_add_output_port": "How do I add an output port to a data product?", - + "dp_publication_status": "What's the publication status of {{entityName}}?", + "dp_publish_blockers": "What's blocking {{entityName}} from being published?", "ct_show_failing": "Show me contracts with failing quality checks.", - "ct_explain_quality": "Explain the quality checks defined in this contract.", + "ct_what_covers": "What does {{entityName}} cover?", + "ct_used_by": "Which data products use {{entityName}}?", + "ct_owner": "Who owns {{entityName}}?", "ct_create_contract": "Create a new data contract for my sales schema.", - "ct_add_quality_check": "Help me add a quality check to this contract.", - + "ct_add_quality_check": "Help me add a quality check to {{entityName}}.", + "ct_version_impact": "What changes if I bump the version of {{entityName}}?", "sm_explain_concept_property": "Explain the difference between a concept and a property.", "sm_browse_collections": "What concept collections are available?", "sm_define_vocabulary": "Help me define a vocabulary for our Customer domain.", "sm_suggest_concepts": "What concepts should I create for a Supply Chain model?", - "asset_find_unmapped": "Find tables that need semantic mappings.", "asset_map_columns": "How do I map columns to semantic properties?", - "asset_show_lineage": "Show me the lineage for this asset.", - + "asset_show_lineage": "Show me the lineage for {{entityName}}.", + "asset_built_on": "What data products are built on {{entityName}}?", + "asset_freshness": "What's the freshness of {{entityName}}?", + "asset_quality": "What's the quality status of {{entityName}}?", + "asset_consumers": "Who consumes {{entityName}}?", "comp_low_scores": "Show data products with low compliance scores.", "comp_failing_checks": "Which compliance checks are currently failing?", "comp_create_policy": "Create a new compliance policy for PII data.", - "dom_list_domains": "List all data domains and their owners.", - "dom_domain_health": "What is the health status of this domain?", + "dom_domain_health": "What is the health status of {{entityName}}?", "dom_create_domain": "Help me create a new data domain.", - + "dom_products_in": "What data products live in {{entityName}}?", + "dom_business_terms": "What business terms are tied to {{entityName}}?", + "dom_owner": "Who governs {{entityName}}?", + "dom_health_detail": "What's the health of {{entityName}}?", "rev_pending_reviews": "Show me pending asset reviews.", "rev_start_review": "Start a new review for a table.", - "gov_semantic_coverage": "Why did semantic coverage drop last week?", "gov_domains_ready": "What domains are ready for agent use?", - "cc_table_columns": "What columns does this table have?", "cc_table_owner": "Who owns this table?", "cc_table_usage": "How is this table used?", - "settings_manage_roles": "How do I manage user roles and permissions?", "settings_configure_jobs": "Help me configure background jobs." } diff --git a/src/frontend/src/i18n/locales/en/search.json b/src/frontend/src/i18n/locales/en/search.json index 6eb9828b4..c5f08209b 100644 --- a/src/frontend/src/i18n/locales/en/search.json +++ b/src/frontend/src/i18n/locales/en/search.json @@ -165,7 +165,11 @@ "findCustomerData": "Where can I find customer data?", "dataProductsCost": "How much do all data products cost?", "businessTermsSales": "What business terms are related to sales?", - "showDataProducts": "Show me data products in the Customer domain" + "showDataProducts": "Show me data products in the Customer domain", + "howToCreateProduct": "How do I create my first data product?", + "howToSetupDomains": "How do I set up data domains?", + "whatIsOntos": "What is Ontos and what can I do with it?", + "coreConcepts": "Explain the core concepts: data products, contracts, and domains." }, "messages": { "sessionDeleted": "Session deleted", @@ -182,6 +186,9 @@ "welcome": "Welcome to Ask Ontos!", "welcomeDescription": "Get help with modeling, mappings, product creation, and health insights. Try one of the suggested prompts below or type your own question.", "askingAbout": "Asking about:", + "scopePageSpecific": "Page-specific", + "scopeGeneral": "Ontos (general)", + "scopeThisPage": "This page", "inputPlaceholder": "Ask Ontos anything...", "messageSendFailed": "Failed to send message" } diff --git a/src/frontend/src/i18n/locales/es/copilot-questions.json b/src/frontend/src/i18n/locales/es/copilot-questions.json index fb25c667a..8c23799a4 100644 --- a/src/frontend/src/i18n/locales/es/copilot-questions.json +++ b/src/frontend/src/i18n/locales/es/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "¿Dónde puedo encontrar datos de clientes?", "global_business_terms_sales": "¿Qué términos de negocio están relacionados con ventas?", "global_what_domains_exist": "¿Qué dominios de datos están definidos en la organización?", - "mp_browse_products": "Muéstrame productos de datos en el dominio de Clientes.", "mp_product_cost": "¿Cuánto cuestan todos los productos de datos?", "mp_subscribe_product": "¿Cómo me suscribo a un producto de datos?", - "dc_search_tables": "Buscar tablas que contengan información de clientes.", "dc_find_columns": "Encontrar columnas relacionadas con direcciones de correo electrónico en todas las tablas.", - "search_across_all": "Buscar en todos los productos de datos, contratos y términos.", - "dp_list_domain": "¿Qué productos de datos están disponibles en mi dominio?", - "dp_show_contracts": "¿Qué contratos están vinculados a este producto de datos?", + "dp_show_contracts": "¿Qué contratos están vinculados a {{entityName}}?", + "dp_quality_score": "¿Cuál es la puntuación de calidad de datos de {{entityName}}?", + "dp_owner": "¿Quién es el propietario de {{entityName}}?", + "dp_schema": "Muéstrame el esquema de {{entityName}}.", + "dp_last_updated": "¿Cuándo se actualizó {{entityName}} por última vez?", + "dp_subscribe": "¿Cómo me suscribo a {{entityName}}?", + "dp_consumers": "¿Quién consume {{entityName}}?", "dp_draft_product": "Ayúdame a crear un borrador de un nuevo producto de datos a partir de tablas existentes.", "dp_package_tables": "Ayúdame a empaquetar tablas analíticas como producto de datos.", "dp_add_output_port": "¿Cómo agrego un puerto de salida a un producto de datos?", - + "dp_publication_status": "¿Cuál es el estado de publicación de {{entityName}}?", + "dp_publish_blockers": "¿Qué impide que {{entityName}} se publique?", "ct_show_failing": "Muéstrame contratos con controles de calidad fallidos.", - "ct_explain_quality": "Explica los controles de calidad definidos en este contrato.", + "ct_what_covers": "¿Qué cubre {{entityName}}?", + "ct_used_by": "¿Qué productos de datos usan {{entityName}}?", + "ct_owner": "¿Quién es el propietario de {{entityName}}?", "ct_create_contract": "Crear un nuevo contrato de datos para mi esquema de ventas.", - "ct_add_quality_check": "Ayúdame a agregar un control de calidad a este contrato.", - + "ct_add_quality_check": "Ayúdame a agregar un control de calidad a {{entityName}}.", + "ct_version_impact": "¿Qué cambia si actualizo la versión de {{entityName}}?", "sm_explain_concept_property": "Explica la diferencia entre un concepto y una propiedad.", "sm_browse_collections": "¿Qué colecciones de conceptos están disponibles?", "sm_define_vocabulary": "Ayúdame a definir un vocabulario para nuestro dominio de Clientes.", "sm_suggest_concepts": "¿Qué conceptos debería crear para un modelo de cadena de suministro?", - "asset_find_unmapped": "Encontrar tablas que necesitan mapeos semánticos.", "asset_map_columns": "¿Cómo mapeo columnas a propiedades semánticas?", - "asset_show_lineage": "Muéstrame el linaje de este activo.", - + "asset_show_lineage": "Muéstrame el linaje de {{entityName}}.", + "asset_built_on": "¿Qué productos de datos están construidos sobre {{entityName}}?", + "asset_freshness": "¿Cuál es la frescura de {{entityName}}?", + "asset_quality": "¿Cuál es el estado de calidad de {{entityName}}?", + "asset_consumers": "¿Quién consume {{entityName}}?", "comp_low_scores": "Mostrar productos de datos con puntuaciones de cumplimiento bajas.", "comp_failing_checks": "¿Qué controles de cumplimiento están fallando actualmente?", "comp_create_policy": "Crear una nueva política de cumplimiento para datos personales.", - "dom_list_domains": "Listar todos los dominios de datos y sus responsables.", - "dom_domain_health": "¿Cuál es el estado de salud de este dominio?", + "dom_domain_health": "¿Cuál es el estado de salud de {{entityName}}?", "dom_create_domain": "Ayúdame a crear un nuevo dominio de datos.", - + "dom_products_in": "¿Qué productos de datos pertenecen a {{entityName}}?", + "dom_business_terms": "¿Qué términos de negocio están vinculados a {{entityName}}?", + "dom_owner": "¿Quién gobierna {{entityName}}?", + "dom_health_detail": "¿Cuál es la salud de {{entityName}}?", "rev_pending_reviews": "Muéstrame las revisiones de activos pendientes.", "rev_start_review": "Iniciar una nueva revisión para una tabla.", - "gov_semantic_coverage": "¿Por qué bajó la cobertura semántica la semana pasada?", "gov_domains_ready": "¿Qué dominios están listos para el uso de agentes?", - "cc_table_columns": "¿Qué columnas tiene esta tabla?", "cc_table_owner": "¿Quién es el propietario de esta tabla?", "cc_table_usage": "¿Cómo se utiliza esta tabla?", - "settings_manage_roles": "¿Cómo gestiono los roles de usuario y los permisos?", "settings_configure_jobs": "Ayúdame a configurar los trabajos en segundo plano." } diff --git a/src/frontend/src/i18n/locales/es/search.json b/src/frontend/src/i18n/locales/es/search.json index c3a7434d5..ccf1da6cf 100644 --- a/src/frontend/src/i18n/locales/es/search.json +++ b/src/frontend/src/i18n/locales/es/search.json @@ -124,5 +124,10 @@ "deleteSessionFailed": "Error al eliminar la sesión", "loadFailed": "Error al cargar la búsqueda LLM. Por favor intente más tarde." } + }, + "copilot": { + "scopePageSpecific": "Específico de la página", + "scopeGeneral": "Ontos (general)", + "scopeThisPage": "Esta página" } } diff --git a/src/frontend/src/i18n/locales/fr/copilot-questions.json b/src/frontend/src/i18n/locales/fr/copilot-questions.json index 6d02f80d0..07d3b43ba 100644 --- a/src/frontend/src/i18n/locales/fr/copilot-questions.json +++ b/src/frontend/src/i18n/locales/fr/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "Où puis-je trouver les données clients ?", "global_business_terms_sales": "Quels termes métier sont liés aux ventes ?", "global_what_domains_exist": "Quels domaines de données sont définis dans l'organisation ?", - "mp_browse_products": "Montrez-moi les produits de données du domaine Client.", "mp_product_cost": "Combien coûtent tous les produits de données ?", "mp_subscribe_product": "Comment m'abonner à un produit de données ?", - "dc_search_tables": "Rechercher des tables contenant des informations clients.", "dc_find_columns": "Trouver les colonnes liées aux adresses e-mail dans toutes les tables.", - "search_across_all": "Rechercher dans tous les produits de données, contrats et termes.", - "dp_list_domain": "Quels produits de données sont disponibles dans mon domaine ?", - "dp_show_contracts": "Quels contrats sont liés à ce produit de données ?", + "dp_show_contracts": "Quels contrats sont liés à {{entityName}} ?", + "dp_quality_score": "Quel est le score de qualité des données de {{entityName}} ?", + "dp_owner": "Qui est propriétaire de {{entityName}} ?", + "dp_schema": "Montrez-moi le schéma de {{entityName}}.", + "dp_last_updated": "Quand {{entityName}} a-t-il été mis à jour pour la dernière fois ?", + "dp_subscribe": "Comment m'abonner à {{entityName}} ?", + "dp_consumers": "Qui consomme {{entityName}} ?", "dp_draft_product": "Aidez-moi à ébaucher un nouveau produit de données à partir de tables existantes.", "dp_package_tables": "Aidez-moi à packager des tables analytiques en produit de données.", "dp_add_output_port": "Comment ajouter un port de sortie à un produit de données ?", - + "dp_publication_status": "Quel est le statut de publication de {{entityName}} ?", + "dp_publish_blockers": "Qu'est-ce qui empêche {{entityName}} d'être publié ?", "ct_show_failing": "Montrez-moi les contrats avec des contrôles qualité en échec.", - "ct_explain_quality": "Expliquez les contrôles qualité définis dans ce contrat.", + "ct_what_covers": "Que couvre {{entityName}} ?", + "ct_used_by": "Quels produits de données utilisent {{entityName}} ?", + "ct_owner": "Qui est propriétaire de {{entityName}} ?", "ct_create_contract": "Créer un nouveau contrat de données pour mon schéma de ventes.", - "ct_add_quality_check": "Aidez-moi à ajouter un contrôle qualité à ce contrat.", - + "ct_add_quality_check": "Aidez-moi à ajouter un contrôle qualité à {{entityName}}.", + "ct_version_impact": "Que change-t-il si j'incrémente la version de {{entityName}} ?", "sm_explain_concept_property": "Expliquez la différence entre un concept et une propriété.", "sm_browse_collections": "Quelles collections de concepts sont disponibles ?", "sm_define_vocabulary": "Aidez-moi à définir un vocabulaire pour notre domaine Client.", "sm_suggest_concepts": "Quels concepts devrais-je créer pour un modèle de chaîne logistique ?", - "asset_find_unmapped": "Trouver les tables nécessitant des mappings sémantiques.", "asset_map_columns": "Comment mapper des colonnes à des propriétés sémantiques ?", - "asset_show_lineage": "Montrez-moi le lignage de cet actif.", - + "asset_show_lineage": "Montrez-moi le lignage de {{entityName}}.", + "asset_built_on": "Quels produits de données sont construits sur {{entityName}} ?", + "asset_freshness": "Quelle est la fraîcheur de {{entityName}} ?", + "asset_quality": "Quel est l'état de qualité de {{entityName}} ?", + "asset_consumers": "Qui consomme {{entityName}} ?", "comp_low_scores": "Montrer les produits de données avec de faibles scores de conformité.", "comp_failing_checks": "Quels contrôles de conformité sont actuellement en échec ?", "comp_create_policy": "Créer une nouvelle politique de conformité pour les données personnelles.", - "dom_list_domains": "Lister tous les domaines de données et leurs responsables.", - "dom_domain_health": "Quel est l'état de santé de ce domaine ?", + "dom_domain_health": "Quel est l'état de santé de {{entityName}} ?", "dom_create_domain": "Aidez-moi à créer un nouveau domaine de données.", - + "dom_products_in": "Quels produits de données appartiennent à {{entityName}} ?", + "dom_business_terms": "Quels termes métier sont liés à {{entityName}} ?", + "dom_owner": "Qui gouverne {{entityName}} ?", + "dom_health_detail": "Quelle est la santé de {{entityName}} ?", "rev_pending_reviews": "Montrez-moi les revues d'actifs en attente.", "rev_start_review": "Démarrer une nouvelle revue pour une table.", - "gov_semantic_coverage": "Pourquoi la couverture sémantique a-t-elle baissé la semaine dernière ?", "gov_domains_ready": "Quels domaines sont prêts pour l'utilisation par des agents ?", - "cc_table_columns": "Quelles colonnes cette table possède-t-elle ?", "cc_table_owner": "À qui appartient cette table ?", "cc_table_usage": "Comment cette table est-elle utilisée ?", - "settings_manage_roles": "Comment gérer les rôles et les permissions des utilisateurs ?", "settings_configure_jobs": "Aidez-moi à configurer les tâches en arrière-plan." } diff --git a/src/frontend/src/i18n/locales/fr/search.json b/src/frontend/src/i18n/locales/fr/search.json index e84e7d6ce..d7c324140 100644 --- a/src/frontend/src/i18n/locales/fr/search.json +++ b/src/frontend/src/i18n/locales/fr/search.json @@ -124,5 +124,10 @@ "deleteSessionFailed": "Échec de la suppression de la session", "loadFailed": "Échec du chargement de la recherche LLM. Veuillez réessayer plus tard." } + }, + "copilot": { + "scopePageSpecific": "Spécifique à la page", + "scopeGeneral": "Ontos (général)", + "scopeThisPage": "Cette page" } } diff --git a/src/frontend/src/i18n/locales/it/copilot-questions.json b/src/frontend/src/i18n/locales/it/copilot-questions.json index efbc33eb9..33254e40f 100644 --- a/src/frontend/src/i18n/locales/it/copilot-questions.json +++ b/src/frontend/src/i18n/locales/it/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "Dove posso trovare i dati dei clienti?", "global_business_terms_sales": "Quali termini aziendali sono collegati alle vendite?", "global_what_domains_exist": "Quali domini dati sono definiti nell'organizzazione?", - "mp_browse_products": "Mostrami i prodotti dati nel dominio Cliente.", "mp_product_cost": "Quanto costano tutti i prodotti dati?", "mp_subscribe_product": "Come mi iscrivo a un prodotto dati?", - "dc_search_tables": "Cerca tabelle contenenti informazioni sui clienti.", "dc_find_columns": "Trova colonne relative agli indirizzi email in tutte le tabelle.", - "search_across_all": "Cerca tra tutti i prodotti dati, contratti e termini.", - "dp_list_domain": "Quali prodotti dati sono disponibili nel mio dominio?", - "dp_show_contracts": "Quali contratti sono collegati a questo prodotto dati?", + "dp_show_contracts": "Quali contratti sono collegati a {{entityName}}?", + "dp_quality_score": "Qual è il punteggio di qualità dei dati di {{entityName}}?", + "dp_owner": "Chi è il proprietario di {{entityName}}?", + "dp_schema": "Mostrami lo schema di {{entityName}}.", + "dp_last_updated": "Quando è stato aggiornato {{entityName}} l'ultima volta?", + "dp_subscribe": "Come mi iscrivo a {{entityName}}?", + "dp_consumers": "Chi consuma {{entityName}}?", "dp_draft_product": "Aiutami a creare una bozza di un nuovo prodotto dati da tabelle esistenti.", "dp_package_tables": "Aiutami a impacchettare tabelle analitiche come prodotto dati.", "dp_add_output_port": "Come aggiungo una porta di uscita a un prodotto dati?", - + "dp_publication_status": "Qual è lo stato di pubblicazione di {{entityName}}?", + "dp_publish_blockers": "Cosa impedisce la pubblicazione di {{entityName}}?", "ct_show_failing": "Mostrami i contratti con controlli qualità falliti.", - "ct_explain_quality": "Spiega i controlli qualità definiti in questo contratto.", + "ct_what_covers": "Cosa copre {{entityName}}?", + "ct_used_by": "Quali prodotti dati usano {{entityName}}?", + "ct_owner": "Chi è il proprietario di {{entityName}}?", "ct_create_contract": "Crea un nuovo contratto dati per il mio schema vendite.", - "ct_add_quality_check": "Aiutami ad aggiungere un controllo qualità a questo contratto.", - + "ct_add_quality_check": "Aiutami ad aggiungere un controllo qualità a {{entityName}}.", + "ct_version_impact": "Cosa cambia se incremento la versione di {{entityName}}?", "sm_explain_concept_property": "Spiega la differenza tra un concetto e una proprietà.", "sm_browse_collections": "Quali collezioni di concetti sono disponibili?", "sm_define_vocabulary": "Aiutami a definire un vocabolario per il nostro dominio Cliente.", "sm_suggest_concepts": "Quali concetti dovrei creare per un modello di catena di fornitura?", - "asset_find_unmapped": "Trova tabelle che necessitano di mappature semantiche.", "asset_map_columns": "Come mappo le colonne alle proprietà semantiche?", - "asset_show_lineage": "Mostrami la lineage di questo asset.", - + "asset_show_lineage": "Mostrami la lineage di {{entityName}}.", + "asset_built_on": "Quali prodotti dati sono costruiti su {{entityName}}?", + "asset_freshness": "Qual è la freschezza di {{entityName}}?", + "asset_quality": "Qual è lo stato di qualità di {{entityName}}?", + "asset_consumers": "Chi consuma {{entityName}}?", "comp_low_scores": "Mostra prodotti dati con punteggi di conformità bassi.", "comp_failing_checks": "Quali controlli di conformità stanno attualmente fallendo?", "comp_create_policy": "Crea una nuova politica di conformità per i dati personali.", - "dom_list_domains": "Elenca tutti i domini dati e i loro responsabili.", - "dom_domain_health": "Qual è lo stato di salute di questo dominio?", + "dom_domain_health": "Qual è lo stato di salute di {{entityName}}?", "dom_create_domain": "Aiutami a creare un nuovo dominio dati.", - + "dom_products_in": "Quali prodotti dati appartengono a {{entityName}}?", + "dom_business_terms": "Quali termini aziendali sono collegati a {{entityName}}?", + "dom_owner": "Chi governa {{entityName}}?", + "dom_health_detail": "Qual è la salute di {{entityName}}?", "rev_pending_reviews": "Mostrami le revisioni di asset in sospeso.", "rev_start_review": "Avvia una nuova revisione per una tabella.", - "gov_semantic_coverage": "Perché la copertura semantica è diminuita la settimana scorsa?", "gov_domains_ready": "Quali domini sono pronti per l'uso da parte degli agenti?", - "cc_table_columns": "Quali colonne ha questa tabella?", "cc_table_owner": "Chi è il proprietario di questa tabella?", "cc_table_usage": "Come viene utilizzata questa tabella?", - "settings_manage_roles": "Come gestisco i ruoli utente e i permessi?", "settings_configure_jobs": "Aiutami a configurare i lavori in background." } diff --git a/src/frontend/src/i18n/locales/it/search.json b/src/frontend/src/i18n/locales/it/search.json index c3fbd5623..c012ec116 100644 --- a/src/frontend/src/i18n/locales/it/search.json +++ b/src/frontend/src/i18n/locales/it/search.json @@ -124,5 +124,10 @@ "deleteSessionFailed": "Eliminazione sessione non riuscita", "loadFailed": "Caricamento ricerca LLM non riuscito. Riprova più tardi." } + }, + "copilot": { + "scopePageSpecific": "Specifico della pagina", + "scopeGeneral": "Ontos (generale)", + "scopeThisPage": "Questa pagina" } } diff --git a/src/frontend/src/i18n/locales/ja/copilot-questions.json b/src/frontend/src/i18n/locales/ja/copilot-questions.json index 2e5694b0b..ccdca2357 100644 --- a/src/frontend/src/i18n/locales/ja/copilot-questions.json +++ b/src/frontend/src/i18n/locales/ja/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "顧客データはどこにありますか?", "global_business_terms_sales": "営業に関連するビジネス用語は何ですか?", "global_what_domains_exist": "組織で定義されているデータドメインは何ですか?", - "mp_browse_products": "顧客ドメインのデータプロダクトを表示してください。", "mp_product_cost": "すべてのデータプロダクトのコストはいくらですか?", "mp_subscribe_product": "データプロダクトをサブスクライブするにはどうすればよいですか?", - "dc_search_tables": "顧客情報を含むテーブルを検索してください。", "dc_find_columns": "すべてのテーブルからメールアドレス関連のカラムを検索してください。", - "search_across_all": "すべてのデータプロダクト、契約、用語を横断検索してください。", - "dp_list_domain": "私のドメインで利用可能なデータプロダクトは何ですか?", - "dp_show_contracts": "このデータプロダクトにリンクされている契約はどれですか?", + "dp_show_contracts": "{{entityName}}にリンクされている契約はどれですか?", + "dp_quality_score": "{{entityName}}のデータ品質スコアはどれくらいですか?", + "dp_owner": "{{entityName}}の所有者は誰ですか?", + "dp_schema": "{{entityName}}のスキーマを表示してください。", + "dp_last_updated": "{{entityName}}が最後に更新されたのはいつですか?", + "dp_subscribe": "{{entityName}}をサブスクライブするにはどうすればよいですか?", + "dp_consumers": "{{entityName}}を利用しているのは誰ですか?", "dp_draft_product": "既存のテーブルから新しいデータプロダクトの草案を作成してください。", "dp_package_tables": "分析テーブルをデータプロダクトとしてパッケージ化してください。", "dp_add_output_port": "データプロダクトに出力ポートを追加するにはどうすればよいですか?", - + "dp_publication_status": "{{entityName}}の公開ステータスはどうなっていますか?", + "dp_publish_blockers": "{{entityName}}の公開を妨げているものは何ですか?", "ct_show_failing": "品質チェックに失敗している契約を表示してください。", - "ct_explain_quality": "この契約で定義されている品質チェックを説明してください。", + "ct_what_covers": "{{entityName}}は何をカバーしていますか?", + "ct_used_by": "{{entityName}}を使用しているデータプロダクトはどれですか?", + "ct_owner": "{{entityName}}の所有者は誰ですか?", "ct_create_contract": "営業スキーマ用の新しいデータ契約を作成してください。", - "ct_add_quality_check": "この契約に品質チェックを追加してください。", - + "ct_add_quality_check": "{{entityName}}に品質チェックを追加してください。", + "ct_version_impact": "{{entityName}}のバージョンを上げると何が変わりますか?", "sm_explain_concept_property": "コンセプトとプロパティの違いを説明してください。", "sm_browse_collections": "利用可能なコンセプトコレクションは何ですか?", "sm_define_vocabulary": "顧客ドメインの語彙を定義してください。", "sm_suggest_concepts": "サプライチェーンモデルにはどのようなコンセプトを作成すべきですか?", - "asset_find_unmapped": "セマンティックマッピングが必要なテーブルを見つけてください。", "asset_map_columns": "カラムをセマンティックプロパティにマッピングするにはどうすればよいですか?", - "asset_show_lineage": "このアセットのリネージを表示してください。", - + "asset_show_lineage": "{{entityName}}のリネージを表示してください。", + "asset_built_on": "{{entityName}}の上に構築されているデータプロダクトは何ですか?", + "asset_freshness": "{{entityName}}の鮮度はどうですか?", + "asset_quality": "{{entityName}}の品質ステータスはどうですか?", + "asset_consumers": "{{entityName}}を利用しているのは誰ですか?", "comp_low_scores": "コンプライアンススコアの低いデータプロダクトを表示してください。", "comp_failing_checks": "現在失敗しているコンプライアンスチェックはどれですか?", "comp_create_policy": "個人情報向けの新しいコンプライアンスポリシーを作成してください。", - "dom_list_domains": "すべてのデータドメインとその所有者を一覧表示してください。", - "dom_domain_health": "このドメインの健全性ステータスはどうですか?", + "dom_domain_health": "{{entityName}}の健全性ステータスはどうですか?", "dom_create_domain": "新しいデータドメインの作成を手伝ってください。", - + "dom_products_in": "{{entityName}}に属するデータプロダクトは何ですか?", + "dom_business_terms": "{{entityName}}に関連するビジネス用語は何ですか?", + "dom_owner": "{{entityName}}のガバナンス担当者は誰ですか?", + "dom_health_detail": "{{entityName}}の健全性はどうですか?", "rev_pending_reviews": "保留中のアセットレビューを表示してください。", "rev_start_review": "テーブルの新しいレビューを開始してください。", - "gov_semantic_coverage": "先週セマンティックカバレッジが低下した理由を教えてください。", "gov_domains_ready": "エージェント利用の準備ができているドメインはどれですか?", - "cc_table_columns": "このテーブルにはどのようなカラムがありますか?", "cc_table_owner": "このテーブルの所有者は誰ですか?", "cc_table_usage": "このテーブルはどのように使用されていますか?", - "settings_manage_roles": "ユーザーロールと権限はどのように管理しますか?", "settings_configure_jobs": "バックグラウンドジョブの設定を手伝ってください。" } diff --git a/src/frontend/src/i18n/locales/ja/search.json b/src/frontend/src/i18n/locales/ja/search.json index 50c8c5193..260cf714c 100644 --- a/src/frontend/src/i18n/locales/ja/search.json +++ b/src/frontend/src/i18n/locales/ja/search.json @@ -124,5 +124,10 @@ "deleteSessionFailed": "セッションの削除に失敗しました", "loadFailed": "LLM検索の読み込みに失敗しました。後でもう一度お試しください。" } + }, + "copilot": { + "scopePageSpecific": "ページ固有", + "scopeGeneral": "Ontos(全般)", + "scopeThisPage": "このページ" } } diff --git a/src/frontend/src/i18n/locales/nl/copilot-questions.json b/src/frontend/src/i18n/locales/nl/copilot-questions.json index f54cee3b2..7fee8eb67 100644 --- a/src/frontend/src/i18n/locales/nl/copilot-questions.json +++ b/src/frontend/src/i18n/locales/nl/copilot-questions.json @@ -9,54 +9,60 @@ "global_find_customer_data": "Waar kan ik klantgegevens vinden?", "global_business_terms_sales": "Welke bedrijfstermen zijn gerelateerd aan verkoop?", "global_what_domains_exist": "Welke datadomeinen zijn gedefinieerd in de organisatie?", - "mp_browse_products": "Toon mij dataproducten in het Klantdomein.", "mp_product_cost": "Hoeveel kosten alle dataproducten?", "mp_subscribe_product": "Hoe abonneer ik me op een dataproduct?", - "dc_search_tables": "Zoek naar tabellen met klantinformatie.", "dc_find_columns": "Vind kolommen gerelateerd aan e-mailadressen in alle tabellen.", - "search_across_all": "Zoek in alle dataproducten, contracten en termen.", - "dp_list_domain": "Welke dataproducten zijn beschikbaar in mijn domein?", - "dp_show_contracts": "Welke contracten zijn gekoppeld aan dit dataproduct?", + "dp_show_contracts": "Welke contracten zijn gekoppeld aan {{entityName}}?", + "dp_quality_score": "Wat is de datakwaliteitsscore van {{entityName}}?", + "dp_owner": "Wie is de eigenaar van {{entityName}}?", + "dp_schema": "Toon mij het schema van {{entityName}}.", + "dp_last_updated": "Wanneer is {{entityName}} voor het laatst bijgewerkt?", + "dp_subscribe": "Hoe abonneer ik me op {{entityName}}?", + "dp_consumers": "Wie gebruikt {{entityName}}?", "dp_draft_product": "Help me een nieuw dataproduct te ontwerpen op basis van bestaande tabellen.", "dp_package_tables": "Help me analysetabellen als dataproduct te verpakken.", "dp_add_output_port": "Hoe voeg ik een uitvoerpoort toe aan een dataproduct?", - + "dp_publication_status": "Wat is de publicatiestatus van {{entityName}}?", + "dp_publish_blockers": "Wat blokkeert de publicatie van {{entityName}}?", "ct_show_failing": "Toon mij contracten met mislukte kwaliteitscontroles.", - "ct_explain_quality": "Leg de kwaliteitscontroles uit die in dit contract zijn gedefinieerd.", + "ct_what_covers": "Wat dekt {{entityName}}?", + "ct_used_by": "Welke dataproducten gebruiken {{entityName}}?", + "ct_owner": "Wie is de eigenaar van {{entityName}}?", "ct_create_contract": "Maak een nieuw datacontract voor mijn verkoopschema.", - "ct_add_quality_check": "Help me een kwaliteitscontrole toe te voegen aan dit contract.", - + "ct_add_quality_check": "Help me een kwaliteitscontrole toe te voegen aan {{entityName}}.", + "ct_version_impact": "Wat verandert er als ik de versie van {{entityName}} verhoog?", "sm_explain_concept_property": "Leg het verschil uit tussen een concept en een eigenschap.", "sm_browse_collections": "Welke conceptcollecties zijn beschikbaar?", "sm_define_vocabulary": "Help me een vocabulaire te definiëren voor ons Klantdomein.", "sm_suggest_concepts": "Welke concepten moet ik maken voor een supply chain model?", - "asset_find_unmapped": "Vind tabellen die semantische mappings nodig hebben.", "asset_map_columns": "Hoe map ik kolommen naar semantische eigenschappen?", - "asset_show_lineage": "Toon mij de herkomst van dit asset.", - + "asset_show_lineage": "Toon mij de herkomst van {{entityName}}.", + "asset_built_on": "Welke dataproducten zijn gebouwd op {{entityName}}?", + "asset_freshness": "Wat is de versheid van {{entityName}}?", + "asset_quality": "Wat is de kwaliteitsstatus van {{entityName}}?", + "asset_consumers": "Wie gebruikt {{entityName}}?", "comp_low_scores": "Toon dataproducten met lage compliance-scores.", "comp_failing_checks": "Welke compliance-controles falen momenteel?", "comp_create_policy": "Maak een nieuw compliance-beleid voor persoonsgegevens.", - "dom_list_domains": "Lijst alle datadomeinen en hun eigenaren op.", - "dom_domain_health": "Wat is de gezondheidsstatus van dit domein?", + "dom_domain_health": "Wat is de gezondheidsstatus van {{entityName}}?", "dom_create_domain": "Help me een nieuw datadomein te maken.", - + "dom_products_in": "Welke dataproducten horen bij {{entityName}}?", + "dom_business_terms": "Welke bedrijfstermen zijn gekoppeld aan {{entityName}}?", + "dom_owner": "Wie beheert {{entityName}}?", + "dom_health_detail": "Hoe gezond is {{entityName}}?", "rev_pending_reviews": "Toon mij openstaande asset-beoordelingen.", "rev_start_review": "Start een nieuwe beoordeling voor een tabel.", - "gov_semantic_coverage": "Waarom is de semantische dekking vorige week gedaald?", "gov_domains_ready": "Welke domeinen zijn klaar voor agentgebruik?", - "cc_table_columns": "Welke kolommen heeft deze tabel?", "cc_table_owner": "Wie is de eigenaar van deze tabel?", "cc_table_usage": "Hoe wordt deze tabel gebruikt?", - "settings_manage_roles": "Hoe beheer ik gebruikersrollen en machtigingen?", "settings_configure_jobs": "Help me achtergrondtaken te configureren." } diff --git a/src/frontend/src/i18n/locales/nl/search.json b/src/frontend/src/i18n/locales/nl/search.json index ed0b3c55a..8300caad0 100644 --- a/src/frontend/src/i18n/locales/nl/search.json +++ b/src/frontend/src/i18n/locales/nl/search.json @@ -124,6 +124,11 @@ "deleteSessionFailed": "Kon sessie niet verwijderen", "loadFailed": "Kon LLM zoeken niet laden. Probeer het later opnieuw." } + }, + "copilot": { + "scopePageSpecific": "Pagina-specifiek", + "scopeGeneral": "Ontos (algemeen)", + "scopeThisPage": "Deze pagina" } } diff --git a/src/frontend/src/stores/copilot-store.ts b/src/frontend/src/stores/copilot-store.ts index 8d20a6aee..ce481817a 100644 --- a/src/frontend/src/stores/copilot-store.ts +++ b/src/frontend/src/stores/copilot-store.ts @@ -13,23 +13,44 @@ export interface CopilotPageContext { selectedEntity?: CopilotEntity; } +/** + * Context scope toggle, surfaced via the "Asking about" chip dropdown. + * + * - `'page'`: the copilot uses the current `pageContext` (page name, + * feature id, selected entity) to bias both the starter prompts and + * the chat payload — same behavior as before this setting existed. + * - `'general'`: the copilot ignores page context. Starter prompts + * shrink to global ones and the chat request omits the page-context + * fields so the backend treats it as a scope-free question. + */ +export type CopilotContextScope = 'page' | 'general'; + interface CopilotState { isOpen: boolean; pageContext: CopilotPageContext | null; + contextScope: CopilotContextScope; actions: { togglePanel: () => void; openPanel: () => void; closePanel: () => void; setContext: (pageName: string, pageUrl: string, selectedEntity?: CopilotEntity, featureId?: string) => void; clearContext: () => void; + setContextScope: (scope: CopilotContextScope) => void; }; } const VISITED_KEY = 'copilot-sidebar-visited'; +const CONTEXT_SCOPE_KEY = 'copilot-context-scope'; + +function loadContextScope(): CopilotContextScope { + const stored = localStorage.getItem(CONTEXT_SCOPE_KEY); + return stored === 'general' ? 'general' : 'page'; +} export const useCopilotStore = create<CopilotState>()((set) => ({ isOpen: localStorage.getItem(VISITED_KEY) !== 'true', pageContext: null, + contextScope: loadContextScope(), actions: { togglePanel: () => set((state) => { if (state.isOpen) localStorage.setItem(VISITED_KEY, 'true'); @@ -43,5 +64,9 @@ export const useCopilotStore = create<CopilotState>()((set) => ({ setContext: (pageName, pageUrl, selectedEntity, featureId) => set({ pageContext: { pageName, pageUrl, featureId, selectedEntity } }), clearContext: () => set({ pageContext: null }), + setContextScope: (scope) => { + localStorage.setItem(CONTEXT_SCOPE_KEY, scope); + set({ contextScope: scope }); + }, }, })); diff --git a/src/frontend/src/types/llm-search.ts b/src/frontend/src/types/llm-search.ts index ef9edd141..0b430cdfb 100644 --- a/src/frontend/src/types/llm-search.ts +++ b/src/frontend/src/types/llm-search.ts @@ -140,11 +140,19 @@ export interface ToolSource { error?: string; } +export type AdoptionMode = 'blank' | 'active'; + export interface LLMSearchStatus { enabled: boolean; endpoint?: string; model_name?: string; disclaimer: string; + /** + * Workspace adoption mode used to pick mode-aware starter prompts. + * `null` (or absent) means the backend snapshot was unavailable; + * the UI should fall back to its default starter list in that case. + */ + adoption_mode?: AdoptionMode | null; }