diff --git a/alembic/versions/20260624_000010_add_generation_run_tables.py b/alembic/versions/20260624_000010_add_generation_run_tables.py new file mode 100644 index 00000000..6fc6472b --- /dev/null +++ b/alembic/versions/20260624_000010_add_generation_run_tables.py @@ -0,0 +1,194 @@ +"""Add durable generation-run and event-log tables.""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "20260624_000010" +down_revision = "20260601_000009" +branch_labels = None +depends_on = None + + +def _enum(name: str, *values: str) -> postgresql.ENUM: + """Return an existing-or-creatable PostgreSQL enum.""" + return postgresql.ENUM(*values, name=name, create_type=False) + + +def _create_enums() -> None: + """Create generation-run PostgreSQL enums.""" + _enum( + "generation_run_status", + "pending", + "running", + "paused", + "succeeded", + "failed", + "cancelled", + ).create(op.get_bind(), checkfirst=True) + _enum("quality_mode", "draft_without_qa").create(op.get_bind(), checkfirst=True) + _enum("qa_status", "skipped").create(op.get_bind(), checkfirst=True) + + +def _drop_enums() -> None: + """Drop generation-run PostgreSQL enums.""" + _enum("qa_status", "skipped").drop(op.get_bind(), checkfirst=True) + _enum("quality_mode", "draft_without_qa").drop(op.get_bind(), checkfirst=True) + _enum( + "generation_run_status", + "pending", + "running", + "paused", + "succeeded", + "failed", + "cancelled", + ).drop(op.get_bind(), checkfirst=True) + + +def _create_generation_runs_table() -> None: + """Create durable generation-run records.""" + op.create_table( + "generation_runs", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + "episode_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("episodes.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "source_bundle_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("ingestion_jobs.id"), + nullable=False, + ), + sa.Column("actor", sa.String(length=240), nullable=False), + sa.Column( + "status", + _enum( + "generation_run_status", + "pending", + "running", + "paused", + "succeeded", + "failed", + "cancelled", + ), + nullable=False, + ), + sa.Column("current_node", sa.String(length=160), nullable=True), + sa.Column("budget_snapshot", postgresql.JSONB(), nullable=False), + sa.Column("configuration", postgresql.JSONB(), nullable=False), + sa.Column( + "quality_mode", + _enum("quality_mode", "draft_without_qa"), + nullable=False, + ), + sa.Column("qa_status", _enum("qa_status", "skipped"), nullable=True), + sa.Column("skip_qa_rationale", sa.Text(), nullable=True), + sa.Column( + "idempotency_principal_id", + sa.String(length=200), + nullable=False, + ), + sa.Column("idempotency_key", sa.String(length=512), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("error_category", sa.String(length=120), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint( + "idempotency_principal_id", + "idempotency_key", + name="uq_generation_runs_idempotency_principal_key", + ), + ) + op.create_index( + "ix_generation_runs_episode_id", + "generation_runs", + ["episode_id"], + ) + op.create_index("ix_generation_runs_status", "generation_runs", ["status"]) + op.create_index( + "ix_generation_runs_started_at", + "generation_runs", + ["started_at"], + ) + + +def _drop_generation_runs_table() -> None: + """Drop durable generation-run records.""" + op.drop_index("ix_generation_runs_started_at", table_name="generation_runs") + op.drop_index("ix_generation_runs_status", table_name="generation_runs") + op.drop_index("ix_generation_runs_episode_id", table_name="generation_runs") + op.drop_table("generation_runs") + + +def _create_generation_events_table() -> None: + """Create append-only generation-run event records.""" + op.create_table( + "generation_events", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + "generation_run_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("generation_runs.id"), + nullable=False, + ), + sa.Column("seq", sa.Integer(), nullable=False), + sa.Column("kind", sa.String(length=160), nullable=False), + sa.Column("payload", postgresql.JSONB(), nullable=False), + sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint( + "generation_run_id", + "seq", + name="uq_generation_events_run_seq", + ), + ) + op.create_index( + "ix_generation_events_generation_run_id", + "generation_events", + ["generation_run_id"], + ) + + +def _drop_generation_events_table() -> None: + """Drop append-only generation-run event records.""" + op.drop_index( + "ix_generation_events_generation_run_id", + table_name="generation_events", + ) + op.drop_table("generation_events") + + +def upgrade() -> None: + """Create generation-run persistence tables.""" + _create_enums() + _create_generation_runs_table() + _create_generation_events_table() + + +def downgrade() -> None: + """Drop generation-run persistence tables.""" + _drop_generation_events_table() + _drop_generation_runs_table() + _drop_enums() diff --git a/alembic/versions/20260624_000011_add_episode_tei_revisioning.py b/alembic/versions/20260624_000011_add_episode_tei_revisioning.py new file mode 100644 index 00000000..b091c571 --- /dev/null +++ b/alembic/versions/20260624_000011_add_episode_tei_revisioning.py @@ -0,0 +1,71 @@ +"""Add episode TEI revision metadata.""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "20260624_000011" +down_revision = "20260624_000010" +branch_labels = None +depends_on = None + + +def _qa_status_enum() -> postgresql.ENUM: + """Return the existing QA status PostgreSQL enum.""" + return postgresql.ENUM("skipped", name="qa_status", create_type=False) + + +def upgrade() -> None: + """Add optimistic TEI revision metadata to episodes.""" + op.add_column( + "episodes", + sa.Column("tei_revision", sa.Integer(), server_default="1", nullable=False), + ) + op.create_check_constraint( + "ck_episodes_tei_revision_positive", + "episodes", + "tei_revision >= 1", + ) + op.add_column( + "episodes", + sa.Column("tei_content_hash", sa.String(length=128), nullable=True), + ) + op.add_column( + "episodes", + sa.Column("qa_status", _qa_status_enum(), nullable=True), + ) + op.add_column( + "episodes", + sa.Column( + "last_generation_run_id", + postgresql.UUID(as_uuid=True), + nullable=True, + ), + ) + op.create_foreign_key( + "fk_episodes_last_generation_run_id_generation_runs", + "episodes", + "generation_runs", + ["last_generation_run_id"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade() -> None: + """Remove optimistic TEI revision metadata from episodes.""" + op.drop_constraint( + "fk_episodes_last_generation_run_id_generation_runs", + "episodes", + type_="foreignkey", + ) + op.drop_column("episodes", "last_generation_run_id") + op.drop_column("episodes", "qa_status") + op.drop_column("episodes", "tei_content_hash") + op.drop_constraint( + "ck_episodes_tei_revision_positive", + "episodes", + type_="check", + ) + op.drop_column("episodes", "tei_revision") diff --git a/alembic/versions/20260624_000012_add_ingestion_job_owner.py b/alembic/versions/20260624_000012_add_ingestion_job_owner.py new file mode 100644 index 00000000..34fbaae2 --- /dev/null +++ b/alembic/versions/20260624_000012_add_ingestion_job_owner.py @@ -0,0 +1,29 @@ +"""Add an authenticated-principal owner to ingestion jobs.""" + +import sqlalchemy as sa + +from alembic import op + +revision = "20260624_000012" +down_revision = "20260624_000011" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Persist optional ownership for jobs created before authentication.""" + op.add_column( + "ingestion_jobs", + sa.Column("owner_principal_id", sa.String(length=240), nullable=True), + ) + op.create_index( + "ix_ij_owner_created", + "ingestion_jobs", + ["owner_principal_id", "created_at", "id"], + ) + + +def downgrade() -> None: + """Remove ingestion-job ownership.""" + op.drop_index("ix_ij_owner_created", "ingestion_jobs") + op.drop_column("ingestion_jobs", "owner_principal_id") diff --git a/docs/adr/adr-009-source-to-script-rest-vertical-slice.md b/docs/adr/adr-009-source-to-script-rest-vertical-slice.md index a3aabf34..c5f5c97d 100644 --- a/docs/adr/adr-009-source-to-script-rest-vertical-slice.md +++ b/docs/adr/adr-009-source-to-script-rest-vertical-slice.md @@ -127,7 +127,9 @@ vertical-slice tasks: The first task owns upload resources, ingestion-job source attachment, and presenter-profile use through existing reusable reference-document semantics. The second task depends on the first task and owns the generation-run contract, -the explicit no-QA quality mode, run polling, and TEI retrieval. +the explicit no-QA quality mode, run polling, and TEI retrieval. ADR 017 +records its execution, persistence, recovery-hook, and content-negotiation +decisions. The target API is `/v1`. Existing unversioned routes do not need preservation when the vertical slice is implemented because the project has not reached @@ -248,7 +250,8 @@ the same first-write-wins and conflict-detection invariants. ## References -See also the system design[^7] and TUI API design.[^8] +See also the system design[^7], TUI API design,[^8] and the no-QA execution +decision.[^9] [^1]: [RFC 6129: The `application/tei+xml` media type](https://datatracker.ietf.org/doc/html/rfc6129) [^2]: [ADR 014: Hexagonal architecture enforcement](adr-014-hexagonal-architecture-enforcement.md) @@ -258,3 +261,4 @@ See also the system design[^7] and TUI API design.[^8] [^6]: [Stripe idempotent request guidance](https://docs.stripe.com/api/idempotent_requests) [^7]: [Episodic podcast generation system design](../episodic-podcast-generation-system-design.md) [^8]: [Episodic TUI API design](../episodic-tui-api-design.md) +[^9]: [ADR 017: No-QA generation execution and TEI persistence](adr-017-no-qa-generation-run-execution-and-tei-persistence.md) diff --git a/docs/adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md b/docs/adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md new file mode 100644 index 00000000..6e153c5e --- /dev/null +++ b/docs/adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md @@ -0,0 +1,132 @@ +# ADR-017: No-QA generation execution and TEI persistence + +## Status + +Accepted for roadmap item `4.3.2`. + +## Context + +ADR 009 defines the source-to-script REST vertical slice. Its second task must +turn a ready ingestion job into a durable generation run, execute one draft +without the full quality-assurance (QA) graph, persist validated Text Encoding +Initiative (TEI) P5, and expose polling and download resources. + +The broader Celery and LangGraph execution model is not yet ready to own this +slice. The first implementation still needs explicit lifecycle ownership, +recovery hooks, optimistic TEI updates, stable failures, and an upgrade path to +the later iterative workflow. + +## Decision + +Introduce `GenerationRunLauncher` as the scheduling port and implement +`InProcessGenerationRunLauncher` in the API process. The launcher is a +degenerate task-resume adapter: it accepts a run identifier, claims the pending +run conditionally, opens fresh units of work for background writes, and records +ordered lifecycle events. It bounds concurrency and keeps strong task +references so shutdown can drain or cancel scheduled work. Celery dispatch is +deferred until the worker boundary owns generation-run execution. + +The launcher resolves bound host and guest reference-document revisions for the +episode's series and supplies them, together with ingestion sources, to the +`DraftScriptGenerator` port. `LLMDraftScriptGenerator` is the single-pass +implementation. Roadmap item `4.4.1` may replace its one-pass policy with the +full duration-aware and QA-gated graph without changing the run or launcher +ports. + +Before a run is created, `materialise_episode_from_ingestion` creates the +canonical episode using the ready ingestion job as both source bundle and +stable episode identifier. Generated TEI is validated before persistence. +Episode updates increment `tei_revision`, retain the writing run identifier, +quality mode, QA status, and content hash, and use optimistic revision checks +to reject concurrent writers. + +The first deployment assumes one API worker owns an in-process run. Durable +schema fields record conditional pending-to-running claims, `started_at`, +`lease_expires_at`, and terminal error categories; SQL claim outcomes are also +logged. Lease fields support inspection only: operators may manually mark an +expired run failed, while automatic lease recovery and reassignment remain +roadmap item `2.6.2`. + +Admission is bounded before task allocation. The launcher admits at most +`max_concurrency + max_pending_runs` runs (currently four active and sixteen +additional pending by default). When that capacity is exhausted, the API +records a terminal `launcher.overloaded` failure on the already-created run and +returns `503 Service Unavailable`. Shutdown closes admission before it cancels +and drains the strong task registry, so no new work can race with teardown. + +Shutdown is serialized across the process boundaries: the launcher is shut down +first, the LLM provider client is closed second, and the database engine is +disposed last. A cancelled task shields its terminal failure write, allowing +the run to receive `run.failed` with `launcher.shutdown` before the database +becomes unavailable. After draining, the launcher applies the same failure +write for tasks cancelled before `_run_task` begins; terminal guards prevent a +duplicate event when cancellation was already handled. A process restart still +loses the in-memory task registry; lease fields support inspection and the +documented privileged manual-failure procedure only. This slice deliberately +provides no automatic reaper, retry, or reassignment of expired runs. + +The API command and launcher execution are traced as separate spans. The SQL +store logs claim outcomes, and the test tracer remains injectable through the +same port. Production-wide metrics, lease/recovery metrics, and retry metrics +remain follow-up work; stable failure categories still make persisted outcomes +searchable without putting run identifiers into metric labels. + +Each generation run has an authenticated principal owner. Production runtime +configuration requires a bearer credential and principal identifier, and the +API derives the persisted run actor from that trusted principal rather than the +request body. Resource reads authorize against that owner and use the same +not-found response for absent and inaccessible runs, events, ingestion jobs, +and generated TEI. + +The claim transaction commits the conditional status transition and +`run.started` event before a fresh read unit of work loads the episode, +source-document metadata, and presenter bindings. That unit of work closes +before upload content is hydrated. The launcher then hydrates bounded source +content from object storage outside the database unit of work, so potentially +slow object-store I/O neither retains the claim lock nor monopolizes a database +connection. The source limits bound count, each uploaded stream, aggregate +input, and normalized text; an overflow produces the stable +`generation.source_limit` terminal category without retaining or emitting +source content. + +Generation-run creation accepts only `quality_mode=draft_without_qa` in this +slice. A recognized `qa_gated` request returns `422 Unprocessable Entity`; +malformed or missing required fields return `400 Bad Request`. Episode TEI +returns `404 Not Found` until a generated draft and its provenance metadata +exist. + +`GET /v1/episodes/{episode_id}/tei` uses HTTP content negotiation rather than a +separate export resource. The default representation is a JSON envelope. +`Accept: application/tei+xml` returns raw XML with `Content-Disposition`, +`ETag`, and the TEI media type. JSON and XML have representation-specific +ETags; a matching `If-None-Match` returns `304 Not Modified` without a body. +Unsupported media types return `406 Not Acceptable`. + +## Consequences + +### Positive + +- Clients can create, replay, poll, and diagnose durable generation runs. +- The process-local launcher is replaceable without changing HTTP or domain + contracts. +- Presenter profiles and ingestion sources reach the generator through + canonical ports rather than transport data. +- Optimistic TEI revisioning prevents silent concurrent overwrites. +- Raw TEI download does not depend on audio or export-job infrastructure. + +### Negative + +- In-process work is not shared across API replicas and cannot survive process + loss. Deploy this slice with one owning worker until Celery dispatch lands. +- Automatic stuck-run recovery is not included; operators must use lease, + status, and event evidence for manual intervention. +- Episode materialization currently reuses the ingestion-job identifier, which + couples the first generation route to the intake bundle identity. +- No-QA output is explicitly a draft and must not be represented as approved. + +## References + +- [ADR 007: Durable generation checkpoints](adr-007-durable-generation-checkpoints.md) +- [ADR 009: Source-to-script REST vertical slice](adr-009-source-to-script-rest-vertical-slice.md) +- [ADR 015: Upload and idempotency ports](adr-015-upload-and-idempotency-ports.md) +- [Episodic podcast generation system design](../episodic-podcast-generation-system-design.md) diff --git a/docs/contents.md b/docs/contents.md index bf80ee56..8c013ee6 100644 --- a/docs/contents.md +++ b/docs/contents.md @@ -101,11 +101,19 @@ or delivery planning. - source-intake upload storage and idempotency port decisions. - [ADR 016: Adopt Skylos dead-code detection](adr/adr-016-adopt-skylos-dead-code-detection.md) - blocking static dead-code detection and exception policy. +- [ADR 017: No-QA generation execution and TEI persistence][adr-017] + - generation launcher, draft persistence, recovery, and TEI retrieval + decisions. + +[adr-017]: adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md ## Execution plans - [Benchmark pyscn and Skylos dead-code detection](execplans/benchmark-pyscn-skylos-dead-code.md) - comparative evaluation plan for the two Python dead-code scanners. +- [No-QA generation runs and TEI-P5 retrieval]( + execplans/4-3-2-no-qa-generation-runs-and-tei-p5-retrieval.md) + - implementation plan for roadmap task 4.3.2. - [Reference binding resolution](execplans/1-4-3-reference-binding-resolution.md) - implementation plan for roadmap task 1.4.3. - [Scaffold Falcon HTTP services on Granian](execplans/1-5-1-scaffold-falcon-http-services-on-granian.md) diff --git a/docs/developers-guide.md b/docs/developers-guide.md index ed626d46..81317eb9 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -22,8 +22,11 @@ Accepted design decisions relevant to current implementation work: - [`adr-014-hexagonal-architecture-enforcement.md`](adr/adr-014-hexagonal-architecture-enforcement.md) - [`adr-015-generation-run-port-split.md`](adr/adr-015-generation-run-port-split.md) - [`adr-015-upload-and-idempotency-ports.md`](adr/adr-015-upload-and-idempotency-ports.md) +- [`adr-017-no-qa-generation-run-execution-and-tei-persistence.md`][adr-017] - [`episodic-podcast-generation-system-design.md`](episodic-podcast-generation-system-design.md) +[adr-017]: adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md + ## Local development - Use `uv` to manage the virtual environment and dependencies. @@ -263,6 +266,22 @@ Runtime environment: `FilesystemObjectStore` for source-intake upload bytes. The runtime fails fast when the value is missing, because `POST /v1/uploads` cannot accept payloads without an object-store adapter. +- `OPENAI_BASE_URL` and `OPENAI_API_KEY` are optional paired settings. Set both + to configure the OpenAI-compatible provider; setting only one is invalid and + prevents startup. +- `DRAFT_MODEL` defaults to `gpt-4o-mini`. When set explicitly, it must be a + non-empty string after whitespace is trimmed. +- `GENERATION_MAX_OUTPUT_TOKENS` and `GENERATION_MAX_RESPONSE_BYTES` are + optional positive-integer settings. They default to `4096` and `1048576`, + respectively. The token setting becomes the output-token cap passed to the + LLM request; the byte setting is enforced against the UTF-8 provider response + before the generated JSON is parsed. +- When the OpenAI settings are configured, startup builds an + `OpenAICompatibleLLMAdapter` and an `InProcessGenerationRunLauncher`. When + they are not configured, generation-run creation returns + `503 Service Unavailable` because no launcher is available. +- During shutdown, the runtime shuts down the launcher (cancelling and + draining its scheduled tasks) before closing the provider client. Health contract: @@ -324,9 +343,12 @@ Authorization scaffold: - Every `/v1` request passes through `AuthorizationMiddleware` before resource dispatch. Health checks remain operator endpoints and are not authorized by this scaffold. -- `ApiDependencies.authorization` accepts an `AuthorizationPort`; production - wiring currently defaults to `PermitAll`, so existing clients do not need an - `Authorization` header yet. +- `ApiDependencies.authorization` accepts an `AuthorizationPort`. Production + requests must carry `Authorization: Bearer ` and production wiring uses + `StaticBearerTokenAuthorization`, configured by the required + `API_AUTHORIZATION_BEARER_TOKEN` and `API_AUTHORIZATION_PRINCIPAL_ID` + settings. Test composition may use `PermitAll` deliberately; it is not part + of the production runtime. - Authorization adapters receive an `AuthorizationContext` containing the HTTP method, request path, and raw `Authorization` header. The port is async, so future policy adapters can call external identity or permission services. @@ -335,12 +357,16 @@ Authorization scaffold: the Falcon request context before resource dispatch; source-intake idempotency scopes keys from that trusted context rather than from client-controlled principal headers. +- Generation-run creation records that trusted principal as the run actor and + ignores the request body's actor value. Ingestion jobs retain their owner so + creation, run polling, event listing, and TEI retrieval can return the same + not-found response for absent and inaccessible resources. - Non-permit decisions short-circuit with the canonical error envelope: `unauthorized` returns `401`, and `forbidden` returns `403`. - Authorization adapter failures short-circuit with `service_unavailable` and `503`, so policy-backend outages are not reported as resource failures. -- Roadmap item `5.1` is expected to replace the default permit-all adapter with - policy-backed role or scope checks. +- Roadmap item `5.1` is expected to add policy-backed role or scope checks + behind the authorization port. Testing guidance: @@ -1479,6 +1505,13 @@ The port surface is intentionally split: - `GenerationRunRepository` creates, fetches, lists, and updates run state. - `GenerationEventLog` appends events and allocates per-run `EventSeq` values inside the adapter. +- `GenerationRunEventStore` composes the run repository and event log for + durable generation-run storage. `CanonicalUnitOfWork.generation_runs` exposes + this port; `SqlAlchemyUnitOfWork` binds a `SqlAlchemyGenerationRunStore` to + its session, so callers use the same unit of work and own its commit or + rollback. `GenerationRunStorageRuntime` supplies the adapter's clock and UUID + providers, with production defaults and deterministic injection available to + tests. - `GenerationCheckpointPort` creates checkpoints and records reviewer responses through the `Checkpoint.respond(...)` domain transition. - `GenerationRunPort` composes the three sub-ports for callers that need the @@ -1496,6 +1529,202 @@ review checkpoint attached to a generation run. It is not the same as suspend/resume state through `CheckpointPort`. Do not convert between the two implicitly; bridge logic belongs in the later orchestration and REST work. +### No-QA launcher and TEI retrieval + +The no-QA path is deliberately a small application service around the +`GenerationRunLauncher` scheduling seam. `GenerationRunsResource` creates the +run only after `materialise_episode_from_ingestion` has materialized a ready +ingestion job into a canonical episode, placeholder TEI header, and source +documents. It commits that request-scoped unit of work before calling +`launch(run_id)`. `EpisodeTeiResource` remains the retrieval boundary: it +serves the persisted episode TEI as JSON by default or raw +`application/tei+xml` with content negotiation. + +`DraftScriptGenerator` is the generation seam. The launcher projects canonical +source documents and resolved host or guest reference-document revisions into +`DraftScriptRequest`; `LLMDraftScriptGenerator` is the current single-pass +`LLMPort` implementation. The generator returns `DraftScriptResult`, but does +not open a unit of work or persist domain entities. Lifecycle repositories +receive the frozen, slotted `GenerationRunStatusUpdate` value object so status, +current node, terminal time, and optional failure details travel as one +immutable command. + +`persist_draft_script` is the persistence boundary. It validates the generated +TEI, applies the expected `tei_revision` check, and updates the episode with +the TEI, content hash, generation-run identifier, and `QaStatus.SKIPPED`. The +`EpisodeTeiUpdate` command carries that TEI, QA status, generation-run +provenance, timestamp, and expected revision as one immutable update. The SQL +episode repository updates only when the stored revision equals +`expected_revision`, then increments the revision; a lost compare-and-set is +reported as an episode revision conflict. The surrounding unit of work still +controls commit or rollback. Materialization and persistence use +`CanonicalUnitOfWork` repository ports and leave commit or rollback to their +caller. The launcher therefore uses detached units of work: one to claim and +load inputs, another for `draft.generated`, and another to persist TEI, cost +records, ordered terminal events, and status. A request-scoped unit of work +must never be captured by a background task. + +When configured, `CostRecorder` records the provider call and final run roll-up +in the persistence unit of work. It first pins the immutable provider pricing +selected for the run, then records usage with a run-scoped idempotency key. +`PRICING_SNAPSHOT_DIRECTORY` is optional: its default is +`config/pricing-snapshots`; a configured relative path is resolved from the +repository root, and startup rejects a path that is not an existing directory. +The runtime constructs `FilePricingCatalogue` from the validated directory, so +the same pricing source is used when costs are recorded. + +Generation input is bounded before it reaches the draft provider. The optional +`GENERATION_MAX_SOURCE_COUNT`, `GENERATION_MAX_SOURCE_BYTES`, +`GENERATION_MAX_AGGREGATE_SOURCE_BYTES`, and +`GENERATION_MAX_NORMALIZED_SOURCE_BYTES` settings must be positive integers. +The launcher rejects a source as soon as its stream or normalized text exceeds +one of these limits and records the stable `generation.source_limit` failure +category without logging source text. The claim transaction commits the +conditional claim and `run.started` event. A detached read unit of work then +loads the episode, source-document metadata, and presenter bindings and closes +before bounded object-store or filesystem streaming starts. Hydration and text +normalization therefore happen after the metadata read unit of work has closed, +so external reads do not retain a generation-run row lock or database +connection. + +Generation-run polling, event listing, and TEI retrieval emit the bounded +`generation_run.read`, `generation_run.events.list`, and `episode_tei.read` +spans. Their only route attributes are operation, outcome, representation, +pagination mode, and stable failure category; they never include content, +principal identifiers, idempotency keys, or run identifiers. + +The API composition root emits `generation_api_request_total` and +`generation_api_request_latency_ms` for generation creation, polling, event +listing, and TEI retrieval. Their only labels are `operation` and the bounded +`outcome` (`success`, `rejected`, `failed`, or `not_modified` for TEI cache +responses); they contain no request data or principal identity. + +`GET /v1/generation-runs/{run_id}/events` returns append-only events after an +optional `after_seq` cursor, with `items`, `after_seq`, `limit`, `offset`, and +`total` in its response envelope. It orders by sequence and treats `after_seq` +as an exclusive lower bound. Cursor pagination is distinct from the general +offset-pagination convention: callers may use `after_seq` or a non-zero +`offset`, but not both; the API rejects that combination before the event store +is queried. + +Admission is bounded before task allocation. The default capacity is four +running tasks plus sixteen admitted pending tasks; exhaustion raises +`GenerationRunAdmissionError`, and the API records `launcher.overloaded` on the +run before returning `503 Service Unavailable`. Claims remain conditional +`pending -> running` transitions with a lease. A process restart does not +recover the in-memory task registry: inspect an expired lease and use the +documented manual-failure procedure, because this slice has no automatic +reaper, retry, or reassignment path. + +The lifecycle ordering is part of the contract. `launch` retains a strong task +reference; `drain` waits for all retained tasks; `shutdown` closes admission, +cancels unfinished tasks, and drains them. Cancellation shields its failure +recording so the run receives `run.failed` with `launcher.shutdown`. Runtime +shutdown calls launcher shutdown, then closes the LLM provider, and only then +disposes the database engine. After the drain, shutdown applies a fallback +cancellation write for each task that was unfinished when cancellation began. +This covers tasks cancelled before `_run_task` starts; terminal-state guards +prevent duplicate failure events when the task handled cancellation itself. The +launcher emits a `generation_run.execute` span and bounded terminal, +draft-error, QA-bypass, and latency metrics; the API command span and stable +failure categories provide the corresponding request and outcome trace without +unbounded run identifiers in metric labels. + +### Manual recovery of expired generation-run leases + +Use this procedure when a process restart or worker loss leaves a run in +`running`. The `started_at` value records when the conditional claim won, and +`lease_expires_at` is the deadline used to identify an expired lease. Inspect +both fields with the run status and event log before taking action: + +```sql +SELECT id, status, started_at, lease_expires_at, + idempotency_principal_id, idempotency_key, + error_category, error_message +FROM generation_runs +WHERE id = :run_id; +``` + +Proceed only when the status is `running`, `lease_expires_at` is non-null and +earlier than the current UTC time, and the owning process is no longer able to +finish the run. A worker claim is itself conditional on `status = 'pending'`; a +claim that updates zero rows has lost the race or found a terminal run. Do not +reset an expired `running` row to `pending` or launch a replacement: this slice +has no automatic reaper or reassignment path. + +To fail the run manually, use a privileged database transaction. The locking +query below selects only a running run with a non-null expired lease. If it +returns no row, execute `ROLLBACK` and stop: this is an explicit no-op, and the +event and status update must not be executed. If it returns the requested run, +keep the transaction open, append the failure event, update the terminal state, +and then commit both writes together. + +```sql +BEGIN; + +SELECT id +FROM generation_runs +WHERE id = :run_id + AND status = 'running' + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= CURRENT_TIMESTAMP +FOR UPDATE; + +-- If no row is returned, execute ROLLBACK and stop. Do not execute the +-- statements below. + +-- After the qualifying row is locked, append run.failed with the same +-- error_category and error_message, using the next per-run event sequence. +INSERT INTO generation_events + (id, generation_run_id, seq, kind, payload, occurred_at, created_at) +SELECT :event_id, :run_id, + (SELECT COALESCE(MAX(seq), 0) + 1 + FROM generation_events + WHERE generation_run_id = :run_id), + 'run.failed', + jsonb_build_object( + 'error_category', 'launcher.lease_expired', + 'error_message', 'Generation lease expired; failed manually.' + ), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP; + +UPDATE generation_runs +SET status = 'failed', + current_node = 'failed', + ended_at = CURRENT_TIMESTAMP, + error_message = 'Generation lease expired; failed manually.', + error_category = 'launcher.lease_expired', + updated_at = CURRENT_TIMESTAMP +WHERE id = :run_id +RETURNING id; + +COMMIT; +``` + +The resulting semantics are compare-and-act: only the qualifying row is locked, +and a missing row causes an explicit rollback with no persisted event or status +change. The `run.failed` event and `failed` terminal update are committed +atomically after that lock succeeds. + +Keep `idempotency_principal_id` and `idempotency_key` unchanged. They remain +attached to the failed generation run so a replay is tied to the existing +record rather than creating an untracked replacement. Automatic recovery and +reassignment remain roadmap item `2.6.2`. + +TEI representation selection is centralized in +`episodic.api.resources.episode_tei.negotiate_tei_media_type`. Keep JSON as the +default, raw XML for `application/tei+xml`, and `406` for unsupported types. +Both serialized representations receive representation-specific strong ETags; a +matching `If-None-Match` (including `*`) returns `304` with no body. Apply +`Content-Disposition` only to the raw TEI representation. + +The end-to-end contract lives in +`tests/features/no_qa_generation_slice.feature`. Its steps start Vidai Mock +through the shared process helper, configure deterministic valid and invalid +draft completions, inject provider failure with `X-Vidai-Chaos-Drop`, and drive +the real Falcon, SQLAlchemy, launcher, and LLM adapter stack. CI installs the +pinned Vidai Mock binary before tests; local runs skip only when the binary is +absent. + ### Maintainer rules - Keep the planner strict: parse model output into typed DTOs immediately and diff --git a/docs/episodic-podcast-generation-system-design.md b/docs/episodic-podcast-generation-system-design.md index 4331eb68..dfbff9c0 100644 --- a/docs/episodic-podcast-generation-system-design.md +++ b/docs/episodic-podcast-generation-system-design.md @@ -998,6 +998,14 @@ post-merge `source_documents`. The first implementation serves only `POST /v1/uploads`; resumable upload initialization and direct byte `PUT` routes are deferred until a concrete S3-compatible adapter lands. +ADR 017 records the implemented second task. A process-local launcher claims a +pending run, resolves series-level host and guest profiles, invokes the +single-pass draft generator, and writes validated TEI through an optimistic +episode revision update. Run status and append-only events are durable even +though scheduling is currently in-process. Lease timestamps, conditional +claims, and stuck-run metrics support manual recovery until Celery assumes the +launcher port. + TEI retrieval remains attached to the episode resource rather than to export jobs. `GET /v1/episodes/{episode_id}/tei` returns a JSON envelope by default, including TEI XML, content hash, revision, last generation run id, quality @@ -1006,6 +1014,11 @@ the request uses `Accept: application/tei+xml`; this response uses the TEI media type registered by RFC 6129 and includes `Content-Disposition: attachment`. +Generation creation accepts only `draft_without_qa`; the run and episode both +record skipped-QA provenance and the supplied rationale. Recognized but +unsupported QA-gated requests return `422`, malformed requests return `400`, +and TEI retrieval returns `404` before generation has persisted a draft. + #### Content Request Decision Tree The decision tree below expands the routing logic that determines which diff --git a/docs/episodic-tui-api-design.md b/docs/episodic-tui-api-design.md index 2afc16b5..1c023540 100644 --- a/docs/episodic-tui-api-design.md +++ b/docs/episodic-tui-api-design.md @@ -225,11 +225,12 @@ ingestion-job source. ### Generation runs -Each generation run is a first-class resource with an append-only event log and -checkpoint support for human-in-the-loop intervention. +Each generation run is created from a ready ingestion job and is a first-class +resource with an append-only event log and checkpoint support for +human-in-the-loop intervention. ```plaintext -POST /v1/episodes/{episode_id}/generation-runs +POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs Body: template_id, quality_mode, prompt_overrides, budget_hints, skip_qa_rationale, actor Header: Idempotency-Key @@ -670,7 +671,8 @@ _Table: WebSocket close codes and their meanings._ ### Start run then watch events For screen readers: the following sequence diagram shows a TUI client starting -a generation run via REST, then subscribing to live events via WebSocket. +a generation run for a ready ingestion job via REST, then subscribing to live +events via WebSocket. ```mermaid sequenceDiagram @@ -679,7 +681,7 @@ sequenceDiagram participant WS as WS /ws/runs/{run_id} participant Orchestrator - TUI->>REST: POST /v1/episodes/{id}/generation-runs (Idempotency-Key: {uuid}) + TUI->>REST: POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs (Idempotency-Key: {uuid}) Note right of TUI: quality_mode=draft_without_qa REST-->>TUI: 202 Accepted {run_id} TUI->>WS: connect @@ -748,7 +750,7 @@ _Figure 3: Source upload and ingestion job lifecycle._ ### Source-to-script completion For screen readers: the following sequence diagram shows the REST-only path -from completed source ingestion to TEI-P5 XML download. +from a ready ingestion job to TEI-P5 XML download. ```mermaid sequenceDiagram @@ -756,7 +758,7 @@ sequenceDiagram participant REST as REST API participant Generator as Generation Orchestrator - Client->>REST: POST /v1/episodes/{id}/generation-runs (Idempotency-Key: {uuid}) + Client->>REST: POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs (Idempotency-Key: {uuid}) Note right of Client: quality_mode=draft_without_qa REST-->>Client: 202 Accepted {run_id} Client->>REST: GET /v1/generation-runs/{run_id} @@ -1095,11 +1097,12 @@ erDiagram _Figure: Sequence diagram illustrating the end-to-end flow for starting a generation run and observing events via WebSocket. The user configures -generation parameters through the TUI, which creates a run via REST and -establishes a WebSocket connection to subscribe to events. The orchestrator -publishes events as the run progresses, including checkpoints that require -human approval. The TUI acknowledges each event and submits checkpoint -responses via REST. The sequence concludes when the run completes successfully._ +generation parameters for a ready ingestion job through the TUI, which creates +a run via REST and establishes a WebSocket connection to subscribe to events. +The orchestrator publishes events as the run progresses, including checkpoints +that require human approval. The TUI acknowledges each event and submits +checkpoint responses via REST. The sequence concludes when the run completes +successfully._ ```mermaid sequenceDiagram @@ -1110,7 +1113,7 @@ sequenceDiagram participant Orchestrator User->>TUI: Configure generation parameters - TUI->>REST_v1: POST /v1/episodes/{episode_id}/generation-runs (Idempotency-Key: {uuid}) + TUI->>REST_v1: POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs (Idempotency-Key: {uuid}) REST_v1-->>TUI: 202 Accepted {run_id} TUI->>WS_runs: WebSocket connect /ws/runs/{run_id} diff --git a/docs/execplans/4-3-2-no-qa-generation-runs-and-tei-p5-retrieval.md b/docs/execplans/4-3-2-no-qa-generation-runs-and-tei-p5-retrieval.md new file mode 100644 index 00000000..d2741888 --- /dev/null +++ b/docs/execplans/4-3-2-no-qa-generation-runs-and-tei-p5-retrieval.md @@ -0,0 +1,2128 @@ +# Implement no-QA generation runs and TEI-P5 retrieval (4.3.2) + +This ExecPlan (execution plan) is a living document. The sections `Constraints`, +`Tolerances`, `Risks`, `Progress`, `Surprises & Discoveries`, `Decision Log`, +and `Outcomes & Retrospective` must be kept up to date as work proceeds. + +Status: COMPLETE for the core slice and the scoped PR #141 hardening follow-up + +Current implementation status (core slice completed 2026-07-22): Milestones 0-8 +and the post-implementation correctness review are complete. The REST +resources, TEI retrieval route, upload-backed source hydration, presenter +resolution, idempotent episode materialization, and serialized terminal updates +are implemented. The scoped hardening follow-up completed on 2026-08-20; +roadmap item 4.3.2 is complete for the delivered core slice. Automated +stuck-run recovery and broader operational follow-up remain outside this plan. + +## Purpose / big picture + +After this change an integration client can drive the narrowest useful +source-to-script workflow entirely through JSON/REST (Representational State +Transfer) without touching the quality-assurance (QA), audio, or export-job +pipelines: + +1. Upload show source material and attach presenter context (already delivered + by roadmap task 4.3.1). +2. Create a generation run for an episode with + `quality_mode=draft_without_qa`, a `skip_qa_rationale`, and actor metadata, + protected by an `Idempotency-Key`. +3. Poll the run resource and its append-only event log over REST until the run + reaches a terminal state. +4. Download the resulting Text Encoding Initiative (TEI) P5 script as either a + JSON metadata envelope or an `application/tei+xml` file. + +This validates the `/v1` resource contract defined in +[ADR 009](../adr/adr-009-source-to-script-rest-vertical-slice.md) before the +full approval, QA, audio, and export surfaces land. + +Success is observable by running the end-to-end behavioural scenario +`tests/features/no_qa_generation_slice.feature` (added by this plan) against a +running service backed by a Vidai Mock inference server: a `POST` to +`/v1/ingestion-jobs/{ingestion_job_id}/generation-runs` for a ready ingestion +job returns `202 Accepted` with a `Location` header, polling +`GET /v1/generation-runs/{run_id}` transitions from `pending` to `succeeded`, +and `GET /v1/episodes/{episode_id}/tei` with `Accept: application/tei+xml` +returns a downloadable TEI-P5 document whose `qa_status` is `skipped`. + +## Scope and roadmap relationship + +This task is the second half of the source-to-script vertical slice. It depends +on roadmap items 2.1.1 (the `LLMPort` adapter), 2.4.2 (LangGraph +suspend-and-resume orchestration), and 4.3.1 (source and presenter-profile +intake). + +The vertical-slice design in ADR 009 deliberately cuts across the horizontal +roadmap items 2.6.2 (generation-run persistence) and 2.6.3 (generation-run REST +endpoints). This plan therefore implements the **subset** of durable +generation-run persistence and REST endpoints that the no-QA slice needs, in a +shape that those later tasks extend rather than replace. Out of scope and left +to later tasks: human-review checkpoint persistence (2.6.2), the full +generation-run REST surface including the checkpoint endpoint (2.6.3), the +QA-gated execution graph and the full draft-generation graph (4.4.1), and an +automated stuck-run recovery worker (2.6.2). This plan does, however, add the +durable claim hooks (lease columns and conditional state transitions) and SQL +claim-outcome logging. Production-wide metrics, lease/recovery, and the later +worker remain follow-up work. + +Three slice-shaping decisions were confirmed with the requester before drafting +(see `Decision Log`): + +1. **Execution model:** in-process asynchronous execution behind a launcher + port, with durable run and event records so REST polling works across + requests. Celery dispatch is deferred (ADR 007 records that + LangGraph-to-Celery dispatch has not yet landed; `TaskResumePort` is the + future seam). The launcher is documented as a degenerate `TaskResumePort` so + 2.6.2/4.4.1 can converge it onto the durable checkpoint model. +2. **Episode provisioning:** the episode and its initial canonical TEI are + materialized from the ingestion job's attached sources, so the client flow + stays upload → ingest → generate without a separate episode-creation API. +3. **Generation engine:** a minimal but real single-pass, large language model + (LLM)-driven draft generator behind a `DraftScriptGenerator` port, producing + valid TEI-P5 spoken script, exercised through Vidai Mock. The full + QA-bypass-branching draft graph is left to roadmap item 4.4.1, which swaps + the engine behind the same port. + +## Constraints + +Hard invariants that must hold throughout implementation. Violation requires +escalation, not a workaround. + +- Respect the hexagonal architecture boundary rules enforced by + [ADR 014](../adr/adr-014-hexagonal-architecture-enforcement.md) and + `make lint` (`make check-architecture`). Domain code in `episodic/canonical/` + and `episodic/generation/` must not import Falcon, SQLAlchemy, httpx, Celery, + or LangGraph. The architecture groups are defined by exact module-prefix + lists in `pyproject.toml` (the `domain_ports`, `application`, + `outbound_adapter`, `inbound_adapter`, and `composition_root` groups). There + is **no blanket `episodic.canonical` prefix**, so every new module must be + added to the correct prefix list in `pyproject.toml` or it falls outside + enforcement. +- Preserve TEI-P5 as the canonical episode artefact. All script content is + produced, stored, and served as valid TEI-P5 XML parsed, validated, and + emitted through the `tei-rapporteur` library (see + [TEI Rapporteur users' guide](../tei-rapporteur-users-guide.md)). Do not add + a second TEI parser or hand-roll XML traversal (the spoken-text semantic + contract is owned by + [ADR 006](../adr/adr-006-chrono-spoken-text-semantics.md)). +- Honour the ADR 009 API contract exactly: `quality_mode=draft_without_qa` runs + record `qa_status=skipped`, the requesting actor, and a client-supplied + rationale; side-effecting `POST` requests accept `Idempotency-Key` with + first-write-wins semantics, identical-body replay (including the stored + `Location` and `Retry-After`), and `409 Conflict` on body mismatch; + long-running creation returns `202 Accepted` with `Location` and + `Retry-After`. +- The launcher's asynchronous task MUST own a fresh unit of work obtained from + `uow_factory` and MUST NOT reuse the request's unit of work, session, or any + repository bound to it. The request transaction is closed once the `202` + response is sent. +- All state transitions that can race across workers MUST be conditional + (compare-and-set): `pending → running` is a guarded `UPDATE`, and the episode + TEI write is optimistic on `tei_revision`. No blind status overwrites. +- The TEI download uses the `application/tei+xml` media type registered by + [RFC 6129](https://datatracker.ietf.org/doc/html/rfc6129) with + `Content-Disposition: attachment`. +- Do not modify the public contract or behaviour of existing 4.3.1 intake + endpoints, series-profile, episode-template, reference-document, or + reference-binding resources beyond additive wiring. +- The TEI file download must not require the audio or export-job pipeline. +- All Python quality gates must pass before each milestone is considered + complete: `make check-fmt`, `make typecheck`, `make lint`, `make test`, and + (when models or migrations change) `make check-migrations`. Markdown changes + must pass `make markdownlint` and `make nixie`. + +## Tolerances (exception triggers) + +Stop and escalate (record the situation in `Decision Log` and await direction) +when any of the following is breached. + +- Scope: if a single milestone requires net changes to more than 12 source + files or more than roughly 600 net lines, stop and re-segment. +- Interface: if delivering the slice forces a breaking change to an existing + public `/v1` route, the `LLMPort`, the `GenerationRunPort` sub-protocols, or + the `CanonicalUnitOfWork` surface, stop and escalate. +- Dependencies: if a new third-party runtime dependency (beyond what is already + in `pyproject.toml`, and beyond the Vidai Mock test-only binary) is required, + stop and escalate. +- Idempotency adapter: if the durable SQLAlchemy `IdempotencyStore` adapter that + `POST /v1/uploads` relies on turns out to be missing rather than present, + implementing it is an escalation-gated sub-task (it is a prerequisite the + slice assumes 4.3.1 already shipped), not silent extra work. +- Migrations: if `make check-migrations` reports drift that cannot be resolved + by an additive, reversible migration, stop and escalate. +- Iterations: if a focused test still fails after 3 genuine fix attempts, stop + and escalate with the failing transcript. +- Architecture: if satisfying a requirement appears to require a domain module + to import an infrastructure library, stop and escalate rather than relaxing + `make check-architecture`. +- Ambiguity: if an unanticipated design fork materially changes the externally + observable contract, stop and present options with trade-offs. + +## Risks + +- Risk: the launcher is implemented as a background task that closes over the + request's unit of work or session, causing use-after-free on a recycled + connection (silent cross-episode corruption or `IllegalStateChangeError`). + Severity: high. Likelihood: high (this is the natural-but-wrong + implementation). Mitigation: a hard constraint (above) plus a dedicated + Milestone 4 test that launches through real `asyncio.create_task` against + py-pglite and asserts the task writes through its own session while the + request unit of work is already closed. The "drive deterministically" + shortcut covers logic only; one test must exercise the detached-session path. +- Risk: a run is left stuck in `running` after a deploy/restart/crash, its + idempotency record stuck `InFlight`, so every retry with the same key returns + `409` forever and the client cannot resubmit. Severity: high. Likelihood: + high (happens on every deploy overlapping an in-flight run). Mitigation: + persist `started_at` (indexed) and add `lease_expires_at` and + `error_category` columns now (additive); make `pending → running` a + conditional update; and document the current manual-failure limitation. + Production-wide metrics and automated lease recovery are deferred to later + hardening and roadmap item 2.6.2. +- Risk: idempotency partial failure — the run row is created but task scheduling + fails, orphaning a `pending` run while the idempotency record is `failed` or + `complete`. Severity: high. Likelihood: medium. Mitigation: specify the + ordering — create the run row and schedule the task inside the same `work()`; + if scheduling fails, mark the run `failed` with an `error_message` (do not + merely fail the idempotency record) so the orphan is visible and the manual + recovery can identify it. +- Risk: the generated draft is not valid TEI-P5 and fails `tei-rapporteur` + validation on persistence, raising inside the detached task with no Falcon + handler. Severity: high. Likelihood: medium. Mitigation: the launcher's outer + try/except covers generation AND persistence AND validation, mapping any + exception to `run.failed` + `error_message` + terminal status with a distinct + `tei.invalid` event kind; a unit test feeds invalid TEI and asserts `failed` + rather than a leaked exception. +- Risk: snapshot tests are non-deterministic because TEI `xml:id` values or + timestamps vary per run. Severity: high. Likelihood: high (without + mitigation; the codebase reaches for `uuid.uuid4().hex` for fresh ids + elsewhere). Mitigation: a deterministic id scheme (`sp-1`, `p-1`, …) and the + existing injected `clock()` provider routed into the generator, persistence + service, and run store; freeze the clock and pin ids in snapshot tests. +- Risk: LLM draft spend is never ledgered because the cost recorder is not wired + into the launcher's composition root (a regression against the just-landed + 2.4.4 cost accounting and an ADR 009 observability requirement). Severity: + medium. Likelihood: high (the plan's earlier "if available" hedge would + silently skip it). Mitigation: wiring `CostRecorder` (over + `SqlAlchemyCostLedgerStore`) into the composition root and the launcher is an + explicit Milestone 4 deliverable, not a conditional. +- Risk: detached tasks are garbage-collected mid-run or run unbounded, + competing with request handling on the event loop. Severity: medium. + Likelihood: medium. Mitigation: hold strong task references in a registry, + bound concurrency with a semaphore, drain the registry in a shutdown hook + (marking drained runs `failed`), and document the single-worker assumption in + the new ADR. +- Risk: scope bleed into roadmap items 2.6.2, 2.6.3, or 4.4.1. + Severity: medium. Likelihood: medium. Mitigation: the + `Scope and roadmap relationship` section fixes the boundary. + +## Progress + +- [x] (completed, 2026-06-24) M0: Branch, plan baseline, and red + end-to-end scaffold. Implementation approval was given in this Lody session; + the branch already had the requested name, and branch tracking plus + PR/session metadata were aligned. The red behavioural scaffold now exists at + `tests/features/no_qa_generation_slice.feature` and + `tests/steps/test_no_qa_generation_slice.py`; the focused run produced + `7 xfailed in 0.48s` as expected. Deterministic gates passed: + `make check-fmt`, `make typecheck`, `make lint`, `make test`, + `make markdownlint`, and `make nixie`. CodeRabbit review completed with zero + findings. +- [x] (completed, 2026-06-24) M1: Domain model extensions (quality mode, QA + status, rationale). Red evidence captured the missing + `episodic.canonical.generation_quality` module. Green focused evidence: + `tests/test_generation_run_domain.py`, + `tests/test_generation_run_port_contract.py`, + `tests/test_generation_run_properties.py`, and + `tests/steps/test_generation_run_lifecycle_steps.py` passed with + `38 passed in 1.43s`. Full deterministic gates passed: `make check-fmt`, + `make typecheck`, `make lint`, `make test`, `make markdownlint`, and + `make nixie`. CodeRabbit review completed with zero findings after the + required rate-limit backoff and retry. +- [x] (completed, 2026-06-24) M2a: Durable generation-run and event + persistence. Initial orientation found that storage models are surfaced + through `episodic/canonical/storage/models.py` for Alembic metadata, and + `SqlAlchemyUnitOfWork` wires repositories during `__aenter__`, matching the + pattern M2a should extend. Red storage tests were added at + `tests/canonical_storage/test_generation_runs.py`; the focused run failed + during collection because `GenerationEventRecord` is not yet exported from + `episodic.canonical.storage`. Green focused evidence: the new SQLAlchemy + storage tests passed with `8 passed in 3.53s`, and the updated generation-run + port contract/property tests passed with `17 passed in 0.68s`. After the + checkpoint adapter split, the focused storage/port/lifecycle suite passed with + `29 passed in 4.29s`; full deterministic gates passed: `make check-fmt`, + `make typecheck`, `make lint`, `make test`, and `make check-migrations`. + CodeRabbit review completed with zero findings. +- [x] (completed, 2026-06-24) M2b: Episode TEI revisioning columns and + optimistic update. Red evidence captured the missing + `episodic.canonical.episode_errors` module after adding failing storage + tests. Green focused evidence: `tests/canonical_storage/test_episodes.py`, + `tests/canonical_storage/test_episode_tei_updates.py`, and + `tests/test_protocol_stubs.py` passed with `64 passed in 3.51s`. The new + migration applies through `20260624_000011`, and full deterministic gates + passed: `make check-fmt`, `make typecheck`, `make lint`, `make test`, + `make check-migrations`, `make markdownlint`, and `make nixie`. CodeRabbit + review completed with zero findings. +- [x] (completed, 2026-06-24) M3: Draft script generator port and TEI + persistence service. Orientation found that the installed `tei_rapporteur` + Python binding accepts `utterance` payloads emitted as ``, while + `...

...

` XML is not currently accepted by + `parse_xml`. Red focused evidence captured the missing + `episodic.generation.draft_script` and + `episodic.canonical.generation_persistence` modules. Green focused evidence: + `tests/test_draft_script_generation.py` and + `tests/test_generation_persistence.py` passed with `9 passed in 2.82s`. Full + deterministic gates passed: `make check-fmt`, `make typecheck`, `make lint`, + `make test` (`988 passed, 2 skipped, 7 xfailed`), `make markdownlint`, and + `make nixie`. CodeRabbit review completed with zero findings. +- [x] (completed, 2026-07-22) M4: In-process + launcher, lifecycle events, cost wiring, and observability. The launcher now + claims pending no-QA runs, records `run.started`, emits `draft.generated` + before TEI persistence, persists valid TEI, records cost ledger entries when + a recorder is configured, and marks terminal success/failure states with + stable error categories. Green focused evidence: + `tests/test_generation_run_launcher.py` and + `tests/test_env_runtime_wiring.py` passed with `15 passed in 14.24s`. Full + deterministic gates passed: `make check-fmt`, `make typecheck`, `make lint`, + `make test` (`994 passed, 2 skipped, 7 xfailed`), `make markdownlint`, and + `make nixie`. The 2026-07-22 milestone revalidation also passed + `make check-migrations`; `make test` reported + `1066 passed, 1 skipped, 7 xfailed`. CodeRabbit reviewed the complete branch + delta and reported zero findings. +- [x] (completed, 2026-07-22) M5: Generation-run REST + endpoints with idempotency (including Location/Retry-After replay). Red + evidence was HTTP 404 for both creation tests before route registration. + Green focused evidence covers create, replay, conflict, validation, polling, + event cursor pagination, domain/storage mapping, and existing intake + idempotency with `42 passed in 6.22s`. Full gates passed; `make test` reported + `1068 passed, 1 skipped, 7 xfailed`. After staging the complete milestone + delta, CodeRabbit reviewed the new resource and tests explicitly and reported + zero findings. +- [x] (completed, 2026-07-22) M6: Episode TEI retrieval + endpoint with content negotiation. Red evidence was HTTP 404 after a draft + had been persisted because the route was absent. Green evidence covers the + pre-draft 404, default JSON envelope, raw `application/tei+xml` attachment, + content disposition, ETag, and unsupported-media 406; the combined M5-M6 + endpoint suite reports `3 passed`. Full gates passed; `make test` reported + `1069 passed, 1 skipped, 7 xfailed`. A staged-delta CodeRabbit review + included the new resource and tests and reported zero findings. +- [x] (completed, 2026-07-22) M7: End-to-end behavioural slice with Vidai + Mock. The seven strict xfails are now live scenarios driving the Falcon API, + SQL persistence, in-process launcher, OpenAI-compatible adapter, and Vidai + Mock. Focused evidence is `7 passed in 5.36s` with `vidaimock 0.1.3`. Full + gates passed: `make test` reported `1076 passed, 1 skipped`; formatting, + typing, lint, migration, Markdown, and Mermaid gates also passed. CodeRabbit + reviewed the staged milestone, including both BDD modules, and reported zero + findings. +- [x] (completed, 2026-07-22) M8: Documentation, roadmap update, and final + gates. ADR 017, system/user/developer guidance, repository indexes, and the + completed roadmap checkbox are present. Markdown and Mermaid gates passed, + and final CodeRabbit review reported zero findings. +- [x] (completed, 2026-07-22) Post-implementation correctness review: + presenter-profile resolution was already present at PR head. The remaining + corrections now hydrate `upload:` sources from object storage, persist and + lock the ingestion-job-to-episode association during materialization, and + lock generation-run rows before mutable status updates. Focused lint and + regression evidence passes with `30 passed`. Full deterministic gates pass: + `make check-fmt`, `make lint`, `make typecheck`, `make check-migrations`, + `make test` (`1078 passed, 1 skipped`), `make markdownlint`, and + `make nixie`. CodeRabbit reviewed the complete branch delta and reported zero + findings. +- [x] (completed, 2026-07-22) Review follow-up: verified the live BDD fix plus + the row-locking, `error_category`, and idempotency work in the current tree; + the remaining review findings are stale. +- [x] (completed, 2026-07-23) Review follow-up at commit `42757a3`: verified + and fixed the still-valid runtime LLM wiring, `READY_FOR_GENERATION` + boundary, exact TEI validation, `Error`-suffixed episode exceptions, shared + SHA-256 helper, launcher semaphore cancellation safety, SQL paging validation + tests, and concurrent claim coverage. Skipped stale/already-fixed findings + for executable BDD scenarios, SQL status-row lock/error_category roundtrip, + in-memory `error_category` and duplicate exception wrappers, stable + materialization identity, roadmap/plan completion, and users/developer docs + coverage. No metric-driven split of `_runs_for_episode` and no new tracing + infrastructure or source-count policy were introduced from broad warnings + without a scoped design. Focused suites passed; deterministic gates passed on + 2026-07-23: `make check-fmt`, `make lint`, `make typecheck`, `make test` + (`1089 passed, 1 skipped`), `make check-migrations`, `make markdownlint`, + `make nixie`, and `mbake`. Final CodeRabbit review is not yet claimed. +- [x] (completed, 2026-07-23) Post-completion quality check: live PR + reconciliation found current CodeScene diagnostics in + `episodic/canonical/domain.py` (two complex `__post_init__` methods), + `episodic/api/runtime.py` (`_load_runtime_config` and module average), + `tests/test_generation_run_launcher.py` (duplication plus missing diagnostic + assertion messages), and `tests/test_generation_persistence.py` + (duplication). The implementation extracted behaviour-preserving + validation/config/test helpers and added assertion context. Focused tests + passed with `34 passed, 1 environment warning`; `cs check` now reports + `10.00` with no findings for all four files. The 2026-07-23 quality milestone + passed `make check-fmt`, `make lint`, `make typecheck`, `make test` + (`1089 passed, 1 skipped`), `make check-migrations`, `make markdownlint`, + `make nixie`, `mbake validate Makefile`, and `git diff --check`. + `coderabbit review --agent` ran against clean commit `e02f2c3`, returned + terminal `review_completed` with zero findings, and did not require a + rate-limit retry. + +- [x] (completed, 2026-07-31) Rebase and final validation: rebased onto + `origin/configure-df12-lints` at `1d49da451abd6f7d276e91693d945ce5e5b7945a`; + repaired the replay artefacts and DF12 lint-base findings without weakening + lint configuration. The exact final evidence is: `make check-fmt` passed for + 467 files; `make test` passed with 1091 passed, 1 skipped, and 54 snapshots; + `make typecheck`, `make lint`, `make check-migrations`, `make markdownlint` + (0 issues in 112 files), `make nixie`, `mbake validate Makefile`, and + `git diff --check` all passed. Before publication, + `origin/configure-df12-lints` advanced through 14 unrelated lint follow-up + commits to final target `5fd1f97d375f5703b90c5cb366f6a45a2d229e37`. Weave + predicted no overlap; the second 38-commit replay was conflict-free, the + final target is an ancestor, and `uv.lock` remains byte-identical. The + complete nine-gate suite passed again on the final base with the same 1091 + passed, 1 skipped, and 54 snapshots. `make typecheck` exited successfully + with three unused-ignore warnings confined to target-only + `tests/test_serializers.py`. Final review evidence: + `coderabbit review --agent --base origin/configure-df12-lints` completed in + 177.037s with zero findings and no rate-limit retry. Superseding final-base + review after the conflict-free replay onto final target + `5fd1f97d375f5703b90c5cb366f6a45a2d229e37`: + `coderabbit review --agent --base origin/configure-df12-lints` completed in + 98.255s with zero findings and no rate-limit retry. The earlier 177.037s + pre-advance review remains historical evidence. + +- [x] (completed, 2026-07-31) Latest rebase replay and structural validation: + the target was force-rewritten to `origin/configure-df12-lints` at + `de457b7ba3d6a13df8600410a0bc7aea25658ff8`. Obsolete base-owned commits + `f9a4afa` (lint adoption) and `4ed178a` (architecture/lock repair) were + carefully inspected and skipped because rewritten equivalents already exist + in the target. All 39 feature commits replayed. Weave auto-resolved three + entity pairs at very-high confidence; `sem diff` confirmed cohesive feature + extraction. Target `slots=True` in `_TextUploadRequest` survived, as did the + feature workflow artefact fallback. The target lockfile was restored, then + `uv lock` produced byte-identical output. The target is an ancestor and the + tree is marker-free. Final gates passed: `make check-fmt` passed 467 files; + `make test` passed 1,091, skipped 1, with 54 snapshots (26 warnings); + `make typecheck` passed with 3 unused-ignore warnings; and `make lint` and + `make check-migrations` passed. `make markdownlint` initially found the three + new `artefact` spellings; after correction to `artefact`, it passed. + `make nixie`, `mbake validate Makefile`, and `git diff --check` passed. + CodeRabbit was attempted twice with `--base origin/configure-df12-lints`; + both attempts stopped externally at `preparing_sandbox` with no findings, + diagnostic, or rate-limit indication, so no `vsleep` was warranted. A third + attempt from clean committed head `27477971b2153a5ba5f86f85d4ad991cea128cb6` + stopped externally at `connecting_to_review_service` after ~6.4s, with no + findings or rate-limit indication. An explicit lease-protected force push + replaced remote `74b8bd1` with `2747797`; PR #141 was retargeted to + `configure-df12-lints`; and `gh stack link --base main 220 141` created + remote stack #240. Verification showed PR #220 with base `main` and head + `de457b7`, and PR #141 with base `configure-df12-lints` and head `2747797`. + `gh stack view` has no local tracking by design after linking, so PR metadata + and the successful link output are the verification evidence. Push/PR/stack + actions are complete. + +- [x] (completed, 2026-08-11) Rebase and sequential validation: rebased onto + `origin/configure-df12-lints` at `db01ed84e917b5f8411ea7ab75f55c5505d3c972`. + Commit `ddc1c17` was safely skipped because target `698e2fa` is the rewritten + equivalent. Commit `f7adc07` was resolved by taking the target `uv.lock`, + running `uv lock`, and confirming that it matches the target. Forty-one + feature commits replayed; formatter-only commit + `e168e39560cc73ea2b8726ac0e09ddb272cd1e6b` follows. Sequential gate evidence: + `make check-fmt` passed; `make test` 1,099 passed/3 skipped/53 snapshots + passed; `make typecheck` 0 errors/3 warnings; `make lint` passed/Pylint + 10/10. Force-with-lease published remote `70e67b4` as current commit + `f3243528a12515d4fa4d421aa4e73da8d3c506d3`. PR #141 already had the requested + base `configure-df12-lints`; direct `gh pr edit` was rejected because it + belongs to a stack. PR #220 is `main <- configure-df12-lints`, and + `gh stack link --base main 220 141` confirmed that the two-PR stack is + already up to date. Remote head equals local. + +- [x] (completed, 2026-08-15) Documentation clarification: confirmed that + `episode_id` is included in the `202 Accepted` generation-run representation + and is also available from the polled run resource before the client requests + TEI. Updated `docs/users-guide.md` to state this procedure. No full gates + were run for this documentation-only change, as requested. + +- [x] (completed, 2026-08-20) Scoped PR #141 hardening follow-up across + the hardening commits: API/task spans, bounded in-process admission, explicit + overload terminalization, SQL claim outcome logs, representation-specific + ETags, cancellation fallback, trusted-principal ownership checks, typed + duplicate projection handling, bounded source hydration, and claim + linearization are implemented and validated. Production-wide metrics, + automated lease/recovery, retry metrics, generated durable lifecycle proofs, + and in-flight disposal proof remain deferred operational work. +- [x] (recorded, 2026-08-17) Launcher cancellation persistence now has one + owner: the cancelled task records it, while shutdown falls back only for a + coroutine cancelled before it starts. The no-QA BDD helper drains detached + work after each `POST` response because the session-scoped PGlite test server + supports one connection at a time. This keeps the REST response assertions + while launcher/API tests cover non-blocking scheduling; the hardening + follow-up was still in progress at that point; the scoped work was completed + on 2026-08-20 while roadmap item 4.3.2 remained complete. + +- [x] (recorded, 2026-08-17) Current review-remediation decisions: launcher + composition supplies configuration explicitly; episode materialization + validates attached sources before its short ingestion-job reservation lock, + retaining source-document duplicate recovery only after verification; + generation-run `episode_id` and `source_bundle_id` foreign keys remain + enforced, so test fixtures must provision the related episode and + ingestion-job rows; manual recovery SQL uses a conditional lock and commits + the failure event and status atomically; observability logs only safe, + allow-listed attributes; and the series-profile `Idempotency-Key` review item + is stale because that endpoint has no current idempotency contract. +- [x] (recorded, 2026-08-17) Current CodeRabbit follow-up decisions: retain + diagnostic projection IDs, add coverage for cancellation-write shielding and + admission capacity release, clean up test-only types, and retain distinct + `list_runs` and `list_events` contracts. CodeScene's test-stub duplication is + a structural metric only; it requires a narrowly scoped platform suppression, + not a repository refactor or a source-level lint suppression. +- Observation (2026-08-17): the current refactor extracts a private + episode-reuse helper, preserves source paging before the ingestion-job lock, + preserves committing the target episode association before projection, and + leaves duplicate-projection rollback in the public service. +- [x] (recorded, 2026-08-20) Validated the open CodeScene "Code Duplication" + diagnostic on `tests/test_generation_run_port_contract_support.py`: + `cs review` (CLI 1.0.33) confirms one clone group of seven + `NoopGenerationRunPort` methods (`get_run`, `list_runs`, `list_events`, + `get_checkpoint`, `respond_to_checkpoint`, `time_out_checkpoint`, + `cancel_checkpoint`), and `cs review` on + `episodic/canonical/generation_run_ports.py` confirms a second structural + pairing (`GenerationRunRepository.list_runs` with + `GenerationEventLog.list_events`). + `tests/test_generation_run_port_contract.py` scores 10.00 (no findings). Both + findings describe intentional structural similarity in protocol-completeness + scaffolding, not shared domain behaviour: (1) the stub's read bodies return + the empty value for that contract (`None` vs `()` vs zero), its run mutations + raise `RunNotFound`, and its checkpoint mutations raise `CheckpointNotFound`, + with per-method signatures, document param/return/Raises sections, and + distinct keyword-only parameters, mapping one-to-one onto the + `GenerationRunRepository`, `GenerationEventLog`, and + `GenerationCheckpointPort` sub-protocols that `GenerationRunPort` composes; + (2) the port declarations share only the `raise NotImplementedError` marker + body and a `limit`/`offset` paging signature while returning different domain + tuples with different filters, and the recorded decision retains distinct + `list_runs`/`list_events` contracts. The duplication remedy CodeScene itself + prescribes — "extract a shared representation" — is already realized by the + protocol definitions in `generation_run_ports.py`; the stub is contract + derived state, so extracting generic empty-result, pass-through, or not-found + helper methods (or merging run, event-log, and checkpoint operations) would + hide protocol conformance and weaken the direct typed assignment + `noop_port: GenerationRunPort = NoopGenerationRunPort()` in + `TestCompositeProtocol.test_noop_composite_protocol_stub_typechecks` without + removing a single maintenance risk. Plain "Code Duplication" is a core + CodeScene factor — not a configurable `code-health-rules.json` rule and not + suppressible by `@codescene` directives — so the disposition is + `no code changes required; suppress diagnostic` on the CodeScene platform, + per the 2026-08-17 decision. ExecPlan-only change; runtime/test gates + unaffected. + +## Surprises & discoveries + +- Observation: the existing generation-orchestration graph + (`episodic/orchestration/langgraph.py`, + `build_generation_orchestration_graph`) takes an existing `script_tei_xml` + and *enriches* it (default action `GENERATE_SHOW_NOTES`); it does not + generate the initial script from sources, and + `GenerationOrchestrationRequest` requires a non-empty `script_tei_xml`. + Evidence: `episodic/orchestration/_dto.py` (`GenerationOrchestrationRequest` + around line 177; `enabled_action_kinds=(ActionKind.GENERATE_SHOW_NOTES,)` + around line 221). Impact: 4.3.2 introduces a separate minimal + `DraftScriptGenerator`; the orchestration graph is not on the critical path + for this slice. +- Observation: there is **no** service that materializes a `CanonicalEpisode` + from intake-stage (4.3.1) sources. `_create_canonical_episode` + (`episodic/canonical/services.py` around line 91) needs a `TeiHeader` parsed + from caller-supplied TEI, and the 4.3.1 intake path + (`episodic/canonical/source_intake_service.py`) creates only ingestion jobs + and sources — never an episode. The draft TEI is *produced by generation*, so + there is an ordering problem. Evidence: greps over `episodes.add`, + `source_intake_service.py`. Impact: Milestone 3 adds an explicit "materialize + episode from ingestion job" step that creates the episode with a minimal + placeholder TEI header before generation, then the launcher updates it with + the generated script. This is a new step, not pure reuse; the construction + helpers may be reused but the path is new. +- Observation: `GenerationRun.source_bundle_id` exists with no producer; the + ingestion job whose sources materialize the episode is the natural bundle. + Evidence: `episodic/canonical/domain.py` around line 139; no `SourceBundle` + aggregate exists. Impact: map `source_bundle_id` to the ingestion job id. +- Observation: `GenerationRun` and `CanonicalEpisode` are `frozen=True` with no + field defaults; adding fields breaks ~5 `GenerationRun(` and ~18 + `CanonicalEpisode(` construction sites and the full-repr snapshot + `tests/__snapshots__/test_generation_run_domain.ambr`. Evidence: `domain.py` + around lines 133 and 308. Impact: give the new fields safe defaults where + possible and regenerate the affected snapshot as an explicit M1 step. +- Observation: Vidai Mock subprocess fixtures already exist in the repo + (`tests/steps/test_guest_bios_steps.py` has `_start_vidaimock_process`, + `_await_port_ready`, `_terminate_process_gracefully`, and a `shutil.which` + skip); the `model="vidai-mock"` name convention is established. Evidence: + `tests/steps/test_guest_bios_steps.py`, + `tests/steps/generation_orchestration_vidaimock.py`, + `tests/_guest_bios_helpers.py`. Impact: Milestone 7 reuses the already + extracted `generation_orchestration_vidaimock.py` process helper and narrows + its context requirement to a structural protocol rather than writing a third + process manager. +- Observation: cancelling the launcher during replay-scenario teardown can + interrupt PGlite database work and make later scenarios lose the shared test + server. Impact: the M7 context drains scheduled generation before launcher + shutdown, matching the application's graceful-shutdown contract and keeping + the seven-scenario suite deterministic. +- Observation: malformed draft JSON is classified as `response.format` and + does not emit `tei.invalid`; that event is reserved for failures while + constructing or validating TEI. Impact: the malformed-completion scenario + returns structurally valid draft JSON containing an XML-invalid speaker, + which exercises the intended `tei.invalid` branch. +- Observation: the M7 documentation reconciliation exposed that launcher + claims still supplied `presenter_profiles=()` and the original BDD presenter + step created no documents or bindings. Impact: the launcher now resolves + series-level bindings for the materialized episode and projects host and + guest revisions into `DraftPresenterProfile`; the BDD setup creates both + bound revisions, and focused launcher plus slice evidence is `12 passed`. +- Observation: the post-turn lint hook found the final no-QA BDD step module at + 408 lines, above the repository's 400-line module limit. Impact: scenario + resource cleanup moved into `NoQaGenerationSliceContext.tear_down`, where the + launcher, adapter, and Vidai Mock process lifecycle now remain cohesive. The + step module is 400 lines; focused scenarios and the full formatting, lint, + and type-check gates pass. +- Observation: on 2026-06-24 the local branch was already named + `4-3-2-no-qa-generation-runs-and-tei-p5-retrieval` and the matching remote + branch existed at `origin`, but the worktree did not have an upstream branch + configured. The original Concrete steps section still referenced the planning + worktree path. Evidence: `git branch --show-current`, + `git ls-remote --heads origin 4-3-2-no-qa-generation-runs-and-tei-p5-retrieval`, + and `git branch -vv`. Impact: Milestone 0 sets tracking with + `git branch --set-upstream-to` instead of renaming an already correctly named + branch, and the Concrete steps path is updated to this implementation + worktree. +- Observation: the first M1 `coderabbit review --agent` request hit a + recoverable CodeRabbit rate limit. Impact: the implementation followed the + required backoff command, `vsleep $(shuf -i 45-90 -n 1)m`, then retried the + review successfully before starting M2a. +- Observation: adding SQLAlchemy persistence plus guarded run claiming pushed + the in-memory generation-run adapter over the 400-line project limit. Impact: + the checkpoint methods were extracted into + `episodic/canonical/adapters/generation_checkpoints.py` as a mixin, keeping + the run/event adapter focused while preserving the existing public in-memory + test adapter surface. +- Observation: SQL event sequence allocation cannot rely on `max(seq) + 1` + alone; concurrent appenders could read the same maximum and one would fail + the unique `(generation_run_id, seq)` constraint. Impact: the SQLAlchemy + store locks the owning generation-run row before allocating the next event + sequence, serializing appenders per run without introducing a separate + sequence table. +- Observation: adding optimistic episode TEI updates would have pushed the + existing storage repository and episode test module beyond the project + 400-line guideline. Impact: the SQLAlchemy episode repository now lives in + `episodic/canonical/storage/episode_repository.py`, and the TEI update tests + live in `tests/canonical_storage/test_episode_tei_updates.py`, keeping the + changed modules focused. +- Observation: `tei_rapporteur.parse_xml` rejects `` blocks in the current + binding (`unknown variant sp`) but accepts body `utterance` payloads, which + emit as TEI `` elements. Impact: Milestone 3 uses + `tei_rapporteur.from_dict` with `utterance`/`paragraph` blocks for the + minimal draft script, preserving TEI validation without hand-written XML. +- Observation: `ingestion_jobs.target_episode_id` has a foreign key to + `episodes.id`, so an intake job cannot point at an episode id that the + materialization step has not created yet. Impact: M3 materialization treats a + `NULL` target episode as the normal pre-generation state and allocates the + episode id while projecting attached intake sources into canonical source + documents. +- Observation: M4 source documents preserve rich source text in + `SourceDocument.metadata["content"]` when it is available, while older or URI + only sources may only have `source_uri`. Impact: the launcher builds + `DraftScriptSource.content` from non-blank metadata content first, falling + back to `source_uri` so legacy rows still produce deterministic generator + requests. +- Observation: upload-backed source documents intentionally retain an + `upload:` provenance URI, while their bytes live only in the + configured object store. Impact: launcher source projection now reads those + bytes, decodes UTF-8 text (including a byte-order mark), normalizes line + endings, and rejects missing, undecodable, or empty upload content before an + LLM request can be made. +- Observation: a null `IngestionJob.target_episode_id` caused each + materialization retry to allocate another episode and duplicate its source + projections. Impact: materialization now locks the ingestion-job row, creates + the episode, and persists the association in one transaction; subsequent + calls return the original episode. +- Observation: separate SQLAlchemy units of work could both read a running + generation run before either committed a terminal update. Impact: + `update_run_status` now acquires a row lock before checking terminal state, + so a later writer observes the committed terminal status and raises the + existing `RunAlreadyTerminal` exception. +- Observation: an intermediate 2026-07-22 current-state audit found no + generation-run or episode-TEI REST route under `episodic/api`; the only API + integration added by M4 is launcher dependency/runtime wiring. The seven + scenarios in `tests/steps/test_no_qa_generation_slice.py` were still marked + `xfail(strict=True)` and their creation steps deliberately fail with + `4.3.2 no-QA source-to-script slice is not implemented yet`. Impact: this + prevented premature completion, and Milestones 5-7 subsequently added the + missing routes and activated all seven scenarios. +- Observation: after rebasing onto `origin/main`, updated SQLAlchemy typing + required count queries in the storage tests to use `scalar_one()`. The local + `act` artefact server also lacks the request schema used by the current + `actions/upload-artifact`; the workflow helper now skips only the explicit + `unknown field "mime_type"` incompatibility and preserves all other workflow + failures. Impact: the complete post-rebase test suite passes without masking + unrelated workflow errors. +- Observation: M5 polling exposed that `error_category` was written to the SQL + record by launcher failure updates but was absent from `GenerationRun`, the + SQL record mapper, and the in-memory adapter replacement. Impact: M5 restores + the field across the domain and both adapters so failed-run polling can + satisfy the behavioural contract instead of silently dropping the stable + failure category. + +- Observation: rebasing onto `origin/configure-df12-lints` required one + explicit conflict in `episodic/canonical/storage/repositories.py`. The + post-rebase sem/Weave audit also found replay artefacts without conflict + markers: duplicated `generation_runs` module prose/type aliases, a stale + `tests.workflow_test_utils` import after the target moved helpers, and an + extracted episode repository call to target-removed `_add_record`. Impact: + these were repaired using target-branch conventions while preserving the + extracted `SqlAlchemyEpisodeRepository` with TEI updates and the target + branch's unrelated repositories. +- Observation: before publication, the target advanced through 14 unrelated + lint follow-up commits to final target + `5fd1f97d375f5703b90c5cb366f6a45a2d229e37`. Weave predicted no overlap, and + the second 38-commit replay was conflict-free; the final target is an + ancestor and `uv.lock` remains byte-identical. Impact: the initial explicit + `1d49da4` conflict history remains recorded, while publication used the + conflict-free replay. +- Observation: the DF12 lint base exposed assertion-message, alias, + suppression-rationale, wrapper, future-annotations, and module-size findings. + Impact: the fixes preserved behaviour and did not weaken lint configuration. +- Observation: the force-rewritten target at + `de457b7ba3d6a13df8600410a0bc7aea25658ff8` already contains rewritten + equivalents of base-owned commits `f9a4afa` (lint adoption) and `4ed178a` + (architecture/lock repair). Impact: those commits were carefully inspected + and skipped, while all 39 feature commits were replayed. +- Observation: Weave auto-resolved three entity pairs at very-high confidence, + and `sem diff` confirmed cohesive feature extraction. Target `slots=True` in + `_TextUploadRequest` and the feature workflow artefact fallback survived; the + restored target lockfile remained byte-identical after `uv lock`. The target + is an ancestor and the tree is marker-free. Impact: structural validation and + all final deterministic gates passed. `make markdownlint` initially found the + three new `artefact` spellings; correcting them to `artefact` made the gate + pass. +- Observation: two CodeRabbit attempts with + `--base origin/configure-df12-lints` both stopped externally at + `preparing_sandbox`, with no findings, diagnostic, or rate-limit indication. + Impact: no `vsleep` was warranted. +- Observation: a third CodeRabbit attempt from clean committed head + `27477971b2153a5ba5f86f85d4ad991cea128cb6` stopped externally at + `connecting_to_review_service` after ~6.4s, with no findings or rate-limit + indication. Impact: CodeRabbit did not produce a completed review. +- Observation: an explicit lease-protected force push replaced remote + `74b8bd1` with `2747797`; PR #141 was retargeted to `configure-df12-lints`; + and `gh stack link --base main 220 141` created remote stack #240. + Verification showed PR #220 with base `main` and head `de457b7`, and PR #141 + with base `configure-df12-lints` and head `2747797`. `gh stack view` has no + local tracking by design after linking, so PR metadata and successful link + output are verification. Impact: push/PR/stack actions are complete. +- Observation: the 2026-08-11 replay encountered rewritten target history rather + than requiring the earlier base-owned work to be replayed again. Evidence: + `ddc1c17` matched target `698e2fa`, and the `f7adc07` lockfile resolution + matched the target after `uv lock`. Impact: the rebase retained the 41 + feature commits and the formatter-only commit without introducing dependency + churn. + +## Decision log + +- Decision: execution model is in-process async behind a `GenerationRunLauncher` + port; Celery dispatch deferred; the launcher is documented as a degenerate + `TaskResumePort`. Rationale: ADR 007 records that LangGraph-to-Celery + dispatch has not landed and that `TaskResumePort` is the future seam; the + slice only needs the 202/poll/download contract, which durable run/event + records satisfy without a worker. Documenting the convergence keeps + 2.6.2/4.4.1 collapsing onto the checkpoint model instead of replacing the + launcher. Confirmed with requester on 2026-06-15. Date/Author: 2026-06-15, + planning agent. +- Decision: the episode is materialized from the ingestion job's attached + sources (with a placeholder TEI header created before generation) rather than + via a new `POST /v1/episodes`. Rationale: keeps the client flow upload → + ingest → generate; aligns with `GenerationRun.source_bundle_id`; avoids + adding episode-CRUD surface owned by later roadmap work. Confirmed with + requester on 2026-06-15. Date/Author: 2026-06-15, planning agent. +- Decision: 4.3.2 ships a minimal real single-pass `DraftScriptGenerator` + behind a port; the full QA-bypass draft graph is left to 4.4.1. Rationale: + roadmap 4.4.1 (requires 4.3.2) owns the draft generation graph with explicit + QA bypass; a port keeps the engine swappable. Confirmed with requester on + 2026-06-15. Date/Author: 2026-06-15, planning agent. +- Decision: 4.3.2 implements durable SQLAlchemy persistence for generation runs + and events plus the generation-run REST endpoints it needs, as a subset of + 2.6.2/2.6.3. Rationale: cross-process REST polling is impossible with the + in-memory adapter (ADR 007 calls it tests/local only). Date/Author: + 2026-06-15, planning agent. +- Decision: an unsupported-but-recognized `quality_mode` value returns + `422 Unprocessable Entity`; a malformed body or missing/blank + `skip_qa_rationale` returns `400 Bad Request`. Rationale: a shape-valid body + carrying `quality_mode="qa_gated"` is semantically unsupported in this slice; + 422 lets `qa_gated` later become a `202` without flipping a wire status from + `400` (which clients may treat as "never valid") — `422 → 202` is + forward-compatible, `400 → 202` is not. Aligns with the TUI design error + table. Date/Author: 2026-06-15, planning agent. +- Decision: `GET /v1/episodes/{id}/tei` before any draft exists returns + `404 Not Found` (recorded as a contract, not a TODO, so it does not silently + flip to `200` empty later, which would be breaking). Date/Author: 2026-06-15, + planning agent. +- Decision: optimistic concurrency (`expected_revision`) is out of scope for + this slice's read-only `GET /tei` and append-only run creation, but the + episode `tei_revision` integer returned as the envelope `version` is the + exact value a later `expected_revision` (on `PUT /tei` / `PATCH /script`) + will compare against, so the contract composes later without renaming. + Date/Author: 2026-06-15, planning agent. +- Decision: the no-QA slice keeps idempotency in Python (reusing + `IdempotencyStore` and `run_idempotent`); no Rust extension or formal proof + is introduced. ADR 009 anticipates Kani/Verus only if the idempotency state + machine moves into Rust; that is a future task. Date/Author: 2026-06-15, + planning agent. +- Decision: treat the 2026-06-24 user request to proceed as ExecPlan approval + and move this document from draft to in-progress. Rationale: the plan + approval gate is satisfied by the explicit implementation request, and the + requester also required the plan to remain current during delivery. + Date/Author: 2026-06-24, implementation agent. +- Decision: preserve the existing `update_run_status(...) -> GenerationRun` + port contract for ordinary lifecycle updates and add + `claim_run_for_execution(...) -> GenerationRun | None` for the guarded + `pending -> running` transition. Rationale: this gives the launcher an + explicit compare-and-set operation that reports whether it won without + weakening the existing status-update API. `None` means another worker claimed + the run first; missing or terminal runs still raise the existing domain + errors. Date/Author: 2026-06-24, implementation agent. +- Decision: allocate SQL generation-event sequences while holding a row-level + lock on the parent `generation_runs` record. Rationale: the append-only log + requires gap-free, monotonic sequence numbers per run; locking the parent run + row is the smallest durable serialization point already present in the schema + and keeps the event table append-only. Date/Author: 2026-06-24, + implementation agent. +- Decision: episode TEI updates use an `EpisodeTeiUpdate` request object + instead of expanding `EpisodeRepository.update` with several scalar keyword + parameters. Rationale: the update operation must carry TEI XML, QA status, + generation-run provenance, expected revision, and an optional timestamp as + one coherent command; grouping them keeps the port stable and avoids a long, + error-prone parameter list. Date/Author: 2026-06-24, implementation agent. +- Decision: lifecycle status mutations use the frozen, slotted + `GenerationRunStatusUpdate` parameter object rather than five scalar update + arguments. Rationale: status, node, completion time, and optional failure + details form one coherent command; the object removes the excess-arguments + warning while preserving the repository contract and domain exceptions. + Date/Author: 2026-06-26, implementation agent. +- Decision: suppress the module-level CodeScene “Overall Code Complexity” + diagnostic for `episodic/canonical/adapters/generation_runs.py` rather than + split `_runs_for_episode`. Rationale: configured Ruff C901 at complexity 8 + passes; a diagnostic threshold of 4 reports only `_runs_for_episode` at 6. + That method cohesively owns indexed retrieval, optional status filtering, and + pagination, while `_require_mutable_run` already centralizes mutable-run + validation. Splitting the retrieval solely to lower a file-average metric + would add metric-driven churn without a concrete design improvement. + Date/Author: 2026-07-22, implementation agent. +- Decision: before materialization, the `ingestion_job_id` in + `POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs` names the ready + ingestion job and is reused as the newly materialized episode id; it is also + stored as `GenerationRun.source_bundle_id`. Rationale: the request contract + has no separate source-bundle field, and intake jobs cannot reference a + not-yet-created episode because of the foreign key. Using one stable + identifier preserves the documented upload → ingest → generate flow without + adding an episode creation endpoint. Date/Author: 2026-07-22, implementation + agent. + +- Decision: serialize episode materialization on the ingestion-job row and + persist `target_episode_id` after the episode row is flushed, all within the + caller's transaction. Rationale: the existing foreign key requires the + episode to exist before association, while the row lock ensures concurrent + and repeated requests converge without orphan episodes or duplicate source + projections. Date/Author: 2026-07-22, implementation agent. +- Decision: resolve `upload:` source provenance through `ObjectStorePort` at + launcher claim time and treat uploads as normalized UTF-8 text. Rationale: + attachment metadata does not contain normal upload bytes, and passing the + provenance URI to the LLM loses the user's source material; resolving at the + launcher keeps object-storage access out of the draft-generator port. + Date/Author: 2026-07-22, implementation agent. +- Decision: use the generation-run row lock already employed for event + sequencing to serialize status mutations. Rationale: after a competing + terminal commit, the waiting transaction reloads terminal state through the + existing mutable-run guard and preserves the established domain exception + without introducing a second transition API. Date/Author: 2026-07-22, + implementation agent. + +- Decision: rebase onto `origin/configure-df12-lints` at + `1d49da451abd6f7d276e91693d945ce5e5b7945a`, resolving the single explicit + repository conflict by preserving the extracted `SqlAlchemyEpisodeRepository` + with TEI updates and retaining the target branch's unrelated repositories. + Rationale: preserve both branches' intended changes while adopting the target + branch's current structure. Date/Author: 2026-07-31, implementation agent. +- Decision: use the target branch's `uv.lock` as the rebase baseline and run + `uv lock`; retain the result because it remained byte-identical to the target + lockfile. Rationale: avoid unrelated dependency churn. Date/Author: + 2026-07-31, implementation agent. +- Decision: repair all sem/Weave replay artefacts and DF12 lint-base findings + using target conventions, without weakening lint configuration. Rationale: + the artefacts were structurally valid enough to evade conflict markers but + were semantically stale; the lint findings required behaviour-preserving + source corrections. Date/Author: 2026-07-31, implementation agent. +- Decision: use final target + `5fd1f97d375f5703b90c5cb366f6a45a2d229e37` after the target advanced through + 14 unrelated lint follow-up commits. Rationale: Weave predicted no overlap, + the second 38-commit replay was conflict-free, the final target is an + ancestor, and `uv.lock` remains byte-identical; the initial `1d49da4` + conflict history remains the record of the first rebase. Date/Author: + 2026-07-31, implementation agent. +- Decision: accept the final-base validation as complete. Rationale: the + complete nine-gate suite passed again with the same 1091 passed, 1 skipped, + and 54 snapshots; `make typecheck` exited successfully, with three + unused-ignore warnings confined to target-only `tests/test_serializers.py`. + Date/Author: 2026-07-31, implementation agent. +- Decision: replay all 39 feature commits onto the force-rewritten target + `origin/configure-df12-lints` at `de457b7ba3d6a13df8600410a0bc7aea25658ff8`, + skipping base-owned commits `f9a4afa` (lint adoption) and `4ed178a` + (architecture/lock repair) because rewritten equivalents already exist in the + target. Rationale: preserve the complete feature extraction while avoiding + obsolete base history. Date/ Author: 2026-07-31, implementation agent. +- Decision: restore the target lockfile before running `uv lock` and retain the + result because the output is byte-identical. Rationale: avoid unrelated + dependency churn. Date/Author: 2026-07-31, implementation agent. +- Decision: treat the Weave, `sem diff`, surviving-target-feature, + ancestor, and marker-free-tree checks as structural validation, and accept + the final deterministic gates as complete. Rationale: `make check-fmt`, + `make test`, `make typecheck`, `make lint`, `make check-migrations`, the + corrected `make markdownlint`, `make nixie`, `mbake validate Makefile`, and + `git diff --check` all passed. Date/Author: 2026-07-31, implementation agent. +- Decision: do not apply a `vsleep` after the CodeRabbit attempts stopped + externally. Rationale: the first two `preparing_sandbox` stops and the third + `connecting_to_review_service` stop returned no rate-limit indication; the + third attempt also returned no findings after ~6.4s. Date/Author: 2026-07-31, + implementation agent. +- Decision: publish the clean committed head with an explicit lease-protected + force push, retarget PR #141 to `configure-df12-lints`, and link PRs #220 and + #141 with `gh stack link --base main 220 141`. Rationale: the push replaced + remote `74b8bd1` with `2747797`, the link created remote stack #240, and PR + metadata verifies #220 as base `main`/head `de457b7` and #141 as base + `configure-df12-lints`/head `2747797`. `gh stack view` has no local tracking + by design after linking, so the successful link output and PR metadata are + the verification evidence. Push/PR/stack actions are complete. Date/Author: + 2026-07-31, implementation agent. +- Decision: for the 2026-08-11 rebase, use target + `origin/configure-df12-lints` at `db01ed84e917b5f8411ea7ab75f55c5505d3c972`; + safely skip `ddc1c17` because target `698e2fa` is its rewritten equivalent; + and resolve `f7adc07` with the target `uv.lock` followed by `uv lock`, + retaining the matching result. Rationale: preserve target-owned history and + avoid dependency churn while replaying the 41 feature commits. Retain + formatter-only commit `e168e39560cc73ea2b8726ac0e09ddb272cd1e6b`. The + sequential gate evidence is `make check-fmt` passed, `make test` 1,099 + passed/3 skipped/53 snapshots passed, `make typecheck` 0 errors/3 warnings, + and `make lint` passed/Pylint 10/10. Force-with-lease published remote + `70e67b4` as current commit `f3243528a12515d4fa4d421aa4e73da8d3c506d3`. PR + #141 already had the requested base `configure-df12-lints`; direct + `gh pr edit` was rejected because it belongs to a stack. PR #220 is + `main <- configure-df12-lints`, and `gh stack link --base main 220 141` + confirmed that the two-PR stack is already up to date. Remote head equals + local. Date/Author: 2026-08-11, implementation agent. +- Decision: document `episode_id` as available in the `202 Accepted` + generation-run representation and in the polled run resource, and instruct + clients to read it before requesting TEI. Rationale: + `serialize_generation_run` includes `episode_id`, and the run-creation + response uses that representation, so clients can follow the returned run + resource without inferring the episode identifier. Confirmed on 2026-08-15. + Date/Author: 2026-08-15, documentation agent. +- Finding: commit `164528c` makes ETags representation-specific and returns + `304 Not Modified` for matching or wildcard `If-None-Match` validators on + both JSON and TEI representations. Date/Author: 2026-08-17, implementation + agent. +- Finding: commit `9921083` moves the concurrent-claim test's successful-count + assertion before selecting the claimed result, so a mutual-exclusion failure + reports the observed count. Date/Author: 2026-08-17, implementation agent. +- Finding: commit `72a5ef0` completes the public episode-TEI API documentation, + including representation selection, conditional requests, and serializer + fields. Date/Author: 2026-08-17, implementation agent. +- Decision: commit `5c7b8e4` adds a post-drain cancellation fallback for tasks + cancelled before `_run_task` begins; terminal guards preserve one failure + event when cancellation was already handled in the task. Date/Author: + 2026-08-17, implementation agent. + +## Context and orientation + +This is a Python project. The HTTP service is built on Falcon's ASGI +application; persistence uses async SQLAlchemy with PostgreSQL (tested against +py-pglite); orchestration uses LangGraph with a Celery worker boundary. Domain +logic is kept behind ports and adapters (hexagonal architecture). + +Read these documents before starting; they are the source of truth for this +slice: + +- [ADR 009: Source-to-script REST vertical slice](../adr/adr-009-source-to-script-rest-vertical-slice.md) + — the API contract, idempotency rules, observability, partial-failure + recovery, and testing strategy. +- [ADR 007: Durable generation checkpoints](../adr/adr-007-durable-generation-checkpoints.md) + — orchestration boundary and why Celery dispatch is deferred. +- [ADR 006: Chrono spoken-text semantics](../adr/adr-006-chrono-spoken-text-semantics.md) + — which TEI elements count as spoken script (``, ``, `

`, ``, + ``, ``; ``/``/`` excluded). This fixes the + shape the generator must emit and Chrono will later consume. +- [ADR 015: Generation-run port split](../adr/adr-015-generation-run-port-split.md) + and + [ADR 015: Upload and idempotency ports](../adr/adr-015-upload-and-idempotency-ports.md). +- [ADR 014: Hexagonal architecture enforcement](../adr/adr-014-hexagonal-architecture-enforcement.md). +- [Episodic podcast generation system design](../episodic-podcast-generation-system-design.md), + section "Source-to-script vertical slice". +- [Episodic TUI API design](../episodic-tui-api-design.md), sections + "Episodes and TEI" and "Generation runs", and the standard error table. +- [Async SQLAlchemy with PostgreSQL and Falcon](../async-sqlalchemy-with-pg-and-falcon.md). +- [Testing async Falcon endpoints](../testing-async-falcon-endpoints.md). +- [Testing SQLAlchemy with pytest and py-pglite](../testing-sqlalchemy-with-pytest-and-py-pglite.md). +- Read + [Agentic systems with LangGraph and Celery](../agentic-systems-with-langgraph-and-celery.md). + +Relevant skills to load when implementing: `hexagonal-architecture`, +`python-router` (then `python-types-and-apis`, `python-data-shapes`, +`python-errors-and-logging`, `python-testing`, `python-verification`), +`vidai-mock`, and `leta` for code navigation. + +### Key existing code (full repository-relative paths) + +Domain and ports: + +- `episodic/canonical/domain.py` — `GenerationRun` (around line 133), + `GenerationRunStatus` (around line 93), `GenerationEvent` (around line 161), + `CanonicalEpisode` (around line 308), `TeiHeader` (around line 297), + `EpisodeStatus`, `ApprovalState`. +- `episodic/canonical/generation_run_ports.py` — + `GenerationRunRepository`, `GenerationEventLog`, `GenerationCheckpointPort`, + composite `GenerationRunPort`, `EventSeq` newtype (`event_seq` rejects `< 1`; + `list_events(after_seq, limit)` is half-open `(after_seq, …]`, ascending by + `seq`, `limit` a hard cap). +- `episodic/canonical/adapters/generation_runs.py` — + `InMemoryGenerationRunStore` (reference adapter; tests/local only). +- `episodic/canonical/idempotency.py`, `episodic/canonical/upload_protocols.py` + (`IdempotencyStore`), `episodic/canonical/idempotency_service.py` + (`json_body_hash` → `canonical_json_bytes`; SHA-256 over canonical JSON). +- `episodic/canonical/entity_protocols.py` — `EpisodeRepository` + (`add`/`get`/`list_by_ids`; no update yet). Note `SeriesProfileRepository` and + `EpisodeTemplateRepository` expose `update(entity)` mapping a field list via + `_update_entity_fields` — match that convention. +- `episodic/canonical/pagination.py` — `Pagination`. +- `episodic/cost/recorder.py` — `CostRecorder`; + `episodic/cost/storage/adapters.py` — `SqlAlchemyCostLedgerStore`. + +Persistence: + +- `episodic/canonical/storage/entity_models.py` — `EpisodeRecord` + (around line 76; `tei_xml: Text` with sibling `tei_xml_zstd: BYTEA` + compression; `content_hash` columns are `String(128)`), `TeiHeaderRecord` + (around line 26, same compression pattern). +- `episodic/canonical/storage/entity_mappers.py` — `_episode_from_record`, + `_episode_to_record`; `encode_text_for_storage`/`decode_text_from_storage` + compression helpers. +- `episodic/canonical/storage/repositories.py` — `SqlAlchemyEpisodeRepository` + (around line 176); `SeriesProfileRepository.update` (around line 137) is the + `update`-convention exemplar. +- `episodic/canonical/storage/repository_base.py` — `_update_entity_fields`, + `_update_where`. +- `episodic/canonical/storage/uow.py` — `SqlAlchemyUnitOfWork` (exposes + `episodes`, `idempotency = SqlAlchemyIdempotencyStore(...)`, + `workflow_checkpoints`; a `clock()` provider is threaded here). +- `episodic/canonical/storage/workflow_checkpoints.py` — + `SqlAlchemyWorkflowCheckpointStore.save_or_reuse` (the DB-unique-constraint + durable-idempotent primitive the launcher's convergence target uses). +- `alembic/` — migration environment and versioned migrations. + +TEI and generation: + +- `episodic/canonical/tei.py` — `parse_tei_header`. +- `episodic/generation/tei_payload.py` — `body_blocks_payload`, + `build_text_inline`, payload validators. +- `episodic/generation/show_notes.py` — `enrich_tei_with_show_notes` (the + `tei-rapporteur` parse → mutate body blocks → emit pattern to imitate). +- `episodic/generation/chapter_marker_tei.py` — shows the existing + `uuid.uuid4().hex` id habit to AVOID for deterministic output. +- `episodic/canonical/services.py` — `ingest_sources`, + `_create_canonical_episode` (construction helpers). +- `episodic/canonical/source_intake_service.py` — the 4.3.1 intake path and the + `clock()`/`providers` injection convention. + +Orchestration and LLM: + +- `episodic/llm/ports.py` — `LLMPort`, `LLMRequest`, `LLMResponse`, + `LLMUsage`, `LLMTokenBudget`, `ProviderCallUsage`, and the error hierarchy + (`LLMTokenBudgetExceededError`, `LLMProviderResponseError`, + `LLMTransientProviderError`). +- `episodic/llm/openai_api/adapter.py` — `OpenAICompatibleLLMAdapter` (retry, + timeout, token budget). + +HTTP API: + +- `episodic/api/app.py` — `create_app` and the `_register_*` route functions. +- `episodic/api/runtime.py` — the composition root (`composition_root` group); + `uow_factory` returns a fresh `SqlAlchemyUnitOfWork` per call; shutdown hooks + call `engine.dispose`. +- `episodic/api/resources/base.py` — async resource base classes. +- `episodic/api/resources/source_intake.py` — representative idempotent and + paginated resources. +- `episodic/api/serializers.py`, `episodic/api/errors.py` + (`validation_error`, `RevisionConflictError`, `http_error`), + `episodic/api/helpers.py` (`parse_uuid`, `require_payload_dict`, + `require_str`), `episodic/api/dependencies.py` (`ApiDependencies`), + `episodic/api/types.py` (`UowFactory = Callable[[], CanonicalUnitOfWork]`). +- `episodic/api/source_idempotency.py` — `run_idempotent`, + `IdempotencyContext`, `IdempotentResponse` (currently stores only `status` + + `media`), `apply_response`, `_encode_outcome`/`_decode_outcome`. +- `episodic/api/source_intake_support.py` — `json_body_hash` (note: this is the + correct module for the hash helper). + +Tests and fixtures: + +- `tests/fixtures/database.py` (`session_factory`, `migrated_engine`, + `pglite_session`), `tests/fixtures/api.py` (`build_api_dependencies`, + `canonical_api_async_client`), `tests/fixtures/llm.py`, `tests/conftest.py` + (`pytest_plugins`). +- `tests/show_notes_support.py` — `FakeLLMPort`, `valid_llm_response`. +- `tests/test_source_intake_api.py` — httpx `ASGITransport` end-to-end pattern. +- `tests/steps/test_guest_bios_steps.py`, + `tests/steps/generation_orchestration_vidaimock.py` — existing Vidai Mock + subprocess + skip helpers to extract and reuse. +- `tests/features/*.feature`, `tests/steps/test_*.py` — pytest-bdd pattern. +- `tests/__snapshots__/*.ambr` — syrupy snapshots (existing TEI snapshots use + stable caller-supplied ids such as `xml:id="seg-intro"`). + +### Terms + +- TEI-P5: Text Encoding Initiative Guidelines, fifth edition; the canonical XML + format for the episode script. Spoken script uses the Performance Texts + module elements (`` speech, `` label, `` utterance, `

`/ + ``/`` spoken blocks) within ``. +- Generation run: a first-class resource representing one attempt to turn + ingested source material into a script, with a status lifecycle and an + append-only event log. +- Quality mode: the requested generation policy; the only value implemented in + this slice is `draft_without_qa`, which skips QA and records `qa_status` + `skipped`. +- Idempotency key: a client-supplied header making a side-effecting `POST` + safe to retry; identical bodies replay the original response (status, body, + `Location`, `Retry-After`), different bodies for the same key return + `409 Conflict`. +- Long-running operation: an operation that may outlive its initiating request; + returned as a pollable resource with `202 Accepted`, `Location`, and + `Retry-After`. See [RFC 6129](https://datatracker.ietf.org/doc/html/rfc6129) + for the TEI media type and the IBM Cloud / Microsoft long-running-operation + guidance cited in ADR 009 for the polling pattern. + +## Interfaces and dependencies + +The following names must exist at the end of the listed milestone. Prefer these +exact names and paths. Every new `episodic/canonical/*` module must be added to +the matching `pyproject.toml` architecture-group prefix list (`domain_ports` +for pure domain/ports, `application` for services, `outbound_adapter` for +storage/adapters); `episodic/generation/*` and `episodic/canonical/storage/*` +may already be blanket-covered — verify and add prefixes if not. + +### Domain (Milestone 1) — `episodic/canonical/` + +In `episodic/canonical/generation_quality.py` (new, `domain_ports` group, pure): + +```python +import enum + + +class QualityMode(enum.StrEnum): + """Requested generation quality policy for a run.""" + + DRAFT_WITHOUT_QA = "draft_without_qa" + + +class QaStatus(enum.StrEnum): + """Recorded QA outcome for a run and the TEI it produced.""" + + SKIPPED = "skipped" +``` + +Extend `GenerationRun` in `episodic/canonical/domain.py` with three fields +(defaults chosen to minimize construction-site churn) and validation: + +```python +quality_mode: QualityMode = QualityMode.DRAFT_WITHOUT_QA +qa_status: QaStatus | None = None +skip_qa_rationale: str | None = None +``` + +Validation in `__post_init__` (or a validating factory): when +`quality_mode is QualityMode.DRAFT_WITHOUT_QA`, `qa_status` must be +`QaStatus.SKIPPED` and `skip_qa_rationale` must be a non-empty string. Update +the ~5 `GenerationRun(` construction sites and the in-memory adapter, and +regenerate `tests/__snapshots__/test_generation_run_domain.ambr`. + +### Generation-run persistence (Milestone 2a) — `episodic/canonical/storage/` + +- `GenerationRunRecord` and `GenerationEventRecord` SQLAlchemy models + (`generation_runs`, `generation_events` tables) in a new + `episodic/canonical/storage/generation_run_models.py`. `generation_runs` + carries the lifecycle columns plus `started_at` (indexed), `ended_at`, + `error_message`, `error_category`, and `lease_expires_at` (nullable; reserved + for the 2.6.2 reaper). `generation_events` enforces a unique + `(generation_run_id, seq)` and allocates `seq` per run. +- `SqlAlchemyGenerationRunStore` implementing `GenerationRunRepository` and + `GenerationEventLog` (not `GenerationCheckpointPort`; checkpoints are out of + scope) in `episodic/canonical/storage/generation_runs.py`. + `update_run_status` for `pending → running` MUST be a conditional update + (`UPDATE … SET status='running' WHERE id=:id AND status='pending'`) returning + whether it won, so only one worker proceeds. +- `SqlAlchemyUnitOfWork` exposes `generation_runs` (run repository + event log) + so handlers and the launcher share one transaction boundary; mirror the + existing `workflow_checkpoints` attribute pattern. +- An Alembic migration creating the two tables, reversible, passing + `make check-migrations`. + +### Episode TEI revisioning (Milestone 2b) — `episodic/canonical/` + +- Episode columns added to `EpisodeRecord` and mirrored on `CanonicalEpisode` + (with defaults): `tei_revision: int = 0`, `tei_content_hash: str | None` + (`String(128)`), `qa_status: str | None`, + `last_generation_run_id: uuid.UUID | None`. +- `EpisodeRepository.update(self, episode, *, expected_revision: int)` added to + the protocol and implemented via a conditional `_update_where` + (`WHERE tei_revision = :expected_revision`), raising a revision-conflict + domain error on mismatch (the launcher is the sole writer and passes the + current revision). The update path MUST re-run `encode_text_for_storage` when + it rewrites `tei_xml` so the compressed and plain columns stay in sync. +- An Alembic migration adding the four episode columns, reversible. + +### Generation (Milestone 3) — `episodic/generation/` and `episodic/canonical/` + +In `episodic/generation/draft_script.py` (new): + +```python +class DraftScriptGenerator(typing.Protocol): + async def generate( + self, request: DraftScriptRequest + ) -> DraftScriptResult: ... +``` + +- `DraftScriptRequest` carries the source material (normalized text or source + TEI), presenter profiles (host/guest reference-document revisions), the + episode/series identifiers, and an injected `clock()` plus a deterministic id + factory (sequential `sp-1`, `p-1`, …) so output is reproducible. +- `DraftScriptResult` carries the emitted TEI-P5 XML, the content hash (bare + SHA-256 hex; the envelope adds any `sha256:` prefix — pin one and snapshot + it), and the LLM `LLMUsage`/`ProviderCallUsage` for cost accounting. +- `LLMDraftScriptGenerator` is the single-pass implementation using `LLMPort`; + it builds TEI via `tei-rapporteur` `from_dict`/`emit_xml` using ADR-006 + spoken containers (`` with `…

` turns), + validates the result, and maps each `LLMError` subclass to a typed failure. + +In `episodic/canonical/generation_persistence.py` (new, `application` group): + +- `materialise_episode_from_ingestion(...)` creates the episode from the + ingestion job's attached sources with a minimal placeholder TEI header + (reusing `_create_canonical_episode`/`TeiHeader` construction), returning the + episode id; called before the run is launched. +- `persist_draft_script(...)` writes the generated TEI into the episode within + one unit of work: increments `tei_revision` (optimistic), sets + `tei_content_hash`, sets `qa_status='skipped'`, and sets + `last_generation_run_id`. Validation failure raises a typed error the + launcher maps to `run.failed` + `tei.invalid`. + +### Launcher (Milestone 4) — `episodic/generation/` + +```python +class GenerationRunLauncher(typing.Protocol): + async def launch(self, run_id: uuid.UUID) -> None: ... +``` + +`InProcessGenerationRunLauncher` (placed in `episodic/generation/`, the +`application` group; the concrete is constructed in `episodic/api/runtime.py` +and injected via a new `ApiDependencies.launcher: GenerationRunLauncher | None` +field). It: + +- accepts `uow_factory`, the `DraftScriptGenerator`, the persistence service, a + `CostRecorder`, a `clock()`, a bounded-concurrency semaphore, and a task + registry (strong references) drained by a shutdown hook; +- per launched task opens its OWN fresh unit of work (never the request's); +- performs the conditional `pending → running` transition (skips if it did not + win); +- captures the correlation id at schedule time and binds it to the task's + logging context (it runs outside the request context); +- generates, persists, and records cost in-transaction; appends lifecycle + events (`run.started`, `draft.generated`, `tei.persisted`, `run.succeeded`); +- wraps generation AND persistence AND validation in one try/except mapping any + exception to `run.failed` + `error_message` + `error_category` + a terminal + status, emitting `run.failed` (and `tei.invalid` for validation failures), + mapping each `LLMError` subclass to a stable classified message; +- on shutdown drain, marks still-running drained runs `failed`. + +The launcher is a driven port so a Celery adapter replaces it later without +touching the REST or domain layers; the new ADR documents it as a degenerate +`TaskResumePort`. + +### HTTP API (Milestones 5–6) — `episodic/api/` + +New resources, wired by a new `_register_generation_run_routes` and +`_register_episode_tei_routes` in `episodic/api/app.py`: + +- `POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs` — + `GenerationRunsResource` for a ready ingestion job. Body: required + `quality_mode`, `skip_qa_rationale`, `actor`; optional accepted-and-ignored + `template_id`, `prompt_overrides`, `budget_hints` (do not `400` on them; + round-trip them into `configuration`/`budget_snapshot` so they survive). + Validation: missing/blank rationale or malformed body → `400`; + recognized-but-unsupported `quality_mode` → `422`. Returns `202` with + `Location: /v1/generation-runs/{run_id}` and `Retry-After`. The handler + creates the run row AND schedules the launcher inside one `work()`; + scheduling failure marks the run `failed`. +- `GET /v1/generation-runs/{run_id}` — `GenerationRunResource` (run snapshot; + `Retry-After` while non-terminal). +- `GET /v1/generation-runs/{run_id}/events` — `GenerationRunEventsResource` + (`after_seq` cursor + `limit` cap; `offset` deliberately omitted in favour of + the cursor). +- `GET /v1/episodes/{episode_id}/tei` — `EpisodeTeiResource`. + +Idempotency replay fix (Milestone 5): `IdempotentResponse` is extended with +optional `location` and `retry_after`; `_encode_outcome`/`_decode_outcome` +persist them and `apply_response` sets the headers, so a replayed/in-flight +`202` carries `Location` and `Retry-After` as ADR 009 requires. Reuse +`json_body_hash` from `episodic/api/source_intake_support.py` (not +`source_idempotency.py`) and a new operation constant `generation_run.create`. + +New serializers in `episodic/api/serializers.py`: `serialize_generation_run`, +`serialize_generation_event`, `serialize_tei_envelope`. The TEI envelope maps +internal `tei_revision → version` and `tei_content_hash → content_hash`, and +exposes `episode_id`, `tei_header_id`, `tei_xml`, `content_hash`, `version`, +`last_generation_run_id`, `quality_mode`, `qa_status`, and `updated_at`. A +content negotiation helper `negotiate_tei_media_type` in +`episodic/api/helpers.py`; the `tei+xml` response sets +`Content-Type: application/tei+xml` and +`Content-Disposition: attachment; filename="episode-.xml"`, and an `ETag` +derived from `tei_content_hash` for conditional GETs. + +## Plan of work + +Each milestone follows Red-Green-Refactor and ends with the relevant quality +gates. Do not proceed to the next milestone if the current gate fails. Commit +after each green-plus-refactor cycle. + +### Milestone 0 — Branch, plan baseline, red end-to-end scaffold + +Stage A: rename the working branch (see `Concrete steps`) and commit this +ExecPlan. + +Stage B (red): add a strict-xfail end-to-end behavioural feature +`tests/features/no_qa_generation_slice.feature` and a step module +`tests/steps/test_no_qa_generation_slice.py` whose scenarios express the whole +slice, bound with +`@pytest.mark.xfail(strict=True, reason="4.3.2 not implemented")` so +`make test` stays green while the target behaviour is captured. This is the +executable definition of done; the markers are removed in Milestone 7. + +The feature (full text embedded here so the plan is self-contained): + +```gherkin +Feature: No-QA source-to-script generation slice + + The source-to-script workflow generates a draft script without QA and + downloads the TEI for validation over REST + + Background: + Given a Vidai Mock inference server is running + And a series profile exists + And a host presenter profile and a guest presenter profile are bound + + Scenario: Draft generation without QA produces a downloadable TEI-P5 script + Given an ingestion job with an attached source document + When I create a draft-without-qa generation run for the ingested episode + Then the run creation responds 202 Accepted with a Location header + And the response carries a Retry-After header + And the run is created with qa_status "skipped" and the supplied skip_qa_rationale is recorded + When I poll the generation run until it reaches a terminal state + Then the run status is "succeeded" + And the event log contains a "tei.persisted" event + When I fetch the episode TEI as application/tei+xml + Then the response is a TEI-P5 attachment with qa_status "skipped" + And the TEI validates against the Episodic TEI-P5 profile + + Scenario: Reusing an idempotency key with the same body replays the run + Given an ingestion job with an attached source document + When I create a draft-without-qa run twice with the same idempotency key and body + Then both responses describe the same run id + And the replayed response carries the same Location and Retry-After + + Scenario: Reusing an idempotency key with a different body conflicts + Given an ingestion job with an attached source document + When I create a draft-without-qa run, then reuse the key with a different rationale + Then the second response is 409 Conflict + + Scenario: A missing rationale is rejected + Given an ingestion job with an attached source document + When I create a draft-without-qa run without a skip_qa_rationale + Then the response is 400 Bad Request + + Scenario: An unsupported quality mode is unprocessable + Given an ingestion job with an attached source document + When I create a generation run with quality_mode "qa_gated" + Then the response is 422 Unprocessable Entity + + Scenario: Generation failure is reported on the run + Given an ingestion job with an attached source document + And the inference server is configured to fail + When I create a draft-without-qa generation run for the ingested episode + And I poll the generation run until it reaches a terminal state + Then the run status is "failed" + And the run records an error message and an error category + + Scenario: A malformed completion is reported as a failed run + Given an ingestion job with an attached source document + And the inference server is configured to return a non-TEI completion + When I create a draft-without-qa generation run for the ingested episode + And I poll the generation run until it reaches a terminal state + Then the run status is "failed" + And the event log contains a "tei.invalid" event +``` + +Gate: `make check-fmt && make typecheck && make lint && make test` (the new +scenarios are xfail, so the suite stays green). Commit. + +### Milestone 1 — Domain model extensions + +Red: extend `tests/test_generation_run_domain.py` asserting the new +`QualityMode`/`QaStatus` enums and that a `draft_without_qa` run with an empty +rationale raises `ValueError`, and that a valid run round-trips its new fields. +Run; observe failure. + +Green: add `episodic/canonical/generation_quality.py`, extend `GenerationRun` +with the three fields and validation, export from +`episodic/canonical/__init__.py`, update the construction sites and the +in-memory adapter, and add the new module to the `domain_ports` prefix list in +`pyproject.toml`. + +Refactor: regenerate `tests/__snapshots__/test_generation_run_domain.ambr`; +confirm `make lint` (`make check-architecture`) passes. + +Gate: `make check-fmt && make typecheck && make lint && make test`. Commit. + +### Milestone 2a — Durable generation-run and event persistence + +Red: add persistence tests under `tests/canonical_storage/`: + +- `SqlAlchemyGenerationRunStore` round-trips a run, lists runs by episode with + pagination and status filter, appends events with monotonically increasing + `EventSeq` (unique per run), and lists events `after_seq`. +- the conditional `pending → running` update wins exactly once when called + twice concurrently (simulate by two sequential guarded updates; the second + reports it did not win). + +Add a Hypothesis property test: appending N events in arbitrary interleavings +yields strictly increasing seqs and an `after_seq` pagination that partitions +the log with no gaps or duplicates. + +Green: add `generation_run_models.py`, the `generation_runs.py` store, the +`SqlAlchemyUnitOfWork.generation_runs` attribute, the Alembic migration, and the +`outbound_adapter`/`storage` prefix entries if needed. Verify the durable +`SqlAlchemyIdempotencyStore` is already wired on the unit of work; if absent, +escalate per Tolerances. + +Gate: + +```bash +make check-fmt +make typecheck +make lint +make check-migrations +make test +``` + +Commit. + +### Milestone 2b — Episode TEI revisioning columns and optimistic update + +Red: add `tests/canonical_storage/` tests asserting that the episode TEI +`update` increments `tei_revision`, stores `tei_content_hash`, sets `qa_status` +and `last_generation_run_id`, keeps the zstd-compressed column in sync, is +visible in a fresh unit of work, and raises a revision-conflict error when +`expected_revision` does not match. Run; observe failure. + +Green: add the four episode columns and mappers, the +`EpisodeRepository.update(..., expected_revision=...)` method (conditional +`_update_where`, re-encoding `tei_xml`), and an additive reversible migration. + +Gate: full Python gates including `make check-migrations`. Commit. + +### Milestone 3 — Draft script generator, episode materialization, persistence + +Red: add unit tests in `tests/test_draft_script_generation.py` using +`FakeLLMPort` returning a canned script-shaped payload, an injected frozen +clock, and the deterministic id factory. Assert the generator emits TEI-P5 that +`tei-rapporteur` parses and validates, contains the expected ``/``/ +`

` turns, and reports a stable content hash; add a syrupy snapshot of the +emitted TEI XML (clock frozen, ids pinned). Add tests for each `LLMError` +subclass mapping to a typed failure. Add tests for +`materialise_episode_from_ingestion` (creates episode + placeholder TEI from an +ingestion job) and `persist_draft_script` (episode TEI, revision, hash, +`qa_status`, `last_generation_run_id`; invalid TEI raises the typed error). + +Green: implement `DraftScriptGenerator`/`LLMDraftScriptGenerator` +(`episodic/generation/draft_script.py`) and +`episodic/canonical/generation_persistence.py`; add prefix entries. + +Refactor: extract shared TEI-construction helpers into +`episodic/generation/tei_payload.py`; keep generator pure of HTTP/DB. + +Gate: full Python gates. Commit. + +### Milestone 4 — Launcher, lifecycle events, cost wiring, observability + +Red: add tests in `tests/test_generation_run_launcher.py`: + +- a logic test driving the launcher to completion against a persisted `pending` + run and a `FakeLLMPort`, asserting status transitions, appended event kinds + in order, persisted TEI, and recorded cost-ledger entry; +- a failure test asserting an LLM error and an invalid-TEI case each yield + `failed` with `error_message`/`error_category`, `run.failed`, and (for + invalid TEI) a `tei.invalid` event — not a leaked exception; +- a **detached-session** test that launches through real `asyncio.create_task` + against py-pglite and asserts the task writes through its own session while a + request-scoped unit of work is already closed; +- a shutdown-drain test asserting a still-running drained run is marked + `failed`. + +Green: implement `InProcessGenerationRunLauncher`; construct it plus a +`CostRecorder` over `SqlAlchemyCostLedgerStore` in `episodic/api/runtime.py`; +add the `ApiDependencies.launcher` field; bind the correlation id and emit the +ADR-009-required structured logs and metrics (draft latency, terminal-state +counters, draft error rate by category, QA-bypass rate; idempotency +replay/conflict counters are emitted in M5). + +Refactor: ensure the launcher depends only on ports (run store, event log, +generator, persistence, cost recorder, clock), holds strong task references, +and bounds concurrency with a semaphore. + +Gate: full Python gates. Commit. + +### Milestone 5 — Generation-run REST endpoints with idempotency + +Red: add endpoint tests in `tests/test_generation_run_api.py` using +`canonical_api_async_client` + httpx `ASGITransport` (mirror +`tests/test_source_intake_api.py`): valid create → `202` + `Location` + +`Retry-After`, records `qa_status=skipped` and rationale, and triggers the +launcher; missing rationale → `400`; `quality_mode="qa_gated"` → `422`; +`GET /generation-runs/{id}` → snapshot + `Retry-After` while non-terminal; +`GET /generation-runs/{id}/events` paginates by `after_seq`/`limit`; an +idempotent replay returns the same run id AND the same `Location`/ +`Retry-After`; a different body with the same key → `409`. + +Add Hypothesis property tests for idempotency invariants against the in-memory +`IdempotencyStore`/`run_idempotent` (fast, pure): identical body + same key +never creates more than one run; different body + same key → `409`; in-flight +duplicates return the stored `202` metadata including `Location`/`Retry-After`. +Cover the SQL adapter with a couple of example-based tests only (avoid +Hypothesis × py-pglite flake). + +Green: extend `IdempotentResponse` with `location`/`retry_after` and update +`_encode_outcome`/`_decode_outcome`/`apply_response`; implement the three +resources, serializers, and route registration; reuse `run_idempotent`/ +`IdempotencyContext` and `json_body_hash` (from `source_intake_support`). +Inject the launcher and clock via `ApiDependencies`. Emit idempotency +replay/conflict metrics. + +Refactor: keep resources thin (parse → service/port → serialize); map domain +errors through the existing error-envelope helpers. + +Gate: full Python gates. Commit. + +### Milestone 6 — Episode TEI retrieval with content negotiation + +Red: add tests in `tests/test_episode_tei_api.py`: default `GET .../tei` +returns the JSON envelope (with the `tei_revision → version`, +`tei_content_hash → content_hash` mapping and a stable, snapshotted shape under +a frozen clock); `Accept: application/tei+xml` returns the raw TEI-P5 body with +`Content-Type: application/tei+xml`, +`Content-Disposition: attachment; filename="episode-.xml"`, and an `ETag`; +a request before any draft exists returns `404`. + +Green: implement `EpisodeTeiResource`, `negotiate_tei_media_type`, and +`serialize_tei_envelope`; register the route. + +Refactor: factor the content-negotiation helper for reuse; document it in the +developers' guide (Milestone 8). + +Gate: full Python gates. Commit. + +### Milestone 7 — End-to-end behavioural slice with Vidai Mock + +Red→Green: implement the Milestone 0 step module and remove the xfail markers. +Extract the existing Vidai Mock subprocess helpers (`_start_vidaimock_process`, +`_await_port_ready`, `_terminate_process_gracefully`, and the `shutil.which` +skip) from `tests/steps/test_guest_bios_steps.py` into a shared +`tests/fixtures/` module and reuse them. Configure a provider template +returning a deterministic TEI-script-shaped completion; use +`X-Vidai-Chaos-Drop: 100` for the failure scenario and a non-TEI completion +template for the malformed-completion scenario. Drive the full flow with the +async HTTP client. + +Acceptance gate (not optional): the slice's headline scenario MUST be observed +passing at least once on a host where `vidaimock` is available; capture that +transcript in `Artefacts and notes`. The graceful `shutil.which` skip keeps CI +green where the binary is absent, but the plan is not done until the green +transcript exists. Record where CI obtains the `vidaimock` binary. + +Gate: full Python gates including the now-active scenarios. Commit. + +### Milestone 8 — Documentation, roadmap update, final gates + +1. Add `docs/adr/adr-017-no-qa-generation-run-execution-and-tei-persistence.md` + recording: the in-process launcher port (a degenerate `TaskResumePort`) and + the Celery deferral, the single-worker assumption and stuck-run hooks (lease + columns, conditional transitions, manual-fail runbook), episode + materialization from ingestion, the `DraftScriptGenerator` port and its + 4.4.1 successor, episode TEI revisioning and optimistic update, the + 422-vs-400 and 404 contract decisions, and the content-negotiation approach. + Reference it from ADR 009 and the system design. +2. Update `docs/episodic-podcast-generation-system-design.md` + (source-to-script vertical slice) with the implemented decisions. +3. Update `docs/users-guide.md` with the trigger → poll → download workflow, + the `Accept: application/tei+xml` download, and the draft/QA-skipped caveat. +4. Update `docs/developers-guide.md` with the content-negotiation helper, the + generation-run launcher seam and its lifecycle/observability conventions, + and the Vidai Mock test harness; add new modules to + `docs/repository-layout.md` and index new docs in `docs/contents.md`. +5. Mark roadmap item 4.3.2 as done in `docs/roadmap.md`. + +Gate: + +```bash +make check-fmt +make typecheck +make lint +make check-migrations +make test +make markdownlint +make nixie +``` + +Commit. + +## Concrete steps + +Run all commands from the repository root +(`/home/leynos/.lody/repos/github---leynos---episodic/worktrees/7cb51da6-204f-4329-8dfd-ca46598e888c`). + +Branch rename and tracking (Milestone 0): + +```bash +git branch -m 4-3-2-no-qa-generation-runs-and-tei-p5-retrieval +git push -u origin 4-3-2-no-qa-generation-runs-and-tei-p5-retrieval +``` + +Quality gates (run sequentially to benefit from build caching; never in +parallel), teeing output for review: + +```bash +make check-fmt 2>&1 | tee /tmp/check-fmt-$(git branch --show-current).out +make typecheck 2>&1 | tee /tmp/typecheck-$(git branch --show-current).out +make lint 2>&1 | tee /tmp/lint-$(git branch --show-current).out +make check-migrations 2>&1 | tee /tmp/check-migrations-$(git branch --show-current).out +make test 2>&1 | tee /tmp/test-$(git branch --show-current).out +``` + +Markdown gates (run unsandboxed where required; see the memory note on nixie): + +```bash +make markdownlint 2>&1 | tee /tmp/markdownlint-$(git branch --show-current).out +make nixie 2>&1 | tee /tmp/nixie-$(git branch --show-current).out +``` + +Vidai Mock for behavioural tests (Milestone 7): + +```bash +command -v vidaimock # confirm the binary is available before expecting M7 to run +``` + +Run a single focused test during Red-Green: + +```bash +uv run pytest tests/test_generation_run_api.py -k idempotency -x -q +``` + +Expected transcripts (illustrative until captured for real in Artefacts): + +```plaintext +$ uv run pytest tests/steps/test_no_qa_generation_slice.py -q # M0 (red, before impl) +XFAIL tests/steps/test_no_qa_generation_slice.py::test_draft_generation ... +N xfailed in N.Ns +``` + +```plaintext +$ command -v vidaimock && vidaimock --version +/home/leynos/.local/bin/vidaimock +vidaimock 0.1.3 +$ uv run pytest tests/steps/test_no_qa_generation_slice.py -q +....... [100%] +7 passed, 1 warning in 5.36s +``` + +CI installs the pinned Vidai Mock release from the `vidaiUK/VidaiMock` GitHub +release archive in `.github/workflows/ci.yml`, verifies its SHA-256 digest, and +places the binary in `$HOME/.local/bin` before running tests. + +## Validation and acceptance + +The slice is done when: + +- `make check-fmt`, `make typecheck`, `make lint`, `make check-migrations`, + `make test`, `make markdownlint`, and `make nixie` all pass. +- The behavioural scenarios in `tests/features/no_qa_generation_slice.feature` + pass, and the headline scenario has been **observed passing at least once** + with a `vidaimock` binary present (transcript captured in + `Artefacts and notes`). The `shutil.which` skip only protects binary-less CI; + it does not satisfy the acceptance gate. +- Manually, against a running service with a Vidai Mock backend, create a run + for a ready ingestion job with: + `POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs` and + `{"quality_mode":"draft_without_qa","skip_qa_rationale":"vertical-slice demo","actor":"editor@example.test"}` + and an `Idempotency-Key` returns `202 Accepted` with + `Location: /v1/generation-runs/{run_id}` and `Retry-After`; polling + `GET /v1/generation-runs/{run_id}` reaches `succeeded`; + `GET /v1/generation-runs/{run_id}/events` shows a `tei.persisted` event; and + `GET /v1/episodes/{episode_id}/tei` with `Accept: application/tei+xml` + downloads a TEI-P5 attachment with `qa_status=skipped`. Replaying the same + request returns the same run id with the same `Location`/`Retry-After`; a + different body with the same key returns `409`. + +Red-Green-Refactor evidence must be recorded per milestone: the focused test +command and its red failure (the M0 scenarios use +`@pytest.mark.xfail(strict=True, ...)` to prove the red stage; markers removed +in M7), the green pass after the minimal change, and the wider-gate pass after +refactor. + +Quality criteria: + +- Tests: unit (`pytest`), behavioural (`pytest-bdd`), snapshot (`syrupy` for TEI + XML and the JSON envelope, with a frozen clock and pinned ids), and property + (`hypothesis` for idempotency invariants and event-seq + monotonicity/pagination) all pass. CrossHair (`make crosshair`) is only + extended if a new pure PEP-316-contracted helper is introduced (e.g. a + content-hash or revision-increment helper); otherwise it is unchanged. +- Observability: the ADR-009-required structured logs, metrics, and + correlation-id propagation across the asyncio boundary are present and + asserted in M4/M5 tests. +- Lint/typecheck/migrations: `make lint` (incl. `make check-architecture`), + `make typecheck`, and `make check-migrations` clean. + +Verification method: run the gate commands after each milestone and run +`coderabbit review --agent`, clearing all concerns, before moving on. +CodeRabbit must only see code that already passes the deterministic gates. + +### Verification note (Rust/Verus and Kani) + +ADR 009 anticipates Kani/Verus only "if the idempotency state machine moves +into Rust". This slice keeps idempotency in Python (reusing the existing +`IdempotencyStore` and `run_idempotent`), so no Rust extension or formal proof +is introduced; the idempotency invariants are covered by Hypothesis property +tests. If a future task moves the state machine to Rust, add the bounded model +checking then. + +## Idempotence and recovery + +- Re-running any gate command is safe and idempotent. +- Database tests reset the `public` schema per test via the py-pglite fixtures. +- Alembic migrations must be reversible (`downgrade` drops the added columns and + tables); verify with an upgrade/downgrade/upgrade cycle locally. +- Generation-run creation is idempotent by `Idempotency-Key`; retrying with the + same key and body returns the original run with its stored `Location` and + `Retry-After`. +- `pending → running` is a conditional update so only one worker proceeds; the + episode TEI write is optimistic on `tei_revision`. This prevents double-spend + and clobbering if a future reaper re-launches. +- If the service restarts mid-run, the run may remain `running` (no automated + reaper in this slice). The `started_at`/`lease_expires_at` columns support + manual recovery; the manual-fail runbook in ADR 017 and the developers' guide + describes how to inspect expiry, conditionally fail a stuck run, append its + failure event, and preserve its idempotency record. The automated reaper is a + 2.6.2 follow-up. +- Known limitation: a `pending`/`running` run that outlives the 24h idempotency + TTL could let a replayed key create a second run; documented for the 2.6.2 + recovery work. + +## Artefacts and notes + +Capture as work proceeds: the M0 red xfail transcript; the M2a/M2b +`make check-migrations` clean output; the M3 emitted-TEI snapshot diff; the M4 +detached-session test transcript; and the M7 green behavioural-slice transcript +on a `vidaimock`-equipped host (mandatory acceptance evidence). + +- M0 red scaffold evidence (2026-06-24): + + ```plaintext + $ uv run pytest tests/steps/test_no_qa_generation_slice.py -q + xxxxxxx [100%] + 7 xfailed in 0.48s + ``` + +- M0 deterministic gate evidence (2026-06-24): + `make check-fmt` reported `420 files already formatted`; `make typecheck` + reported `All checks passed!`; `make lint` passed Hecate and Ruff and rated + Pylint `10.00/10`; `make test` reported + `962 passed, 2 skipped, 7 xfailed in 74.14s`; `make markdownlint` reported + `Summary: 0 error(s)`; and `make nixie` reported all diagrams validated. + +- M0 CodeRabbit evidence (2026-06-24): + `coderabbit review --agent` ended with + `{"type":"complete","status":"review_completed","findings":0}`. + +- M1 red evidence (2026-06-24): + + ```plaintext + E ModuleNotFoundError: No module named + E 'episodic.canonical.generation_quality' + ``` + +- M1 focused green evidence (2026-06-24): + + ```plaintext + $ uv run pytest tests/test_generation_run_domain.py \ + tests/test_generation_run_port_contract.py \ + tests/test_generation_run_properties.py \ + tests/steps/test_generation_run_lifecycle_steps.py -q + ...................................... [100%] + 38 passed in 1.43s + ``` + +- M1 deterministic gate evidence (2026-06-24): + `make check-fmt` reported `421 files already formatted`; `make typecheck` + reported `All checks passed!`; `make lint` passed Hecate and Ruff and rated + Pylint `10.00/10`; `make test` reported + `966 passed, 2 skipped, 7 xfailed in 68.44s`; `make markdownlint` reported + `Summary: 0 error(s)`; and `make nixie` reported all diagrams validated. + +- M1 CodeRabbit evidence (2026-06-24): the first review attempt returned a + recoverable `rate_limit` response; after the required `vsleep` backoff, the + retry ended with + `{"type":"complete","status":"review_completed","findings":0}`. + +- M2a red evidence (2026-06-24): + + ```plaintext + E ImportError: cannot import name 'GenerationEventRecord' from + E 'episodic.canonical.storage' + ``` + +- M2a focused green evidence (2026-06-24): + + ```plaintext + $ uv run pytest tests/canonical_storage/test_generation_runs.py -q + ........ [100%] + 8 passed in 3.53s + + $ uv run pytest tests/test_generation_run_port_contract.py \ + tests/test_generation_run_properties.py -q + ................. [100%] + 17 passed in 0.68s + + $ uv run pytest tests/canonical_storage/test_generation_runs.py \ + tests/test_generation_run_port_contract.py \ + tests/test_generation_run_properties.py \ + tests/steps/test_generation_run_lifecycle_steps.py -q + ............................. [100%] + 29 passed in 4.29s + ``` + +- M2a migration gate evidence (2026-06-24): `make check-migrations` applied + migrations through `20260624_000010` and exited cleanly with no schema drift + reported. + +- M2a deterministic gate evidence (2026-06-24): `make check-fmt` reported + `426 files already formatted`; `make typecheck` reported `All checks passed!`; + `make lint` passed Hecate and Ruff and rated Pylint `10.00/10`; `make test` + reported `975 passed, 2 skipped, 7 xfailed in 73.81s`; and + `make check-migrations` exited cleanly after applying migrations through + `20260624_000010`. + +- M2a CodeRabbit evidence (2026-06-24): `coderabbit review --agent` ended with + `{"type":"complete","status":"review_completed","findings":0}`. + +- Intermediate post-rebase validation evidence (2026-07-22): `make check-fmt`, + `make typecheck`, and `make lint` passed; Pylint rated the branch `10.00/10`. + `make test` reported `1066 passed, 1 skipped, 7 xfailed`. At that point, the + xfails were the unchanged M7 behavioural scaffold, not completed acceptance + evidence. `make markdownlint` and `make nixie` also passed after refreshing + the generated spelling configuration and correcting Markdown drift. + +- Intermediate red-state behavioural evidence (2026-07-22): + + ```plaintext + $ uv run pytest tests/steps/test_no_qa_generation_slice.py -q + xxxxxxx [100%] + 7 xfailed in 0.15s + ``` + + Each scenario was a strict expected failure with the reason + `4.3.2 no-QA source-to-script slice is not implemented yet`. + +- Final behavioural evidence (2026-07-22): + + ```plaintext + $ uv run pytest tests/steps/test_no_qa_generation_slice.py -q + ....... [100%] + 7 passed, 1 warning in 5.36s + ``` + +- Final correctness-review evidence (2026-07-22): focused regression tests + reported `30 passed`; all deterministic gates passed, with `make test` + reporting `1078 passed, 1 skipped`. The complete-branch + `coderabbit review --agent` run reported zero findings. + +- M4 review evidence (2026-07-22): after `make check-fmt`, `make test`, + `make typecheck`, `make lint`, `make check-migrations`, `make markdownlint`, + and `make nixie` passed, `coderabbit review --agent` ended with + `{"type":"complete","status":"review_completed","findings":0}`. + +- M5 review evidence (2026-07-22): `make check-fmt`, `make test`, + `make typecheck`, `make lint`, `make check-migrations`, `make markdownlint`, + and `make nixie` passed. A staged-delta `coderabbit review --agent` included + `episodic/api/resources/generation_runs.py` and + `tests/test_generation_run_api.py` and ended with + `{"type":"complete","status":"review_completed","findings":0}`. + +- M6 review evidence (2026-07-22): `make check-fmt`, `make test`, + `make typecheck`, `make lint`, `make check-migrations`, `make markdownlint`, + and `make nixie` passed. A staged-delta `coderabbit review --agent` included + `episodic/api/resources/episode_tei.py` and `tests/test_episode_tei_api.py` + and ended with `{"type":"complete","status":"review_completed","findings":0}`. + +## Outcomes & retrospective + +The branch has delivered the domain metadata, durable generation-run and event +persistence, episode TEI revisioning, deterministic draft generation and +persistence services, and the in-process launcher with lifecycle events, cost +wiring, failure classification, detached unit-of-work ownership, and shutdown +handling. The later `GenerationRunStatusUpdate` refactor keeps lifecycle +updates cohesive, and `_require_mutable_run` removes duplicated missing and +terminal checks from the in-memory adapter. + +All eight milestones are complete. The implementation exposes durable no-QA +generation runs and event polling, resolves source and presenter context, +persists revisioned TEI with explicit skipped-QA provenance, and serves JSON or +downloadable TEI representations. Upload-backed sources are hydrated from +object storage before generation, repeated materialization converges on the +ingestion job's persisted episode, and terminal status updates are serialized. +Full deterministic gates pass with `1078 passed, 1 skipped`; CodeRabbit's final +complete-branch review reported zero findings. ADR 017 and the maintainer and +user guides record the operational limits and successor work. Automated +stuck-run recovery, the broader checkpoint REST surface, and the full QA-bypass +generation graph remain assigned to 2.6.2, 2.6.3, and 4.4.1. + +## Current progress and decision + +As of `eaea7c9` (2026-08-17), the in-process launcher hardening was present: +admission is bounded before task allocation, overload is persisted as +`launcher.overloaded`, launcher/provider/database shutdown is serialized, and +tracing plus bounded production metrics are injectable. This entry records the +interim implementation progress and associated operational decision; the scoped +hardening follow-up was completed on 2026-08-20, as recorded below. Roadmap +item 4.3.2 is complete for the delivered core slice. Documentation +synchronization and the explicitly deferred automatic recovery and QA-gated +successor remain separate follow-up work. + +Implementation progress across `fde42bc`, `eaea7c9`, and `b4dc700` also covers +the unified cost-recorder protocol and a shorter materialization lock scope: +sources are validated before locking, and the episode/target association is +committed before source projection. Durable claim-outcome logging and race +coverage are present, but broader durability and property-based verification +for this hardening slice remain outstanding. + +Latest rebase qualifier: the structural replay onto the force-rewritten +`de457b7ba3d6a13df8600410a0bc7aea25658ff8` target is recorded above, including +the ancestor and marker-free-tree checks and byte-identical lockfile result. +Final validation passed: `make check-fmt` passed 467 files; `make test` passed +1,091, skipped 1, with 54 snapshots (26 warnings); and `make typecheck` passed +with 3 unused-ignore warnings. `make lint`, `make check-migrations`, corrected +`make markdownlint`, `make nixie`, and `mbake validate Makefile` passed. +`git diff --check` also passed. Two CodeRabbit attempts using +`--base origin/configure-df12-lints` stopped externally at `preparing_sandbox` +with no findings, diagnostic, or rate-limit indication, so no `vsleep` was +warranted. A third attempt from clean committed head +`27477971b2153a5ba5f86f85d4ad991cea128cb6` stopped externally at +`connecting_to_review_service` after ~6.4s, with no findings or rate-limit +indication. An explicit lease-protected force push replaced remote `74b8bd1` +with `2747797`; PR #141 was retargeted to `configure-df12-lints`; and +`gh stack link --base main 220 141` created remote stack #240. PR metadata +verifies #220 as base `main`/head `de457b7` and #141 as base +`configure-df12-lints`/head `2747797`. Because `gh stack view` has no local +tracking by design after linking, the successful link output and PR metadata +are the verification evidence. Push/PR/stack actions are complete. + +2026-08-11 rebase outcome: the branch was rebased onto +`origin/configure-df12-lints` at `db01ed84e917b5f8411ea7ab75f55c5505d3c972`; +`ddc1c17` was safely skipped as the rewritten equivalent of target `698e2fa`; +and `f7adc07` was resolved with the target `uv.lock`, `uv lock`, and +confirmation that the lockfile matches the target. Forty-one feature commits +replayed, followed by formatter-only commit +`e168e39560cc73ea2b8726ac0e09ddb272cd1e6b`. Sequential gate evidence is +`make check-fmt` passed; `make test` 1,099 passed/3 skipped/53 snapshots passed; +`make typecheck` 0 errors/3 warnings; and `make lint` passed/Pylint 10/10. +Force-with-lease published remote `70e67b4` as current commit +`f3243528a12515d4fa4d421aa4e73da8d3c506d3`. PR #141 already had the requested +base `configure-df12-lints`; direct `gh pr edit` was rejected because it +belongs to a stack. PR #220 is `main <- configure-df12-lints`, and +`gh stack link --base main 220 141` confirmed that the two-PR stack is already +up to date. Remote head equals local. + +## Revision note + +Revised 2026-06-15 after a Logisphere community-of-experts design review. +Changes: split Milestone 2 into 2a (runs/events) and 2b (episode TEI columns) +to respect the file/line tolerance; made the launcher own a fresh unit of work +(use-after-free fix) with a dedicated detached-session test; added durable +stuck-run hooks (indexed `started_at`, `lease_expires_at`, `error_category`, +conditional `pending → running`, optimistic TEI update); extended idempotency +replay to carry `Location`/`Retry-After`; corrected the `json_body_hash` module +path and added the `generation_run.create` operation; specified the +request-body required/optional split and the 422-vs-400 and 404 contract +decisions; made cost-recorder wiring an explicit deliverable; specified +deterministic TEI ids and clock injection for stable snapshots; clarified that +episode materialization is a new step (not pure reuse); mandated +`pyproject.toml` architecture-group registration for new modules; broadened +LLM-failure and malformed-completion handling and tests; required observability +acceptance; and turned the Vidai Mock skip into a must-run-once acceptance gate +reusing existing helpers. These strengthen correctness and operability without +enlarging the externally observable contract. + +Revised 2026-07-22 after a PR-head status audit. The audit identified the then +missing M5-M7 REST and behavioural deliverables; subsequent milestones closed +those gaps, completed M8 documentation, and checked the roadmap item complete. +The audit also recorded the post-rebase status-update parameter object and the +CodeScene complexity disposition. + +Revised 2026-07-22 after post-implementation correctness review. Recorded that +presenter resolution was already present, then added object-store hydration for +uploaded source text, transactional ingestion-job episode association, and +row-locked terminal status mutation. Final test evidence is +`1078 passed, 1 skipped`; the complete-branch CodeRabbit review reported zero +findings. Final review evidence: `coderabbit review --agent` ran against commit +`f8609b4` on 2026-07-23 and returned terminal `review_completed` with zero +findings. + +Revised 2026-07-31 after the rebase and final validation pass. The branch was +rebased onto `origin/configure-df12-lints` at +`1d49da451abd6f7d276e91693d945ce5e5b7945a`; one explicit repository conflict +and the marker-free sem/Weave replay artefacts were repaired using target +conventions. The target `uv.lock` remained byte-identical after `uv lock`, and +DF12 lint-base fixes preserved behaviour without weakening configuration. The +new final evidence is recorded in Progress: all named formatting, test, typing, +lint, migration, Markdown, Mermaid, Makefile, and diff checks passed, including +1091 passed tests, 1 skipped test, 54 snapshots, and zero Markdown issues in +112 files. The final review, +`coderabbit review --agent --base origin/configure-df12-lints`, completed in +177.037s with zero findings and no rate-limit retry. + +The target then advanced through 14 unrelated lint follow-up commits before +publication, reaching final target `5fd1f97d375f5703b90c5cb366f6a45a2d229e37`. +Weave predicted no overlap; the second 38-commit replay was conflict-free, the +final target is an ancestor, and `uv.lock` remains byte-identical. The complete +nine-gate suite passed again on that final base with the same 1091 passed, 1 +skipped, and 54 snapshots. `make typecheck` exited successfully with three +unused-ignore warnings confined to target-only `tests/test_serializers.py`. The +initial `1d49da4` conflict history and prior CodeRabbit evidence remain +unchanged. + +The superseding final-base review after the conflict-free replay onto final +target `5fd1f97d375f5703b90c5cb366f6a45a2d229e37`, using +`coderabbit review --agent --base origin/configure-df12-lints`, completed in +98.255s with zero findings and no rate-limit retry. The earlier 177.037s +pre-advance review remains historical evidence. + +Revised 2026-07-31 after the latest force-rewritten-target rebase. Recorded the +`de457b7ba3d6a13df8600410a0bc7aea25658ff8` target, the carefully inspected and +skipped obsolete base-owned commits `f9a4afa` (lint adoption) and `4ed178a` +(architecture/lock repair), all 39 replayed feature commits, Weave's three +very-high-confidence entity-pair resolutions, and the cohesive feature +extraction confirmed by `sem diff`. Also recorded that target `slots=True` in +`_TextUploadRequest`, the feature workflow artefact fallback, the ancestor and +marker-free tree, and the byte-identical target lockfile after `uv lock` +survived. Final gates passed: `make check-fmt` passed 467 files; `make test` +passed 1,091, skipped 1, with 54 snapshots (26 warnings); and `make typecheck` +passed with 3 unused-ignore warnings. `make lint` and `make check-migrations` +passed; `make markdownlint` passed after the three new `artefact` spellings +were corrected to `artefact`; and `make nixie`, `mbake validate Makefile`, and +`git diff --check` passed. CodeRabbit was attempted twice with +`--base origin/configure-df12-lints`; both attempts stopped externally at +`preparing_sandbox` with no findings, diagnostic, or rate-limit indication, so +no `vsleep` was warranted. A third attempt from clean committed head +`27477971b2153a5ba5f86f85d4ad991cea128cb6` stopped externally at +`connecting_to_review_service` after ~6.4s, with no findings or rate-limit +indication. An explicit lease-protected force push replaced remote `74b8bd1` +with `2747797`; PR #141 was retargeted to `configure-df12-lints`; and +`gh stack link --base main 220 141` created remote stack #240. Verification +showed #220 with base `main` and head `de457b7`, and PR #141 with base +`configure-df12-lints` and head `2747797`. `gh stack view` has no local +tracking by design after linking, so PR metadata and successful link output are +verification. Push/PR/stack actions are complete. + +Revised 2026-08-11 after the rebase onto `origin/configure-df12-lints` at +`db01ed84e917b5f8411ea7ab75f55c5505d3c972`. Recorded that `ddc1c17` was safely +skipped because target `698e2fa` is the rewritten equivalent; `f7adc07` was +resolved by taking the target `uv.lock`, running `uv lock`, and confirming it +matches the target; 41 feature commits replayed; and formatter-only commit +`e168e39560cc73ea2b8726ac0e09ddb272cd1e6b` follows. Sequential gate evidence is +`make check-fmt` passed, `make test` 1,099 passed/3 skipped/53 snapshots passed, +`make typecheck` 0 errors/3 warnings, and `make lint` passed/Pylint 10/10. +Force-with-lease published remote `70e67b4` as current commit +`f3243528a12515d4fa4d421aa4e73da8d3c506d3`. PR #141 already had the requested +base `configure-df12-lints`; direct `gh pr edit` was rejected because it +belongs to a stack. PR #220 is `main <- configure-df12-lints`, and +`gh stack link --base main 220 141` confirmed that the two-PR stack is already +up to date. Remote head equals local. + +Revised 2026-08-17 to record the current review-remediation decisions for +explicit launcher configuration, locked and validated ingestion materialization +with verified duplicate recovery, generation-run foreign keys and fixture +requirements, atomic manual recovery, allow-listed observability attributes, +and the stale series-profile `Idempotency-Key` finding. No roadmap status was +changed. + +Hardening progress, 2026-08-20: production runtime now requires an explicitly +configured bearer authorization adapter and assigns ingestion-job ownership and +generation-run actors from the trusted principal. Generation-run, event, and +TEI reads use non-disclosing owner checks and bounded request spans. Source +projection duplicate detection moved from the application service into the +SQLAlchemy adapter's savepoint boundary, returning a typed duplicate result so +the service retains deterministic post-race verification. The launcher commits +its claim and `run.started` event before a brief metadata-read unit of work; +upload hydration begins only after that unit of work closes. Source input has +configured count, per-source, aggregate, and normalized-text bounds. Focused +persistence, API tracing, runtime, and launcher tests pass; broader property +and end-to-end validation remains in progress. + +Hardening progress, 2026-08-20 (continued): source hydration now starts only +after the read unit of work has closed, and a barrier test proves a separate +unit of work can append to the claimed run while object hydration is blocked. +Bounded SQL Hypothesis tests now cover event sequences, paging, +principal-scoped idempotency, claim races, and lease recovery. Launcher +lifecycle generation covers successful completion, provider failure, and +shutdown cancellation. Full repository validation remains pending. + +Hardening completed, 2026-08-20: the remaining PR #141 review findings are +implemented without suppressions. The production composition root now requires +`API_AUTHORIZATION_BEARER_TOKEN` and `API_AUTHORIZATION_PRINCIPAL_ID`, injects +`StaticBearerTokenAuthorization`, persists the trusted principal as the +ingestion-job owner and generation-run actor, and returns the existing +non-disclosing denial response for cross-principal job, run, event, and TEI +access. The application persistence service no longer imports SQLAlchemy or +interprets driver errors: the source-document repository uses a savepoint to +return `SourceDocumentProjectionResult.DUPLICATE` only for its recognized +constraint, while unrelated integrity failures continue to escape unchanged. + +GET resources now emit one bounded `RecordingTracer`-testable span per request: +`generation_run.read`, `generation_run.events.list`, and `episode_tei.read`. +They record only operation, outcome, representation, pagination presence, and +stable failure categories. Launcher source material is bounded by configurable +maximum count, per-source bytes, aggregate bytes, and normalized text bytes; +uploaded streams short-circuit on limit breaches, and filesystem reads are +offloaded with `asyncio.to_thread`. The claim linearization transaction commits +the running transition and `run.started` before all metadata and object-store +hydration, proven with a synchronized independent-unit-of-work test. + +Validation evidence, 2026-08-20: focused API, runtime, SQL property, source +limit, persistence, and launcher suites passed (`36 passed` for the final +structural focused run; prior hardening-focused run passed `83`). +`make check-fmt`, `make typecheck`, and `make lint` passed. `make test` passed +`1,171` tests with one skipped test and 49 snapshots. `make markdownlint`, +`make nixie`, and `make check-migrations` passed; the migration check reported +only the pre-existing SQLAlchemy cycle warning for `episodes`, +`generation_runs`, and `ingestion_jobs`. Commit +`ba61dbad5e2fa8ec3a19b381a93a89227681bd46` records the hardening work and was +pushed to `origin/4-3-2-no-qa-generation-runs-and-tei-p5-retrieval`. + +Post-review maintenance, 2026-08-22: split cohesive source-intake, TEI tracing, +idempotency, and persistence test concerns into focused modules so every module +remains within the 400-line repository limit. The persistence module retains +its public-boundary documentation and behaviour. Validation passed: +`make check-fmt`, `make lint`, `make typecheck`, `make markdownlint`, and +`make nixie`; focused extracted suites passed 42 tests; and `make test` passed +1,214 tests with one skipped test and 49 snapshots. diff --git a/docs/repository-layout.md b/docs/repository-layout.md index 52cb3c1d..79312c31 100644 --- a/docs/repository-layout.md +++ b/docs/repository-layout.md @@ -72,7 +72,8 @@ The `episodic/` package is grouped by feature and boundary: adapters, pagination, ingestion services, profile templates, and reference document functionality. - `episodic/generation/` contains generation services for show notes, chapter - markers, guest biographies, and Text Encoding Initiative (TEI) payloads. + markers, guest biographies, draft scripts, generation-run launching, and Text + Encoding Initiative (TEI) payloads. - `episodic/llm/` contains large language model ports and OpenAI adapter code. - `episodic/orchestration/` contains workflow state, checkpoint payloads, suspend-and-resume support, and executor integration. @@ -110,6 +111,8 @@ The `tests/` directory mirrors the system's test surfaces: - `tests/features/` stores behavioural feature files. - `tests/steps/` stores pytest-bdd step implementations. +- `tests/steps/no_qa_generation_slice_support.py` owns the live Vidai Mock and + application-stack support for the source-to-script behavioural slice. - `tests/fixtures/` stores shared fixture data and architecture-check fixtures. - `tests/canonical_storage/` stores persistence-focused tests for canonical storage behaviour. diff --git a/docs/roadmap.md b/docs/roadmap.md index 4998c38f..cd9137b4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -675,11 +675,11 @@ vertical slice]. - Success: a client can upload a research paper, attach it to an ingestion job, create or bind presenter profile revisions, and observe JSON status until the source context is ready for generation. -- [ ] 4.3.2. Implement no-QA generation runs and TEI-P5 retrieval. +- [x] 4.3.2. Implement no-QA generation runs and TEI-P5 retrieval. - Requires 2.1.1, 2.4.2, and 4.3.1. - - Implement `/v1/episodes/{episode_id}/generation-runs` creation with - `quality_mode=draft_without_qa`, `skip_qa_rationale`, actor metadata, and - `Idempotency-Key` enforcement. + - Implement `/v1/ingestion-jobs/{ingestion_job_id}/generation-runs` creation + for a ready ingestion job with `quality_mode=draft_without_qa`, + `skip_qa_rationale`, actor metadata, and `Idempotency-Key` enforcement. - Implement `/v1/generation-runs/{run_id}` and `/v1/generation-runs/{run_id}/events` as JSON polling surfaces for the draft generation lifecycle. diff --git a/docs/users-guide.md b/docs/users-guide.md index fb22794a..905e1c56 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -86,6 +86,44 @@ different canonical body returns `409 Conflict`. The resumable `POST /v1/uploads/init` flow remains a future extension for an object-store adapter that can use pre-signed upload URLs. +#### Generate and download a no-QA draft + +Before using these endpoints, apply the latest Alembic migrations to the +service database with `alembic upgrade head`. The application does not apply +schema migrations during startup. + +Create a draft run for a ready ingestion job with +`POST /v1/ingestion-jobs/{ingestion_job_id}/generation-runs`. Supply an +`Idempotency-Key` and a JSON body containing `quality_mode` set to +`draft_without_qa` and `skip_qa_rationale`. The server derives the run actor +from the authenticated principal rather than the request body. It returns +`202 Accepted`, a `Location` header for the run, and `Retry-After` guidance. + +Poll the `Location` resource until its status is `succeeded` or `failed`. The +`202 Accepted` representation includes `episode_id`, and the polled run +resource returns it as well. Read `episode_id` from the polled run resource +before requesting the episode TEI. Lifecycle details are available from +`GET /v1/generation-runs/{run_id}/events`. This endpoint uses the `after_seq` +cursor and a `limit` cap; it returns `items`, `after_seq`, `limit`, `offset`, +and `total`. Do not combine a non-zero `offset` with `after_seq`: that request +is rejected as invalid. This cursor contract is distinct from the general +offset-pagination guidance. Replaying the original request with the same key +and body returns the same run and polling headers; changing the body under that +key returns `409 Conflict`. + +Generation applies four positive source limits before a draft-provider request: +`GENERATION_MAX_SOURCE_COUNT` (default 32), `GENERATION_MAX_SOURCE_BYTES` +(default 1 MiB), `GENERATION_MAX_AGGREGATE_SOURCE_BYTES` (default 8 MiB), and +`GENERATION_MAX_NORMALIZED_SOURCE_BYTES` (default 1 MiB). A source exceeding a +limit fails its run with the stable `generation.source_limit` category; source +content is never included in the response or logs. + +After success, request `GET /v1/episodes/{episode_id}/tei` for the JSON +metadata envelope, or add `Accept: application/tei+xml` to download the raw TEI +file. The response includes an entity tag and attachment filename. This path +deliberately bypasses QA: the run and TEI revision are marked `skipped`, and +the output remains an editorial draft until a later review and approval flow. + #### Show notes and chapter markers Show notes are the episode summaries and topic lists that appear alongside a @@ -125,6 +163,14 @@ Chapter markers currently use the configured execution model. A dedicated `chapter_marker_model` setting is planned for a future release, but is not a live configuration option yet. +The HTTP runtime also accepts these optional environment settings for generated +draft output. Each value must be a positive integer: + +- `GENERATION_MAX_OUTPUT_TOKENS` defaults to `4096` and caps the output tokens + passed to the LLM request. +- `GENERATION_MAX_RESPONSE_BYTES` defaults to `1048576` and caps the UTF-8 + response before generated JSON parsing. + #### Resumable orchestration Generation workflows now persist an internal checkpoint before a suspendable @@ -297,6 +343,29 @@ Required environment: `postgresql://...` and normalizes it to the supported async driver automatically. Driver-qualified URLs such as `postgresql+asyncpg://...` and `postgresql+psycopg://...` are also accepted. +- `API_AUTHORIZATION_BEARER_TOKEN` is the bearer token accepted for production + `/v1` requests. +- `API_AUTHORIZATION_PRINCIPAL_ID` identifies the principal associated with + that token. Both authorization settings are required before the service + starts. + +Production clients must include the configured token on every `/v1` request: + +```http +Authorization: Bearer +``` + +The authenticated principal owns the ingestion jobs and generation runs it can +access. Requests for resources owned by another principal use the same +not-found response as requests for resources that do not exist. Health +endpoints are not covered by this `/v1` authorization requirement. + +### Migration note for the next minor release + +Before upgrading to the next minor release, apply the new Alembic revisions. +Production API clients must configure the bearer token and principal settings +above, send the bearer header on every `/v1` request, and adopt the generation +source limits and event cursor contract described in this guide. Health endpoints: diff --git a/episodic/api/app.py b/episodic/api/app.py index e8ffed19..0b8703c6 100644 --- a/episodic/api/app.py +++ b/episodic/api/app.py @@ -12,17 +12,22 @@ app = create_app(dependencies) # Returns a Falcon ASGI app with API routes. """ -import asyncio import typing as typ from falcon import asgi +from episodic.logging import get_logger, log_error + from .authorization import AuthorizationMiddleware from .errors import serialize_http_error from .resources import ( + EpisodeTeiResource, EpisodeTemplateHistoryResource, EpisodeTemplateResource, EpisodeTemplatesResource, + GenerationRunEventsResource, + GenerationRunResource, + GenerationRunsResource, HealthLiveResource, HealthReadyResource, IngestionJobResource, @@ -45,10 +50,98 @@ from .source_intake_support import UploadResourceConfig if typ.TYPE_CHECKING: + from episodic.observability import MetricsPort, MonotonicClockPort + from .dependencies import ApiDependencies, ShutdownHook from .types import UowFactory +logger = get_logger(__name__) + +_GENERATION_REQUEST_TOTAL = "generation_api_request_total" +_GENERATION_REQUEST_LATENCY = "generation_api_request_latency_ms" + + +def _generation_route_operation(path: str) -> str | None: + """Return a bounded operation name for an instrumented API path.""" + if path.startswith("/v1/ingestion-jobs/") and path.endswith("/generation-runs"): + return "generation_run.command" + if path.startswith("/v1/generation-runs/"): + return ( + "generation_run.events.list" + if path.endswith("/events") + else "generation_run.read" + ) + if path.startswith("/v1/episodes/") and path.endswith("/tei"): + return "episode_tei.read" + return None + + +class _GenerationRouteMetricsMiddleware: + """Emit bounded generation-route outcome and latency observations.""" + + def __init__( + self, + metrics: MetricsPort, + monotonic_clock: MonotonicClockPort, + ) -> None: + self._metrics = metrics + self._monotonic_clock = monotonic_clock + + async def process_request( + self, + req: asgi.Request, + resp: asgi.Response, + ) -> None: + """Start timing an instrumented generation route.""" + del resp + operation = _generation_route_operation(req.path) + if operation is not None: + req.context["generation_api_metrics"] = ( + operation, + self._monotonic_clock.monotonic_seconds(), + ) + + async def process_response( + self, + req: asgi.Request, + resp: asgi.Response, + resource: object, + req_succeeded: bool, # noqa: FBT001 # Falcon ASGI middleware contract. + ) -> None: + """Record the completed response without retaining request data.""" + del resource, req_succeeded + observation = req.context.get("generation_api_metrics") + if not isinstance(observation, tuple): + return + operation, started_at = typ.cast( + "tuple[str, float]", + observation, + ) + if not isinstance(operation, str) or not isinstance(started_at, float): + return + outcome = _response_outcome(resp.status) + labels = {"operation": operation, "outcome": outcome} + self._metrics.increment_counter(_GENERATION_REQUEST_TOTAL, labels=labels) + self._metrics.observe_latency_ms( + _GENERATION_REQUEST_LATENCY, + (self._monotonic_clock.monotonic_seconds() - started_at) * 1_000, + labels=labels, + ) + + +def _response_outcome(status: str | int) -> str: + """Map an HTTP response status to a bounded metric outcome.""" + status = str(status) + if status.startswith("304"): + return "not_modified" + if status.startswith("2"): + return "success" + if status.startswith("4"): + return "rejected" + return "failed" + + class _ShutdownHooksMiddleware: """Run injected async cleanup hooks during the ASGI shutdown phase.""" @@ -62,9 +155,16 @@ async def process_shutdown( ) -> None: """Release runtime-managed resources before the process exits.""" del scope, event - await asyncio.gather( - *(shutdown_hook() for shutdown_hook in self._shutdown_hooks) - ) + first_failure: Exception | None = None + for shutdown_hook in self._shutdown_hooks: + try: + await shutdown_hook() + except Exception as exc: # noqa: BLE001 - cleanup must attempt every hook. + log_error(logger, "ASGI shutdown hook failed.", exc_info=True) + if first_failure is None: + first_failure = exc + if first_failure is not None: + raise first_failure def _register_health_routes(app: asgi.App, dependencies: ApiDependencies) -> None: @@ -152,19 +252,64 @@ def _register_intake_routes( app.add_route("/v1/uploads/{upload_id}", UploadResource(uow_factory)) app.add_route("/v1/ingestion-jobs", IngestionJobsResource(uow_factory)) app.add_route( - "/v1/ingestion-jobs/{job_id}", + "/v1/ingestion-jobs/{ingestion_job_id}", IngestionJobResource(uow_factory), ) app.add_route( - "/v1/ingestion-jobs/{job_id}/sources", + "/v1/ingestion-jobs/{ingestion_job_id}/sources", IngestionJobSourcesResource(uow_factory), ) +def _register_generation_run_routes( + app: asgi.App, + uow_factory: UowFactory, + dependencies: ApiDependencies, +) -> None: + app.add_route( + "/v1/ingestion-jobs/{ingestion_job_id}/generation-runs", + GenerationRunsResource( + uow_factory, + launcher=dependencies.launcher, + max_source_count=( + None + if dependencies.generation_source_limits is None + else dependencies.generation_source_limits.max_source_count + ), + tracer=dependencies.tracer, + ), + ) + app.add_route( + "/v1/generation-runs/{run_id}", + GenerationRunResource(uow_factory, tracer=dependencies.tracer), + ) + app.add_route( + "/v1/generation-runs/{run_id}/events", + GenerationRunEventsResource(uow_factory, tracer=dependencies.tracer), + ) + + +def _register_episode_tei_routes( + app: asgi.App, + uow_factory: UowFactory, + dependencies: ApiDependencies, +) -> None: + app.add_route( + "/v1/episodes/{episode_id}/tei", + EpisodeTeiResource(uow_factory, tracer=dependencies.tracer), + ) + + def create_app(dependencies: ApiDependencies) -> asgi.App: """Build and return Falcon ASGI application for canonical APIs.""" app = asgi.App() app.add_middleware(AuthorizationMiddleware(dependencies.authorization)) + app.add_middleware( + _GenerationRouteMetricsMiddleware( + dependencies.metrics, + dependencies.monotonic_clock, + ) + ) if dependencies.shutdown_hooks: # Falcon supports lifespan middleware at runtime, but its exported # middleware type union does not model process_shutdown-only hooks. @@ -181,5 +326,7 @@ def create_app(dependencies: ApiDependencies) -> asgi.App: _register_reference_document_routes(app, uow_factory) _register_reference_binding_routes(app, uow_factory) _register_intake_routes(app, uow_factory, dependencies) + _register_generation_run_routes(app, uow_factory, dependencies) + _register_episode_tei_routes(app, uow_factory, dependencies) return app diff --git a/episodic/api/authorization.py b/episodic/api/authorization.py index f08a08f3..ecff80c7 100644 --- a/episodic/api/authorization.py +++ b/episodic/api/authorization.py @@ -2,6 +2,7 @@ import dataclasses as dc import enum +import hmac import typing as typ import falcon @@ -60,6 +61,34 @@ async def decide( # noqa: PLR6301 - must match AuthorizationPort instance metho return AuthorizationResult(AuthorizationDecision.PERMIT) +@dc.dataclass(frozen=True, slots=True) +class StaticBearerTokenAuthorization: + """Authorize one configured bearer token as one configured principal.""" + + token: str + principal_id: str + + async def decide( + self, + context: AuthorizationContext, + ) -> AuthorizationResult: + """Return the configured principal for a matching bearer token.""" + header = context.authorization_header + if header is None: + return AuthorizationResult(AuthorizationDecision.UNAUTHORIZED) + scheme, separator, token = header.partition(" ") + if not separator or not token: + return AuthorizationResult(AuthorizationDecision.UNAUTHORIZED) + if scheme.casefold() != "bearer": + return AuthorizationResult(AuthorizationDecision.UNAUTHORIZED) + if not hmac.compare_digest(token, self.token): + return AuthorizationResult(AuthorizationDecision.UNAUTHORIZED) + return AuthorizationResult( + AuthorizationDecision.PERMIT, + principal_id=self.principal_id, + ) + + class AuthorizationMiddleware: """Falcon middleware that short-circuits unauthorized `/v1` requests.""" diff --git a/episodic/api/dependencies.py b/episodic/api/dependencies.py index 2779714d..10afeb9a 100644 --- a/episodic/api/dependencies.py +++ b/episodic/api/dependencies.py @@ -10,12 +10,16 @@ import inspect import typing as typ +from episodic.observability import NoopMetrics, NoopTracer, PerfCounterClock + from .authorization import AuthorizationPort, PermitAll if typ.TYPE_CHECKING: from episodic.canonical.health import HealthObserver from episodic.canonical.object_store import ObjectStorePort + from episodic.generation import GenerationRunLauncher, GenerationSourceLimits from episodic.llm import LLMPort + from episodic.observability import MetricsPort, MonotonicClockPort, TracerPort from .types import UowFactory @@ -97,6 +101,11 @@ class ApiDependencies: health_observer: HealthObserver | None = None shutdown_hooks: tuple[ShutdownHook, ...] = () llm_port: LLMPort | None = None + launcher: GenerationRunLauncher | None = None + generation_source_limits: GenerationSourceLimits | None = None + metrics: MetricsPort = dc.field(default_factory=NoopMetrics) + monotonic_clock: MonotonicClockPort = dc.field(default_factory=PerfCounterClock) + tracer: TracerPort = dc.field(default_factory=NoopTracer) authorization: AuthorizationPort = dc.field(default_factory=PermitAll) def __post_init__(self) -> None: diff --git a/episodic/api/resources/__init__.py b/episodic/api/resources/__init__.py index 13c40611..49241853 100644 --- a/episodic/api/resources/__init__.py +++ b/episodic/api/resources/__init__.py @@ -31,11 +31,17 @@ """ from .base import _GetHistoryResourceBase, _GetResourceBase +from .episode_tei import EpisodeTeiResource from .episode_templates import ( EpisodeTemplateHistoryResource, EpisodeTemplateResource, EpisodeTemplatesResource, ) +from .generation_runs import ( + GenerationRunEventsResource, + GenerationRunResource, + GenerationRunsResource, +) from .health import HealthLiveResource, HealthReadyResource from .reference_bindings import ReferenceBindingResource, ReferenceBindingsResource from .reference_documents import ( @@ -60,9 +66,13 @@ ) __all__ = [ + "EpisodeTeiResource", "EpisodeTemplateHistoryResource", "EpisodeTemplateResource", "EpisodeTemplatesResource", + "GenerationRunEventsResource", + "GenerationRunResource", + "GenerationRunsResource", "HealthLiveResource", "HealthReadyResource", "IngestionJobResource", diff --git a/episodic/api/resources/episode_tei.py b/episodic/api/resources/episode_tei.py new file mode 100644 index 00000000..b494decb --- /dev/null +++ b/episodic/api/resources/episode_tei.py @@ -0,0 +1,239 @@ +"""Falcon resource and content negotiation for generated episode TEI.""" + +import hashlib +import json +import typing as typ + +import falcon + +from episodic.api.errors import http_error +from episodic.api.helpers import parse_uuid +from episodic.api.serializers import serialize_tei_envelope +from episodic.api.source_idempotency import principal_id +from episodic.observability import NoopTracer + +if typ.TYPE_CHECKING: + import uuid + + from episodic.api.types import UowFactory + from episodic.canonical.domain import CanonicalEpisode, GenerationRun + from episodic.observability import TracerPort + +_JSON_MEDIA_TYPE = "application/json" +_TEI_MEDIA_TYPE = "application/tei+xml" + + +class EpisodeTeiResource: + """Return generated episode TEI as metadata or an XML attachment. + + Parameters + ---------- + uow_factory : UowFactory + Callable dependency retained by the resource and invoked to construct + an asynchronous unit of work for each request. + """ + + def __init__( + self, uow_factory: UowFactory, *, tracer: TracerPort | None = None + ) -> None: + self._uow_factory = uow_factory + self._tracer = NoopTracer() if tracer is None else tracer + + async def on_get( + self, + req: falcon.Request, + resp: falcon.Response, + episode_id: str, + ) -> None: + """Return generated TEI using the requested representation. + + Parameters + ---------- + req : falcon.Request + Request whose ``Accept`` and ``If-None-Match`` headers select and + condition the representation. + resp : falcon.Response + Response to populate with JSON metadata or an XML attachment. + episode_id : str + Episode UUID from the route path. + + Raises + ------ + falcon.HTTPBadRequest + If ``episode_id`` is not a valid UUID (HTTP 400). + falcon.HTTPNotFound + If the episode is absent or has no generated TEI (HTTP 404). + falcon.HTTPNotAcceptable + If ``Accept`` excludes both supported media types (HTTP 406). + + Notes + ----- + JSON metadata (``application/json``) is the default representation. + ``application/tei+xml`` selects the generated XML attachment. A + matching or wildcard ``If-None-Match`` validator returns HTTP 304 with + no response body. + """ # noqa: DOC501, DOC502 # Indirect exceptions form part of this public contract. + with self._tracer.start_span( + "episode_tei.read", + attributes={"operation": "episode_tei.read"}, + ) as span: + try: + parsed_episode_id = parse_uuid(episode_id, "episode_id") + except falcon.HTTPError: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "invalid_input") + raise + async with self._uow_factory() as uow: + episode = await uow.episodes.get(parsed_episode_id) + run = ( + None + if episode is None or episode.last_generation_run_id is None + else await uow.generation_runs.get_run( + episode.last_generation_run_id + ) + ) + actor = principal_id(req) + if actor is None: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "episode.not_found") + raise _tei_not_found(parsed_episode_id) + if not _has_accessible_draft(episode, run, actor): + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "episode.not_found") + raise _tei_not_found(parsed_episode_id) + episode = typ.cast("CanonicalEpisode", episode) + try: + media_type = negotiate_tei_media_type(req.accept) + except falcon.HTTPNotAcceptable: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "not_acceptable") + raise + span.set_attribute("representation", media_type) + if media_type == _TEI_MEDIA_TYPE: + _apply_tei_attachment(req, resp, episode) + else: + _apply_tei_json(req, resp, episode) + span.set_attribute("outcome", "success") + + +def negotiate_tei_media_type(accept: str | None) -> str: + """Choose JSON metadata or raw TEI XML from an HTTP Accept header. + + Parameters + ---------- + accept + Raw value of the request ``Accept`` header. Missing or blank values + select JSON metadata. + + Returns + ------- + str + ``application/json`` or ``application/tei+xml``, chosen by the + highest acceptable quality value. + + Raises + ------ + falcon.HTTPNotAcceptable + Raised when neither supported representation has a positive quality. + """ # noqa: DOC502 - http_error() preserves the concrete Falcon exception. + if accept is None or not accept.strip(): + return _JSON_MEDIA_TYPE + tei_quality = falcon.mediatypes.quality(_TEI_MEDIA_TYPE, accept) + json_quality = falcon.mediatypes.quality(_JSON_MEDIA_TYPE, accept) + if tei_quality > 0 and tei_quality > json_quality: + return _TEI_MEDIA_TYPE + if json_quality > 0: + return _JSON_MEDIA_TYPE + error = falcon.HTTPNotAcceptable( + description="Accept must allow application/json or application/tei+xml." + ) + http_error( + error, + code="not_acceptable", + details={"supported": [_JSON_MEDIA_TYPE, _TEI_MEDIA_TYPE]}, + ) + raise error + + +def _has_generated_draft(episode: CanonicalEpisode) -> bool: + return ( + episode.last_generation_run_id is not None + and episode.tei_content_hash is not None + and episode.qa_status is not None + ) + + +def _has_accessible_draft( + episode: CanonicalEpisode | None, + run: GenerationRun | None, + actor: str, +) -> bool: + """Return whether the requested actor can retrieve this generated draft.""" + return ( + episode is not None + and _has_generated_draft(episode) + and run is not None + and run.actor == actor + ) + + +def _apply_tei_json( + req: falcon.Request, + resp: falcon.Response, + episode: CanonicalEpisode, +) -> None: + content = json.dumps(serialize_tei_envelope(episode), ensure_ascii=False).encode() + _apply_representation(req, resp, content=content, media_type=_JSON_MEDIA_TYPE) + + +def _apply_tei_attachment( + req: falcon.Request, + resp: falcon.Response, + episode: CanonicalEpisode, +) -> None: + resp.set_header( + "Content-Disposition", + f'attachment; filename="episode-{episode.id}.xml"', + ) + _apply_representation( + req, + resp, + content=episode.tei_xml.encode(), + media_type=_TEI_MEDIA_TYPE, + ) + + +def _apply_representation( + req: falcon.Request, + resp: falcon.Response, + *, + content: bytes, + media_type: str, +) -> None: + """Set one TEI representation or its conditional ``304`` response.""" + etag = _representation_etag(content) + resp.set_header("ETag", f'"{etag}"') + if any(validator in {"*", etag} for validator in req.if_none_match or []): + resp.status = falcon.HTTP_304 + return + resp.status = falcon.HTTP_200 + resp.content_type = media_type + resp.data = content + + +def _representation_etag(content: bytes) -> str: + """Return the strong ETag value for serialized representation bytes.""" + return hashlib.sha256(content).hexdigest() + + +def _tei_not_found(episode_id: uuid.UUID) -> falcon.HTTPNotFound: + return typ.cast( + "falcon.HTTPNotFound", + http_error( + falcon.HTTPNotFound( + description=f"Generated TEI not found for episode: {episode_id}." + ), + code="episode_tei_not_found", + details={"episode_id": str(episode_id)}, + ), + ) diff --git a/episodic/api/resources/generation_runs.py b/episodic/api/resources/generation_runs.py new file mode 100644 index 00000000..32a0f94e --- /dev/null +++ b/episodic/api/resources/generation_runs.py @@ -0,0 +1,558 @@ +"""Falcon resources for no-QA generation-run creation and polling.""" + +import datetime as dt +import typing as typ +import uuid + +import falcon + +from episodic.api.errors import http_error, map_source_intake_error, validation_error +from episodic.api.helpers import parse_uuid, require_payload_dict +from episodic.api.serializers import ( + serialize_generation_event, + serialize_generation_run, +) +from episodic.api.source_idempotency import ( + IdempotencyContext, + IdempotentResponse, + apply_response, + principal_id, + run_idempotent, +) +from episodic.api.source_intake_support import json_body_hash, require_str +from episodic.canonical.domain import GenerationRun, GenerationRunStatus +from episodic.canonical.generation_persistence import ( + DraftScriptPersistenceError, + EpisodeMaterialisationRequest, + materialise_episode_from_ingestion, +) +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.generation_run_errors import RunAlreadyTerminal, RunNotFound +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate, event_seq +from episodic.canonical.source_intake_service import SourceIntakeError +from episodic.generation.launcher import GenerationRunAdmissionError +from episodic.observability import NoopTracer + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from episodic.api.types import JsonPayload, UowFactory + from episodic.canonical.domain import IngestionJob + from episodic.generation.launcher import GenerationRunLauncher + from episodic.observability import TracerPort + +_GENERATION_RUN_OPERATION = "generation_run.create" +_RETRY_AFTER = "1" +_MAX_EVENT_LIMIT = 100 +_DEFAULT_EVENT_LIMIT = 20 + +type Clock = cabc.Callable[[], dt.datetime] +type UuidFactory = cabc.Callable[[], uuid.UUID] + + +def _utc_now() -> dt.datetime: + """Return the current UTC timestamp.""" + return dt.datetime.now(dt.UTC) + + +def _uuid7() -> uuid.UUID: + """Return a time-ordered UUID.""" + return uuid.uuid7() + + +class GenerationRunsResource: + """Create no-QA generation runs for authenticated ingestion-job owners. + + The resource materialises the caller-owned ready ingestion job, persists a + durable generation-run checkpoint, and schedules detached execution. The + authenticated principal becomes the durable run actor; request payloads + cannot select another actor. + """ + + def __init__( # noqa: PLR0913 # HTTP composition requires independent test seams. + self, + uow_factory: UowFactory, + *, + launcher: GenerationRunLauncher | None, + clock: Clock = _utc_now, + uuid_factory: UuidFactory = _uuid7, + max_source_count: int | None = None, + tracer: TracerPort | None = None, + ) -> None: + self._uow_factory = uow_factory + self._launcher = launcher + self._clock = clock + self._uuid_factory = uuid_factory + self._max_source_count = max_source_count + self._tracer = NoopTracer() if tracer is None else tracer + + async def on_post( + self, + req: falcon.Request, + resp: falcon.Response, + ingestion_job_id: str, + ) -> None: + """Materialise and schedule a no-QA generation run. + + Parameters + ---------- + ingestion_job_id + Path identifier for the ready ingestion job owned by the + authenticated principal. + req + Falcon request containing the no-QA command and idempotency key. + resp + Falcon response populated with the accepted run representation. + + Raises + ------ + _ingestion_job_not_found + If the caller has no access to the requested ingestion job. + + Notes + ----- + ``Idempotency-Key`` is required. Replaying the same key for the same + authenticated principal returns the original accepted response. + Malformed input, unknown or inaccessible jobs, invalid source + materialisation, unavailable launcher configuration, idempotency + conflicts, and bounded-admission rejection use canonical HTTP error + responses. + """ + source_bundle_id = parse_uuid(ingestion_job_id, "ingestion_job_id") + payload = require_payload_dict(await req.get_media()) + request = _parse_create_request(payload) + launcher = self._require_launcher() + idempotency_key = req.get_header("Idempotency-Key") + actor = principal_id(req) + if actor is None: + raise _ingestion_job_not_found(source_bundle_id) + + async def work() -> IdempotentResponse: + run = await self._create_run( + source_bundle_id, + request, + actor=actor, + idempotency_key=idempotency_key, + ) + span.set_attribute("run_id", str(run.id)) + try: + await launcher.launch(run.id) + except GenerationRunAdmissionError as exc: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "launcher.overloaded") + await self._mark_launch_failed( + run.id, + error_message=str(exc), + error_category="launcher.overloaded", + ) + raise _generation_overloaded() from exc + except Exception as exc: + span.set_attribute("outcome", "failed") + span.set_attribute("failure_category", "launcher.scheduling") + await self._mark_launch_failed( + run.id, + error_message=str(exc), + error_category="launcher.scheduling", + ) + raise + location = f"/v1/generation-runs/{run.id}" + return IdempotentResponse( + falcon.HTTP_202, + serialize_generation_run(run), + location=location, + retry_after=_RETRY_AFTER, + ) + + with self._tracer.start_span( + "generation_run.command", + attributes={"operation": _GENERATION_RUN_OPERATION}, + ) as span: + result = await run_idempotent( + self._uow_factory, + context=IdempotencyContext( + req=req, + operation=_GENERATION_RUN_OPERATION, + body_hash=json_body_hash(payload), + ), + work=work, + ) + span.set_attribute("outcome", "accepted") + apply_response(resp, result) + + def _require_launcher(self) -> GenerationRunLauncher: + if self._launcher is None: + raise http_error( + falcon.HTTPServiceUnavailable( + description="Generation launcher is not configured." + ), + code="service_unavailable", + ) + return self._launcher + + async def _create_run( + self, + source_bundle_id: uuid.UUID, + request: _CreateGenerationRun, + *, + actor: str, + idempotency_key: str | None, + ) -> GenerationRun: + async with self._uow_factory() as uow: + job = await uow.ingestion_jobs.get(source_bundle_id) + if job is None or not _owns_ingestion_job(job, actor): + raise _ingestion_job_not_found(source_bundle_id) + try: + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=source_bundle_id, + title=f"Episode {source_bundle_id}", + clock=self._clock, + uuid_factory=self._uuid_factory, + max_source_count=( + 32 + if self._max_source_count is None + else self._max_source_count + ), + ), + ) + except SourceIntakeError as exc: + raise map_source_intake_error(exc) from exc + except DraftScriptPersistenceError as exc: + raise _generation_input_error(str(exc)) from exc + now = self._clock() + run = GenerationRun( + id=self._uuid_factory(), + episode_id=episode.id, + source_bundle_id=source_bundle_id, + actor=actor, + status=GenerationRunStatus.PENDING, + current_node=None, + budget_snapshot=request.budget_snapshot, + configuration=request.configuration, + created_at=now, + updated_at=now, + started_at=None, + ended_at=None, + error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale=request.skip_qa_rationale, + ) + run = await uow.generation_runs.create_run( + run, + idempotency_key=idempotency_key, + idempotency_principal_id=actor, + ) + await uow.commit() + return run + + async def _mark_launch_failed( + self, + run_id: uuid.UUID, + *, + error_message: str, + error_category: str, + ) -> None: + now = self._clock() + async with self._uow_factory() as uow: + try: + await uow.generation_runs.update_run_status( + run_id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.FAILED, + current_node=None, + ended_at=now, + error_message=error_message, + error_category=error_category, + ), + ) + except RunAlreadyTerminal, RunNotFound: + await uow.rollback() + else: + await uow.commit() + + +def _generation_overloaded() -> falcon.HTTPServiceUnavailable: + """Build the stable response for a rejected in-process launch.""" + return typ.cast( + "falcon.HTTPServiceUnavailable", + http_error( + falcon.HTTPServiceUnavailable( + description="Generation capacity is temporarily exhausted." + ), + code="generation_overloaded", + ), + ) + + +class GenerationRunResource: + """Return one generation-run polling snapshot.""" + + def __init__( + self, uow_factory: UowFactory, *, tracer: TracerPort | None = None + ) -> None: + self._uow_factory = uow_factory + self._tracer = NoopTracer() if tracer is None else tracer + + async def on_get( + self, + req: falcon.Request, + resp: falcon.Response, + run_id: str, + ) -> None: + """Return the current generation-run state.""" + with self._tracer.start_span( + "generation_run.read", + attributes={"operation": "generation_run.read"}, + ) as span: + try: + parsed_run_id = parse_uuid(run_id, "run_id") + except falcon.HTTPError: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "invalid_input") + raise + actor = principal_id(req) + if actor is None: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) + async with self._uow_factory() as uow: + run = await uow.generation_runs.get_run(parsed_run_id) + if run is None: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) + if run.actor != actor: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) + resp.media = serialize_generation_run(run) + resp.status = falcon.HTTP_200 + if not run.status.is_terminal(): + resp.set_header("Retry-After", _RETRY_AFTER) + span.set_attribute("outcome", "success") + + +class GenerationRunEventsResource: + """Return cursor-paginated events for one generation run.""" + + def __init__( + self, uow_factory: UowFactory, *, tracer: TracerPort | None = None + ) -> None: + self._uow_factory = uow_factory + self._tracer = NoopTracer() if tracer is None else tracer + + async def on_get( + self, + req: falcon.Request, + resp: falcon.Response, + run_id: str, + ) -> None: + """List events after an optional sequence cursor.""" + with self._tracer.start_span( + "generation_run.events.list", + attributes={"operation": "generation_run.events.list"}, + ) as span: + try: + parsed_run_id = parse_uuid(run_id, "run_id") + after_seq = _parse_optional_positive_int(req, "after_seq") + limit = _parse_limit(req) + offset = _parse_offset(req) + if after_seq is not None and offset != 0: + message = "after_seq and offset cannot be combined." + raise validation_error( + message, + field="offset", + constraint="exclusive_with_after_seq", + ) + except falcon.HTTPError: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "invalid_input") + raise + span.set_attribute( + "pagination", "cursor" if after_seq is not None else "offset" + ) + actor = principal_id(req) + if actor is None: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) + async with self._uow_factory() as uow: + run = await uow.generation_runs.get_run(parsed_run_id) + if run is None or run.actor != actor: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) + try: + events = await uow.generation_runs.list_events( + parsed_run_id, + after_seq=None if after_seq is None else event_seq(after_seq), + limit=limit, + offset=offset, + ) + total = await uow.generation_runs.count_events( + parsed_run_id, + after_seq=None if after_seq is None else event_seq(after_seq), + ) + except RunNotFound as exc: + span.set_attribute("outcome", "not_found") + span.set_attribute("failure_category", "run.not_found") + raise _run_not_found(parsed_run_id) from exc + resp.media = { + "items": [serialize_generation_event(event) for event in events], + "after_seq": after_seq, + "limit": limit, + "offset": offset, + "total": total, + } + resp.status = falcon.HTTP_200 + span.set_attribute("outcome", "success") + + +class _CreateGenerationRun(typ.NamedTuple): + skip_qa_rationale: str + configuration: JsonPayload + budget_snapshot: JsonPayload + + +def _parse_create_request(payload: JsonPayload) -> _CreateGenerationRun: + quality_mode = require_str(payload, "quality_mode") + if quality_mode == "qa_gated": + raise typ.cast( + "falcon.HTTPUnprocessableEntity", + http_error( + falcon.HTTPUnprocessableEntity( + description=f"Unsupported quality_mode: {quality_mode}." + ), + code="quality_mode_unsupported", + details={"quality_mode": quality_mode}, + ), + ) + try: + parsed_mode = QualityMode(quality_mode) + except ValueError as exc: + message = f"Invalid quality_mode: {quality_mode!r}." + raise validation_error( + message, + field="quality_mode", + constraint="enum", + ) from exc + typ.assert_type(parsed_mode, QualityMode) + configuration = { + key: payload[key] + for key in ("template_id", "prompt_overrides") + if key in payload + } + budget_snapshot = _optional_mapping(payload, "budget_hints") + return _CreateGenerationRun( + skip_qa_rationale=require_str(payload, "skip_qa_rationale"), + configuration=configuration, + budget_snapshot=budget_snapshot, + ) + + +def _optional_mapping(payload: JsonPayload, field_name: str) -> JsonPayload: + value = payload.get(field_name, {}) + if not isinstance(value, dict): + message = f"{field_name} must be a JSON object." + raise validation_error( + message, + field=field_name, + constraint="object", + ) + return typ.cast("JsonPayload", value) + + +def _parse_optional_positive_int(req: falcon.Request, name: str) -> int | None: + raw = req.get_param(name) + if raw is None: + return None + value = _parse_int(raw, name) + if value < 1: + message = f"{name} must be a positive integer." + raise validation_error( + message, + field=name, + constraint="range", + ) + return value + + +def _parse_limit(req: falcon.Request) -> int: + raw = req.get_param("limit") + value = _DEFAULT_EVENT_LIMIT if raw is None else _parse_int(raw, "limit") + if value < 1 or value > _MAX_EVENT_LIMIT: + message = f"limit must be between 1 and {_MAX_EVENT_LIMIT}." + raise validation_error( + message, + field="limit", + constraint="range", + ) + return value + + +def _parse_offset(req: falcon.Request) -> int: + """Parse the event collection offset.""" + raw = req.get_param("offset") + value = 0 if raw is None else _parse_int(raw, "offset") + if value < 0: + message = "offset must be a non-negative integer." + raise validation_error( + message, + field="offset", + constraint="range", + ) + return value + + +def _parse_int(raw: str, name: str) -> int: + try: + return int(raw) + except ValueError as exc: + message = f"{name} must be an integer." + raise validation_error( + message, + field=name, + constraint="type", + ) from exc + + +def _run_not_found(run_id: uuid.UUID) -> falcon.HTTPNotFound: + return typ.cast( + "falcon.HTTPNotFound", + http_error( + falcon.HTTPNotFound(description=f"Generation run not found: {run_id}."), + code="generation_run_not_found", + details={"run_id": str(run_id)}, + ), + ) + + +def _ingestion_job_not_found(ingestion_job_id: uuid.UUID) -> falcon.HTTPNotFound: + """Build a non-disclosing response for an inaccessible ingestion job.""" + return typ.cast( + "falcon.HTTPNotFound", + http_error( + falcon.HTTPNotFound( + description=f"Ingestion job not found: {ingestion_job_id}." + ), + code="ingestion_job_not_found", + details={"ingestion_job_id": str(ingestion_job_id)}, + ), + ) + + +def _owns_ingestion_job(job: IngestionJob, principal: str | None) -> bool: + """Return whether the server-derived principal can use an ingestion job.""" + return principal is not None and job.owner_principal_id == principal + + +def _generation_input_error(message: str) -> falcon.HTTPUnprocessableEntity: + return typ.cast( + "falcon.HTTPUnprocessableEntity", + http_error( + falcon.HTTPUnprocessableEntity(description=message), + code="generation_input_invalid", + ), + ) diff --git a/episodic/api/resources/source_intake.py b/episodic/api/resources/source_intake.py index 85eda2db..63650501 100644 --- a/episodic/api/resources/source_intake.py +++ b/episodic/api/resources/source_intake.py @@ -32,10 +32,14 @@ parse_upload_form, require_str, ) -from episodic.canonical.domain import IngestionJobListFilters, IntakeState +from episodic.canonical.domain import IngestionJob, IngestionJobListFilters, IntakeState from episodic.canonical.idempotency_service import ( multipart_request_hash, ) +from episodic.canonical.source_intake_errors import ( + IngestionJobNotFoundError, + UploadNotFoundError, +) from episodic.canonical.source_intake_service import ( CreateIngestionJobRequest, SourceIntakeError, @@ -71,6 +75,7 @@ def __init__( async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: """Create one ready upload from multipart form data.""" + owner_principal_id = _require_principal(req) object_store = self._config.object_store if object_store is None: raise http_error( @@ -105,7 +110,7 @@ async def work() -> IdempotentResponse: self._uow_factory, object_store, UploadBytesRequest( - owner_principal_id=principal_id(req), + owner_principal_id=owner_principal_id, content_type=parsed.content_type, declared_size=parsed.declared_size, declared_sha256=parsed.declared_sha256, @@ -143,11 +148,11 @@ async def on_get( upload_id: str, ) -> None: """Return one upload metadata envelope.""" - del req parsed_upload_id = parse_uuid(upload_id, "upload_id") async with self._uow_factory() as uow: try: upload = await get_upload(uow, parsed_upload_id) + _require_upload_owner(upload.owner_principal_id, principal_id(req)) except SourceIntakeError as exc: raise map_source_intake_error(exc) from exc resp.media = serialize_upload(upload) @@ -162,12 +167,21 @@ def __init__(self, uow_factory: UowFactory) -> None: async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: """Create one intake-stage ingestion job.""" + owner_principal_id = _require_principal(req) payload = require_payload_dict(await req.get_media()) body_hash = json_body_hash(payload) series_profile_id = parse_uuid( require_str(payload, "series_profile_id"), "series_profile_id" ) target_episode_id = parse_optional_payload_uuid(payload, "target_episode_id") + if target_episode_id is not None: + raise http_error( + falcon.HTTPBadRequest( + description="target_episode_id is not accepted by this endpoint." + ), + code="validation_error", + details={"field": "target_episode_id", "constraint": "unsupported"}, + ) async def work() -> IdempotentResponse: async with self._uow_factory() as uow: @@ -177,6 +191,7 @@ async def work() -> IdempotentResponse: CreateIngestionJobRequest( series_profile_id=series_profile_id, target_episode_id=target_episode_id, + owner_principal_id=owner_principal_id, ), ) except SourceIntakeError as exc: @@ -196,6 +211,7 @@ async def work() -> IdempotentResponse: async def on_get(self, req: falcon.Request, resp: falcon.Response) -> None: """List intake-stage ingestion jobs.""" + owner_principal_id = _require_principal(req) pagination = parse_pagination(req) series_profile_id = parse_optional_uuid_param(req, "series_profile_id") intake_state = parse_enum_param(req, "intake_state", IntakeState) @@ -205,6 +221,7 @@ async def on_get(self, req: falcon.Request, resp: falcon.Response) -> None: IngestionJobListFilters( series_profile_id=series_profile_id, intake_state=intake_state, + owner_principal_id=owner_principal_id, ), pagination, ) @@ -227,14 +244,14 @@ async def on_get( self, req: falcon.Request, resp: falcon.Response, - job_id: str, + ingestion_job_id: str, ) -> None: """Return the current intake status for one ingestion job.""" - del req - parsed_job_id = parse_uuid(job_id, "job_id") + parsed_job_id = parse_uuid(ingestion_job_id, "ingestion_job_id") async with self._uow_factory() as uow: try: job = await get_ingestion_job_status(uow, parsed_job_id) + _require_ingestion_job_owner(job, principal_id(req)) except SourceIntakeError as exc: raise map_source_intake_error(exc) from exc next_poll = None @@ -257,10 +274,10 @@ async def on_post( self, req: falcon.Request, resp: falcon.Response, - job_id: str, + ingestion_job_id: str, ) -> None: """Attach one upload or remote URI source to an ingestion job.""" - parsed_job_id = parse_uuid(job_id, "job_id") + parsed_job_id = parse_uuid(ingestion_job_id, "ingestion_job_id") payload = require_payload_dict(await req.get_media()) body_hash = json_body_hash(payload) attach_request = build_attach_source_request(parsed_job_id, payload) @@ -268,6 +285,8 @@ async def on_post( async def work() -> IdempotentResponse: async with self._uow_factory() as uow: try: + job = await get_ingestion_job_status(uow, parsed_job_id) + _require_ingestion_job_owner(job, principal_id(req)) source = await attach_source_to_ingestion_job(uow, attach_request) except SourceIntakeError as exc: raise map_source_intake_error(exc) from exc @@ -291,13 +310,15 @@ async def on_get( self, req: falcon.Request, resp: falcon.Response, - job_id: str, + ingestion_job_id: str, ) -> None: """List source attachments for one ingestion job.""" - parsed_job_id = parse_uuid(job_id, "job_id") + parsed_job_id = parse_uuid(ingestion_job_id, "ingestion_job_id") pagination = parse_pagination(req) async with self._uow_factory() as uow: try: + job = await get_ingestion_job_status(uow, parsed_job_id) + _require_ingestion_job_owner(job, principal_id(req)) page = await list_ingestion_job_sources( uow, parsed_job_id, @@ -312,3 +333,33 @@ async def on_get( "total": page.total, } resp.status = falcon.HTTP_200 + + +def _require_ingestion_job_owner( + job: IngestionJob, + principal: str | None, +) -> None: + """Hide ingestion jobs that do not belong to the authenticated principal.""" + if principal is not None and job.owner_principal_id == principal: + return + raise IngestionJobNotFoundError(str(job.id)) + + +def _require_upload_owner( + owner_principal_id: str | None, principal: str | None +) -> None: + """Hide uploads that do not belong to the authenticated principal.""" + if principal is not None and owner_principal_id == principal: + return + raise UploadNotFoundError("upload") + + +def _require_principal(req: falcon.Request) -> str: + """Return the authenticated principal required for collection access.""" + principal = principal_id(req) + if principal is not None: + return principal + raise http_error( + falcon.HTTPUnauthorized(description="Authorization is required."), + code="unauthorized", + ) diff --git a/episodic/api/runtime.py b/episodic/api/runtime.py index e4a40fc2..d8680150 100644 --- a/episodic/api/runtime.py +++ b/episodic/api/runtime.py @@ -1,8 +1,6 @@ """Granian runtime composition root for the Falcon ASGI service.""" import dataclasses as dc -import os -import pathlib import typing as typ import psycopg @@ -10,27 +8,49 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from episodic.canonical.storage import FilesystemObjectStore, SqlAlchemyUnitOfWork -from episodic.logging import get_logger, log_info, log_warning +from episodic.cost.engine import PricingEngine +from episodic.cost.pricing_catalogue import FilePricingCatalogue +from episodic.cost.recorder import CostRecorder +from episodic.generation import ( + InProcessGenerationRunLauncher, + LLMDraftScriptGenerator, + LLMDraftScriptGeneratorConfig, +) +from episodic.llm import LLMProviderOperation, LLMTokenBudget +from episodic.llm.openai_adapter import ( + OpenAICompatibleLLMAdapter, + OpenAICompatibleLLMConfig, +) +from episodic.observability import StructuredLogMetrics, StructuredLogTracer from . import create_app +from .authorization import StaticBearerTokenAuthorization from .dependencies import ApiDependencies, ReadinessProbe, ShutdownHook +from .runtime_config import ( + RuntimeConfig, + _load_runtime_config, +) +from .runtime_config import ( + RuntimeConfigurationError as RuntimeConfigurationError, +) if typ.TYPE_CHECKING: - import collections.abc as cabc - from falcon import asgi + from episodic.canonical.object_store import ObjectStorePort from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + from episodic.llm import LLMPort + from episodic.observability import MetricsPort, ValueMetricsPort from .types import UowFactory @dc.dataclass(frozen=True, slots=True) -class RuntimeConfig: - """Runtime configuration required to boot the Falcon HTTP service.""" +class _GenerationLauncherRuntime: + """Composition inputs for the in-process generation launcher.""" - database_url: str - source_intake_object_store_root: pathlib.Path + metrics: ValueMetricsPort + object_store: ObjectStorePort | None = None _SUPPORTED_POSTGRES_DRIVERS = frozenset({"postgres", "postgresql"}) @@ -39,6 +59,8 @@ class RuntimeConfig: GRANIAN_FACTORY_TARGET = "episodic.api.runtime:create_app_from_env" GRANIAN_INTERFACE = "asgi" HTTP_BIND_PORT = 8080 +_DEFAULT_LLM_PROVIDER_NAME = "openai" +_DRAFT_MAX_INPUT_TOKENS = 32_768 class PsycopgConnectKwargs(typ.TypedDict, total=False): @@ -52,42 +74,23 @@ class PsycopgConnectKwargs(typ.TypedDict, total=False): sslmode: str -logger = get_logger(__name__) - - -def _load_runtime_config( - environ: cabc.Mapping[str, str] | None = None, -) -> RuntimeConfig: - """Read and validate runtime configuration from environment variables.""" - environment = os.environ if environ is None else environ - database_url = environment.get("DATABASE_URL", "").strip() - if not database_url: - msg = "DATABASE_URL must be set before starting the HTTP service." - raise RuntimeError(msg) - object_store_root = environment.get("SOURCE_INTAKE_OBJECT_STORE_ROOT", "").strip() - if not object_store_root: - log_warning( - logger, - "runtime_config_missing setting=%s", - "SOURCE_INTAKE_OBJECT_STORE_ROOT", +def _build_llm_port(config: RuntimeConfig) -> OpenAICompatibleLLMAdapter | None: + """Build the environment-configured OpenAI-compatible LLM adapter.""" + if config.llm_base_url is None or config.llm_api_key is None: + return None + return OpenAICompatibleLLMAdapter( + config=OpenAICompatibleLLMConfig( + base_url=config.llm_base_url, + api_key=config.llm_api_key, + provider_operation=LLMProviderOperation.CHAT_COMPLETIONS, ) - msg = ( - "SOURCE_INTAKE_OBJECT_STORE_ROOT must be set before starting " - "the HTTP service." - ) - raise RuntimeError(msg) - log_info( - logger, - "runtime_config_loaded source_intake_object_store_configured", - ) - return RuntimeConfig( - database_url=database_url, - source_intake_object_store_root=pathlib.Path(object_store_root), ) def _build_database_probe( database_url: str, + *, + metrics: MetricsPort, ) -> tuple[ReadinessProbe, UowFactory, ShutdownHook]: """Build the database readiness probe and unit-of-work factory.""" async_database_url, probe_connection_kwargs = _normalize_database_urls(database_url) @@ -112,7 +115,7 @@ async def check_database() -> bool: return True def uow_factory() -> CanonicalUnitOfWork: - return SqlAlchemyUnitOfWork(session_factory) + return SqlAlchemyUnitOfWork(session_factory, metrics=metrics) return ( ReadinessProbe(name="database", check=check_database), @@ -121,6 +124,50 @@ def uow_factory() -> CanonicalUnitOfWork: ) +def _build_generation_launcher( + uow_factory: UowFactory, + llm_port: LLMPort, + runtime: _GenerationLauncherRuntime, + *, + config: RuntimeConfig, +) -> InProcessGenerationRunLauncher: + """Build the no-QA generation-run launcher when an LLM port is configured.""" + pricing_catalogue = FilePricingCatalogue(config.pricing_snapshot_directory) + + def _cost_recorder(uow: CanonicalUnitOfWork) -> CostRecorder: + return CostRecorder( + ledger=uow.cost_ledger, + pricing_catalogue=pricing_catalogue, + pricing_engine=PricingEngine(), + ) + + return InProcessGenerationRunLauncher( + uow_factory=uow_factory, + draft_generator=LLMDraftScriptGenerator( + llm=llm_port, + config=LLMDraftScriptGeneratorConfig( + model=config.draft_model, + provider_operation=LLMProviderOperation.CHAT_COMPLETIONS, + token_budget=LLMTokenBudget( + max_input_tokens=_DRAFT_MAX_INPUT_TOKENS, + max_output_tokens=config.generation_max_output_tokens, + max_total_tokens=( + _DRAFT_MAX_INPUT_TOKENS + config.generation_max_output_tokens + ), + ), + max_response_bytes=config.generation_max_response_bytes, + ), + ), + object_store=runtime.object_store, + cost_recorder_factory=_cost_recorder, + provider_name=_DEFAULT_LLM_PROVIDER_NAME, + provider_operation=LLMProviderOperation.CHAT_COMPLETIONS.value, + metrics=runtime.metrics, + tracer=StructuredLogTracer(), + source_limits=config.generation_source_limits, + ) + + def _normalize_database_urls(database_url: str) -> tuple[URL, PsycopgConnectKwargs]: """Build async-engine and sync-probe URLs from one operator-facing setting.""" url = make_url(database_url) @@ -191,14 +238,50 @@ def _psycopg_connection_kwargs(url: URL) -> PsycopgConnectKwargs: def create_app_from_env() -> asgi.App: """Build the Falcon ASGI service from environment configuration.""" config = _load_runtime_config() + metrics = StructuredLogMetrics() database_probe, uow_factory, shutdown_hook = _build_database_probe( - config.database_url + config.database_url, + metrics=metrics, ) + object_store = FilesystemObjectStore(config.source_intake_object_store_root) + llm_port = _build_llm_port(config) + tracer = StructuredLogTracer() + if llm_port is None: + launcher = None + shutdown_hooks = (shutdown_hook,) + else: + launcher = _build_generation_launcher( + uow_factory, + llm_port, + _GenerationLauncherRuntime( + metrics=metrics, + object_store=object_store, + ), + config=config, + ) + + async def shutdown_generation() -> None: + """Stop generation work before closing its provider client.""" + try: + await launcher.shutdown() + finally: + await llm_port.aclose() + + shutdown_hooks = (shutdown_generation, shutdown_hook) return create_app( ApiDependencies( uow_factory=uow_factory, - object_store=FilesystemObjectStore(config.source_intake_object_store_root), + object_store=object_store, readiness_probes=(database_probe,), - shutdown_hooks=(shutdown_hook,), + shutdown_hooks=shutdown_hooks, + llm_port=llm_port, + launcher=launcher, + generation_source_limits=config.generation_source_limits, + metrics=metrics, + tracer=tracer, + authorization=StaticBearerTokenAuthorization( + token=config.authorization_bearer_token, + principal_id=config.authorization_principal_id, + ), ) ) diff --git a/episodic/api/runtime_config.py b/episodic/api/runtime_config.py new file mode 100644 index 00000000..ea08620d --- /dev/null +++ b/episodic/api/runtime_config.py @@ -0,0 +1,272 @@ +"""Load validated environment configuration for the Falcon composition root. + +``RuntimeConfig`` is the immutable result consumed by runtime wiring. Use +``_load_runtime_config`` at process startup to turn environment settings into +validated paths, credentials, source limits, and provider limits; invalid +configuration raises ``RuntimeConfigurationError`` before the application +starts serving requests. +""" + +import dataclasses as dc +import os +import pathlib +import typing as typ + +from episodic.generation import GenerationSourceLimits +from episodic.logging import get_logger, log_info, log_warning + +if typ.TYPE_CHECKING: + import collections.abc as cabc + +_DEFAULT_DRAFT_MODEL = "gpt-4o-mini" +_DEFAULT_GENERATION_MAX_OUTPUT_TOKENS = 4_096 +_DEFAULT_GENERATION_MAX_RESPONSE_BYTES = 1_048_576 +_REPOSITORY_ROOT = pathlib.Path(__file__).resolve().parents[2] +_DEFAULT_PRICING_DIRECTORY = _REPOSITORY_ROOT / "config/pricing-snapshots" +_PRICING_DIRECTORY_SETTING = "PRICING_SNAPSHOT_DIRECTORY" + +logger = get_logger(__name__) + + +@dc.dataclass(frozen=True, slots=True) +class RuntimeConfig: + """Runtime configuration required to boot the Falcon HTTP service. + + Attributes + ---------- + database_url : str + SQLAlchemy connection URL for canonical persistence. + source_intake_object_store_root : pathlib.Path + Root directory for accepted uploaded source objects. + llm_base_url : str | None + Optional OpenAI-compatible provider endpoint. + llm_api_key : str | None + Secret provider credential, excluded from representations. + draft_model : str + Model identifier used for no-QA draft generation. + pricing_snapshot_directory : pathlib.Path + Validated immutable YAML pricing catalogue directory. + authorization_bearer_token : str + Secret production bearer token, excluded from representations. + authorization_principal_id : str + Principal assigned to authenticated bearer requests. + generation_source_limits : GenerationSourceLimits + Source-count and byte limits applied before provider requests. + generation_max_output_tokens : int + Maximum provider output-token budget. + generation_max_response_bytes : int + Maximum provider response size before parsing. + + Examples + -------- + ``config = _load_runtime_config(os.environ)`` validates startup settings. + """ + + database_url: str = dc.field(repr=False) + source_intake_object_store_root: pathlib.Path + llm_base_url: str | None + llm_api_key: str | None = dc.field(repr=False) + draft_model: str + pricing_snapshot_directory: pathlib.Path + authorization_bearer_token: str = dc.field(repr=False) + authorization_principal_id: str + generation_source_limits: GenerationSourceLimits + generation_max_output_tokens: int = _DEFAULT_GENERATION_MAX_OUTPUT_TOKENS + generation_max_response_bytes: int = _DEFAULT_GENERATION_MAX_RESPONSE_BYTES + + +class RuntimeConfigurationError(RuntimeError): + """Raised when HTTP-runtime configuration cannot be validated. + + Examples + -------- + Missing required settings cause ``_load_runtime_config`` to raise this + error before application construction. + """ + + +def _required_setting( + environment: cabc.Mapping[str, str], + name: str, + error_message: str, +) -> str: + """Return a required, non-empty environment setting.""" + value = environment.get(name, "").strip() + if not value: + raise RuntimeConfigurationError(error_message) + return value + + +def _generation_source_limits( + environment: cabc.Mapping[str, str], +) -> GenerationSourceLimits: + """Read positive generation-source limits from optional runtime settings.""" + defaults = GenerationSourceLimits() + return GenerationSourceLimits( + max_source_count=_optional_positive_int( + environment, + "GENERATION_MAX_SOURCE_COUNT", + defaults.max_source_count, + ), + max_source_bytes=_optional_positive_int( + environment, + "GENERATION_MAX_SOURCE_BYTES", + defaults.max_source_bytes, + ), + max_aggregate_source_bytes=_optional_positive_int( + environment, + "GENERATION_MAX_AGGREGATE_SOURCE_BYTES", + defaults.max_aggregate_source_bytes, + ), + max_normalized_source_bytes=_optional_positive_int( + environment, + "GENERATION_MAX_NORMALIZED_SOURCE_BYTES", + defaults.max_normalized_source_bytes, + ), + ) + + +def _generation_output_limits( + environment: cabc.Mapping[str, str], +) -> tuple[int, int]: + """Read positive provider output-token and response-byte limits.""" + return ( + _optional_positive_int( + environment, + "GENERATION_MAX_OUTPUT_TOKENS", + _DEFAULT_GENERATION_MAX_OUTPUT_TOKENS, + ), + _optional_positive_int( + environment, + "GENERATION_MAX_RESPONSE_BYTES", + _DEFAULT_GENERATION_MAX_RESPONSE_BYTES, + ), + ) + + +def _draft_model(environment: cabc.Mapping[str, str]) -> str: + """Return the configured draft model or the production default.""" + if "DRAFT_MODEL" not in environment: + return _DEFAULT_DRAFT_MODEL + return _required_setting( + environment, + "DRAFT_MODEL", + "DRAFT_MODEL must be a non-empty string.", + ) + + +def _source_intake_object_store_root( + environment: cabc.Mapping[str, str], +) -> pathlib.Path: + """Return the required source-intake object-store root.""" + try: + value = _required_setting( + environment, + "SOURCE_INTAKE_OBJECT_STORE_ROOT", + "SOURCE_INTAKE_OBJECT_STORE_ROOT must be set before starting " + "the HTTP service.", + ) + except RuntimeConfigurationError: + log_warning( + logger, + "runtime_config_missing setting=%s", + "SOURCE_INTAKE_OBJECT_STORE_ROOT", + ) + raise + return pathlib.Path(value) + + +def _authorization_settings(environment: cabc.Mapping[str, str]) -> tuple[str, str]: + """Return the required production bearer token and principal identifier.""" + return ( + _required_setting( + environment, + "API_AUTHORIZATION_BEARER_TOKEN", + "API_AUTHORIZATION_BEARER_TOKEN must be set before starting " + "the HTTP service.", + ), + _required_setting( + environment, + "API_AUTHORIZATION_PRINCIPAL_ID", + "API_AUTHORIZATION_PRINCIPAL_ID must be set before starting " + "the HTTP service.", + ), + ) + + +def _optional_positive_int( + environment: cabc.Mapping[str, str], + name: str, + default: int, +) -> int: + """Return an optional positive integer setting or its configured default.""" + raw = environment.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as exc: + msg = f"{name} must be a positive integer." + raise RuntimeConfigurationError(msg) from exc + if value < 1: + msg = f"{name} must be a positive integer." + raise RuntimeConfigurationError(msg) + return value + + +def _llm_settings( + environment: cabc.Mapping[str, str], +) -> tuple[str | None, str | None]: + """Return the optional, paired OpenAI-compatible provider settings.""" + base_url = environment.get("OPENAI_BASE_URL", "").strip() or None + api_key = environment.get("OPENAI_API_KEY", "").strip() or None + if (base_url is None) != (api_key is None): + msg = "OPENAI_BASE_URL and OPENAI_API_KEY must be configured together." + raise RuntimeConfigurationError(msg) + return base_url, api_key + + +def _pricing_snapshot_directory( + environment: cabc.Mapping[str, str], +) -> pathlib.Path: + """Return the configured immutable pricing-catalogue directory.""" + configured = environment.get(_PRICING_DIRECTORY_SETTING, "").strip() + candidate = pathlib.Path(configured) if configured else _DEFAULT_PRICING_DIRECTORY + directory = candidate if candidate.is_absolute() else _REPOSITORY_ROOT / candidate + resolved = directory.resolve() + if not resolved.is_dir(): + msg = f"{_PRICING_DIRECTORY_SETTING} must name an existing directory." + raise RuntimeConfigurationError(msg) + return resolved + + +def _load_runtime_config( + environ: cabc.Mapping[str, str] | None = None, +) -> RuntimeConfig: + """Read and validate runtime configuration from environment variables.""" + environment = os.environ if environ is None else environ + llm_settings = _llm_settings(environment) + pricing_snapshot_directory = _pricing_snapshot_directory(environment) + authorization = _authorization_settings(environment) + output_limits = _generation_output_limits(environment) + log_info( + logger, + "runtime_config_loaded source_intake_object_store_configured", + ) + return RuntimeConfig( + database_url=_required_setting( + environment, + "DATABASE_URL", + "DATABASE_URL must be set before starting the HTTP service.", + ), + source_intake_object_store_root=_source_intake_object_store_root(environment), + llm_base_url=llm_settings[0], + llm_api_key=llm_settings[1], + draft_model=_draft_model(environment), + pricing_snapshot_directory=pricing_snapshot_directory, + authorization_bearer_token=authorization[0], + authorization_principal_id=authorization[1], + generation_source_limits=_generation_source_limits(environment), + generation_max_output_tokens=output_limits[0], + generation_max_response_bytes=output_limits[1], + ) diff --git a/episodic/api/serializers.py b/episodic/api/serializers.py index 1bc27ee5..f7970054 100644 --- a/episodic/api/serializers.py +++ b/episodic/api/serializers.py @@ -1,12 +1,21 @@ -"""Response serializers for Falcon profile and template endpoints.""" +"""Serialize canonical resources for Falcon API responses. + +The serializers cover profiles and templates, reusable reference documents, +source-intake resources, generation runs and events, and TEI envelopes. +""" import typing as typ import uuid # noqa: TC003 # This type remains available at runtime for annotation introspection. +from episodic.canonical.generation_quality import QualityMode + if typ.TYPE_CHECKING: from episodic.canonical.domain import ( + CanonicalEpisode, EpisodeTemplate, EpisodeTemplateHistoryEntry, + GenerationEvent, + GenerationRun, IngestionJob, ReferenceBinding, ReferenceDocument, @@ -203,3 +212,72 @@ def serialize_ingestion_job_source( "metadata": source.metadata, "created_at": source.created_at.isoformat(), } + + +def serialize_generation_run(run: GenerationRun) -> dict[str, typ.Any]: + """Serialize a generation-run polling resource.""" + return { + "id": str(run.id), + "episode_id": str(run.episode_id), + "source_bundle_id": str(run.source_bundle_id), + "actor": run.actor, + "status": run.status.value, + "current_node": run.current_node, + "budget_snapshot": run.budget_snapshot, + "configuration": run.configuration, + "quality_mode": run.quality_mode.value, + "qa_status": None if run.qa_status is None else run.qa_status.value, + "skip_qa_rationale": run.skip_qa_rationale, + "error_message": run.error_message, + "error_category": run.error_category, + "started_at": (None if run.started_at is None else run.started_at.isoformat()), + "ended_at": None if run.ended_at is None else run.ended_at.isoformat(), + "created_at": run.created_at.isoformat(), + "updated_at": run.updated_at.isoformat(), + } + + +def serialize_generation_event(event: GenerationEvent) -> dict[str, typ.Any]: + """Serialize one append-only generation event.""" + return { + "id": str(event.id), + "generation_run_id": str(event.generation_run_id), + "seq": int(event.seq), + "kind": event.kind, + "payload": event.payload, + "occurred_at": event.occurred_at.isoformat(), + "created_at": event.created_at.isoformat(), + } + + +def serialize_tei_envelope(episode: CanonicalEpisode) -> dict[str, typ.Any]: + """Serialize a canonical episode's generated TEI into public JSON fields. + + Parameters + ---------- + episode : CanonicalEpisode + Episode containing the generated XML, revision metadata, QA status, + and generation provenance. + + Returns + ------- + dict[str, typ.Any] + JSON-ready envelope mapping ``episode_id`` to ``episode.id``, + ``tei_header_id`` to ``episode.tei_header_id``, ``tei_xml`` to the + generated XML, ``content_hash`` to its hash, and ``version`` to + ``episode.tei_revision``. ``last_generation_run_id`` is the optional + generation-run UUID as a string; ``quality_mode`` is the public + ``draft_without_qa`` value; ``qa_status`` is its optional public value; + and ``updated_at`` is the episode timestamp in ISO 8601 form. + """ + return { + "episode_id": str(episode.id), + "tei_header_id": str(episode.tei_header_id), + "tei_xml": episode.tei_xml, + "content_hash": episode.tei_content_hash, + "version": episode.tei_revision, + "last_generation_run_id": _optional_uuid_str(episode.last_generation_run_id), + "quality_mode": QualityMode.DRAFT_WITHOUT_QA.value, + "qa_status": None if episode.qa_status is None else episode.qa_status.value, + "updated_at": episode.updated_at.isoformat(), + } diff --git a/episodic/api/source_idempotency.py b/episodic/api/source_idempotency.py index 574d556f..7ed36447 100644 --- a/episodic/api/source_idempotency.py +++ b/episodic/api/source_idempotency.py @@ -38,6 +38,8 @@ class IdempotentResponse: status: str media: JsonPayload + location: str | None = None + retry_after: str | None = None @dataclasses.dataclass(frozen=True, slots=True) @@ -81,6 +83,10 @@ def apply_response(resp: falcon.Response, response: IdempotentResponse) -> None: """Apply an idempotent response to the Falcon response object.""" resp.status = response.status resp.media = response.media + if response.location is not None: + resp.location = response.location + if response.retry_after is not None: + resp.set_header("Retry-After", response.retry_after) def principal_id(req: falcon.Request) -> str | None: @@ -156,7 +162,12 @@ def _idempotency_in_flight(record_id: uuid.UUID) -> falcon.HTTPConflict: def _encode_outcome(response: IdempotentResponse) -> bytes: """Serialize an HTTP adapter response for idempotent replay.""" - payload = canonical_json_bytes({"status": response.status, "media": response.media}) + payload = canonical_json_bytes({ + "status": response.status, + "media": response.media, + "location": response.location, + "retry_after": response.retry_after, + }) if len(payload) > _SERIALISED_OUTCOME_MAX_BYTES: raise ValueError(_SERIALISED_OUTCOME_TOO_LARGE) return payload @@ -170,4 +181,6 @@ def _decode_outcome(payload: bytes) -> IdempotentResponse: return IdempotentResponse( status=typ.cast("str", raw["status"]), media=typ.cast("JsonPayload", raw["media"]), + location=typ.cast("str | None", raw.get("location")), + retry_after=typ.cast("str | None", raw.get("retry_after")), ) diff --git a/episodic/canonical/__init__.py b/episodic/canonical/__init__.py index 72ef632b..b6b9a14a 100644 --- a/episodic/canonical/__init__.py +++ b/episodic/canonical/__init__.py @@ -26,6 +26,7 @@ CheckpointResponse, CheckpointStatus, EpisodeStatus, + EpisodeTeiUpdate, EpisodeTemplate, EpisodeTemplateHistoryEntry, GenerationEvent, @@ -55,6 +56,12 @@ SourceDocumentRepository, TeiHeaderRepository, ) +from .episode_errors import ( + EpisodeError, + EpisodeNotFoundError, + EpisodeRevisionConflictError, +) +from .generation_quality import QaStatus, QualityMode from .history_protocols import ( EpisodeTemplateHistoryRepository, SeriesProfileHistoryRepository, @@ -109,8 +116,12 @@ "ConflictOutcome", "EntityKind", "EntityNotFoundError", + "EpisodeError", + "EpisodeNotFoundError", "EpisodeRepository", + "EpisodeRevisionConflictError", "EpisodeStatus", + "EpisodeTeiUpdate", "EpisodeTemplate", "EpisodeTemplateHistoryEntry", "EpisodeTemplateHistoryRepository", @@ -125,6 +136,8 @@ "IngestionStatus", "MultiSourceRequest", "NormalizedSource", + "QaStatus", + "QualityMode", "RawSourceInput", "ReferenceBinding", "ReferenceBindingRepository", diff --git a/episodic/canonical/adapters/generation_checkpoints.py b/episodic/canonical/adapters/generation_checkpoints.py new file mode 100644 index 00000000..043855db --- /dev/null +++ b/episodic/canonical/adapters/generation_checkpoints.py @@ -0,0 +1,326 @@ +"""In-memory checkpoint helpers for generation-run adapters. + +These helpers keep checkpoint state inside the adapter store while preserving +the domain transitions and log events expected by the generation-run layer. + +Examples +-------- +Subclass the mixin on the in-memory generation-run store. +Outcome: checkpoint creation, lookup, and transitions reuse the same lock and +domain logging as the rest of the adapter. +""" + +import dataclasses as dc +import typing as typ + +from episodic.canonical.generation_run_errors import ( + CheckpointAlreadyTerminal, + CheckpointNotFound, + RunNotFound, +) +from episodic.orchestration._types import _log_event + +if typ.TYPE_CHECKING: + import asyncio + import collections.abc as cabc + import datetime as dt + import uuid + + from episodic.canonical.domain import ( + Checkpoint, + CheckpointResponse, + GenerationRun, + ) + + +@dc.dataclass(frozen=True, slots=True) +class _CheckpointTransitionSpec: + """Logging specification for a checkpoint domain transition.""" + + missing_event: str + done_event: str + extra_fields: dict[str, str] = dc.field(default_factory=dict) + + +class InMemoryGenerationCheckpointMixin: + """Checkpoint operations shared by the in-memory generation-run store. + + The mixin centralises checkpoint persistence and state transitions so the + in-memory adapter can reuse the same domain rules as the rest of the + generation-run implementation. + + Examples + -------- + >>> class Store(InMemoryGenerationCheckpointMixin): + ... pass + >>> isinstance(Store(), InMemoryGenerationCheckpointMixin) + True + """ + + _lock: asyncio.Lock + _runs: dict[uuid.UUID, GenerationRun] + _checkpoints: dict[uuid.UUID, Checkpoint] + + async def create_checkpoint(self, checkpoint: Checkpoint) -> Checkpoint: + """Persist a checkpoint in the in-memory store. + + Parameters + ---------- + checkpoint : Checkpoint + Checkpoint to store after validating that its generation run + exists. + + Returns + ------- + Checkpoint + The checkpoint that was stored. + + Raises + ------ + RunNotFound + Raised when the checkpoint references an unknown generation run. + + Examples + -------- + >>> # await store.create_checkpoint(checkpoint) + >>> # returned_checkpoint is checkpoint + True + """ + async with self._lock: + if checkpoint.generation_run_id not in self._runs: + _log_event( + "warning", + "generation_run_store.create_checkpoint_missing_run", + checkpoint_id=str(checkpoint.id), + run_id=str(checkpoint.generation_run_id), + ) + raise RunNotFound(checkpoint.generation_run_id) + self._checkpoints[checkpoint.id] = checkpoint + _log_event( + "info", + "generation_run_store.create_checkpoint", + checkpoint_id=str(checkpoint.id), + run_id=str(checkpoint.generation_run_id), + status=checkpoint.status.value, + ) + return checkpoint + + async def get_checkpoint( + self, + checkpoint_id: uuid.UUID, + ) -> Checkpoint | None: + """Return a checkpoint by identifier. + + Parameters + ---------- + checkpoint_id : uuid.UUID + Identifier of the checkpoint to fetch. + + Returns + ------- + Checkpoint | None + The stored checkpoint when present, otherwise `None`. + + Examples + -------- + >>> # await store.get_checkpoint(checkpoint_id) + >>> # returns the checkpoint when it exists, else None + True + """ + async with self._lock: + return self._checkpoints.get(checkpoint_id) + + async def _apply_checkpoint_transition( + self, + checkpoint_id: uuid.UUID, + transition: cabc.Callable[[Checkpoint], Checkpoint], + spec: _CheckpointTransitionSpec, + ) -> Checkpoint: + """Apply a domain transition to a stored checkpoint under the lock. + + Parameters + ---------- + checkpoint_id : uuid.UUID + Identifier of the checkpoint to transition. + transition : collections.abc.Callable[[Checkpoint], Checkpoint] + Domain transition to apply to the stored checkpoint. + spec : _CheckpointTransitionSpec + Logging labels associated with the transition. + + Returns + ------- + Checkpoint + The updated checkpoint after the domain transition. + + Raises + ------ + CheckpointNotFound + Raised when the checkpoint is not present in the store. + CheckpointAlreadyTerminal + Raised when the domain transition rejects a terminal checkpoint. + + Examples + -------- + >>> # await store._apply_checkpoint_transition(checkpoint_id, transition, spec) + >>> # returns the updated checkpoint when the transition succeeds + True + """ + async with self._lock: + checkpoint = self._checkpoints.get(checkpoint_id) + if checkpoint is None: + _log_event( + "warning", + spec.missing_event, + checkpoint_id=str(checkpoint_id), + **spec.extra_fields, + ) + raise CheckpointNotFound(checkpoint_id) + try: + updated = transition(checkpoint) + except CheckpointAlreadyTerminal: + _log_event( + "warning", + f"{spec.done_event}_already_terminal", + checkpoint_id=str(checkpoint_id), + run_id=str(checkpoint.generation_run_id), + status=checkpoint.status.value, + **spec.extra_fields, + ) + raise + self._checkpoints[checkpoint_id] = updated + _log_event( + "info", + spec.done_event, + checkpoint_id=str(checkpoint_id), + run_id=str(updated.generation_run_id), + status=updated.status.value, + **spec.extra_fields, + ) + return updated + + async def respond_to_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + response: CheckpointResponse, + ) -> Checkpoint: + """Record a reviewer response using the checkpoint domain transition. + + Parameters + ---------- + checkpoint_id : uuid.UUID + Identifier of the checkpoint to respond to. + response : CheckpointResponse + Reviewer response to persist against the checkpoint. + + Returns + ------- + Checkpoint + The updated checkpoint with the reviewer response applied. + + Raises + ------ + CheckpointNotFound + Raised when the checkpoint cannot be found. + CheckpointAlreadyTerminal + Raised when the checkpoint has already reached a terminal state. + + Examples + -------- + >>> # await store.respond_to_checkpoint(checkpoint_id, response=response) + >>> # returns the checkpoint after applying the response + True + """ # noqa: DOC502 # The shared transition helper raises these domain exceptions. + return await self._apply_checkpoint_transition( + checkpoint_id, + lambda cp: cp.respond(response), + _CheckpointTransitionSpec( + missing_event="generation_run_store.respond_checkpoint_missing", + done_event="generation_run_store.respond_checkpoint", + extra_fields={"action": response.action.value}, + ), + ) + + async def time_out_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + at: dt.datetime, + ) -> Checkpoint: + """Record a checkpoint timeout using the domain transition. + + Parameters + ---------- + checkpoint_id : uuid.UUID + Identifier of the checkpoint to time out. + at : datetime.datetime + Timestamp at which the timeout is recorded. + + Returns + ------- + Checkpoint + The updated checkpoint after timing out. + + Raises + ------ + CheckpointNotFound + Raised when the checkpoint cannot be found. + CheckpointAlreadyTerminal + Raised when the checkpoint has already reached a terminal state. + + Examples + -------- + >>> # await store.time_out_checkpoint(checkpoint_id, at=moment) + >>> # returns the checkpoint after applying the timeout + True + """ # noqa: DOC502 # The shared transition helper raises these domain exceptions. + return await self._apply_checkpoint_transition( + checkpoint_id, + lambda cp: cp.time_out(at), + _CheckpointTransitionSpec( + missing_event="generation_run_store.timeout_checkpoint_missing", + done_event="generation_run_store.timeout_checkpoint", + ), + ) + + async def cancel_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + at: dt.datetime, + ) -> Checkpoint: + """Record checkpoint cancellation using the domain transition. + + Parameters + ---------- + checkpoint_id : uuid.UUID + Identifier of the checkpoint to cancel. + at : datetime.datetime + Timestamp at which the cancellation is recorded. + + Returns + ------- + Checkpoint + The updated checkpoint after cancellation. + + Raises + ------ + CheckpointNotFound + Raised when the checkpoint cannot be found. + CheckpointAlreadyTerminal + Raised when the checkpoint has already reached a terminal state. + + Examples + -------- + >>> # await store.cancel_checkpoint(checkpoint_id, at=moment) + >>> # returns the checkpoint after applying the cancellation + True + """ # noqa: DOC502 # The shared transition helper raises these domain exceptions. + return await self._apply_checkpoint_transition( + checkpoint_id, + lambda cp: cp.cancel(at), + _CheckpointTransitionSpec( + missing_event="generation_run_store.cancel_checkpoint_missing", + done_event="generation_run_store.cancel_checkpoint", + ), + ) diff --git a/episodic/canonical/adapters/generation_runs.py b/episodic/canonical/adapters/generation_runs.py index 8fe4a024..77255bd2 100644 --- a/episodic/canonical/adapters/generation_runs.py +++ b/episodic/canonical/adapters/generation_runs.py @@ -1,9 +1,4 @@ -"""In-memory generation-run port adapter. - -This module provides a reference implementation of the generation-run port -protocols for tests and local development. It is ephemeral, single-process -storage and is not a production persistence layer. -""" +"""Ephemeral in-memory generation-run adapter for tests and local development.""" import asyncio import bisect @@ -14,21 +9,24 @@ from episodic.canonical.domain import ( Checkpoint, - CheckpointResponse, GenerationEvent, GenerationRun, GenerationRunStatus, JsonMapping, ) from episodic.canonical.generation_run_errors import ( - CheckpointAlreadyTerminal, - CheckpointNotFound, RunAlreadyTerminal, RunNotFound, ) -from episodic.canonical.generation_run_ports import EventSeq, event_seq +from episodic.canonical.generation_run_ports import ( + EventSeq, + GenerationRunStatusUpdate, + event_seq, +) from episodic.orchestration._types import _log_event +from .generation_checkpoints import InMemoryGenerationCheckpointMixin + type TimeProvider = cabc.Callable[[], dt.datetime] @@ -42,17 +40,24 @@ def _default_time_provider() -> TimeProvider: return _now_utc -@dc.dataclass(frozen=True, slots=True) -class _CheckpointTransitionSpec: - """Logging specification for a checkpoint domain transition.""" - - missing_event: str - done_event: str - extra_fields: dict[str, str] = dc.field(default_factory=dict) +def _event_page_minimum_seq( + *, + after_seq: EventSeq | None, + limit: int, + offset: int, +) -> int: + """Validate event-page arguments and return the cursor boundary.""" + if limit < 0 or offset < 0: + msg = "limit and offset must be non-negative." + raise ValueError(msg) + if after_seq is not None and offset != 0: + msg = "after_seq and offset cannot be combined." + raise ValueError(msg) + return int(after_seq) if after_seq is not None else 0 @dc.dataclass(slots=True) -class InMemoryGenerationRunStore: +class InMemoryGenerationRunStore(InMemoryGenerationCheckpointMixin): """In-memory reference adapter for the composite generation-run port.""" time_provider: TimeProvider = dc.field(default_factory=_default_time_provider) @@ -62,9 +67,28 @@ class InMemoryGenerationRunStore: ) _events: dict[uuid.UUID, list[GenerationEvent]] = dc.field(default_factory=dict) _checkpoints: dict[uuid.UUID, Checkpoint] = dc.field(default_factory=dict) - _idempotency_keys: dict[str, uuid.UUID] = dc.field(default_factory=dict) + _idempotency_keys: dict[tuple[str | None, str], uuid.UUID] = dc.field( + default_factory=dict + ) _lock: asyncio.Lock = dc.field(default_factory=asyncio.Lock) + def _require_mutable_run( + self, + run_id: uuid.UUID, + *, + on_missing: cabc.Callable[[], None], + on_terminal: cabc.Callable[[GenerationRun], None], + ) -> GenerationRun: + """Return a run that may still accept lifecycle mutations.""" + run = self._runs.get(run_id) + if run is None: + on_missing() + raise RunNotFound(run_id) + if run.status.is_terminal(): + on_terminal(run) + raise RunAlreadyTerminal(run_id) + return run + def _runs_for_episode( self, episode_id: uuid.UUID, @@ -74,6 +98,8 @@ def _runs_for_episode( offset: int, ) -> tuple[GenerationRun, ...]: """Return indexed runs for one episode without scanning all runs.""" + if limit == 0: + return () indexed_ids = self._run_ids_by_episode.get(episode_id, []) if status is None: selected_ids = indexed_ids[offset : offset + limit] @@ -92,11 +118,28 @@ def _runs_for_episode( break return tuple(selected) + def _event_page_for_run( + self, + run_id: uuid.UUID, + *, + minimum_seq: int, + limit: int, + offset: int, + ) -> tuple[GenerationEvent, ...]: + """Return one in-memory event page for an existing run.""" + if run_id not in self._runs: + raise RunNotFound(run_id) + events = tuple( + event for event in self._events.get(run_id, []) if event.seq > minimum_seq + ) + return events[offset : offset + limit] + async def create_run( self, run: GenerationRun, *, idempotency_key: str | None = None, + idempotency_principal_id: str | None = None, ) -> GenerationRun: """Create a run, preserving first-write-wins idempotency. @@ -108,7 +151,8 @@ async def create_run( """ async with self._lock: if idempotency_key is not None: - existing_id = self._idempotency_keys.get(idempotency_key) + scope = (idempotency_principal_id, idempotency_key) + existing_id = self._idempotency_keys.get(scope) if existing_id is not None: _log_event( "info", @@ -125,7 +169,7 @@ async def create_run( ) self._events.setdefault(run.id, []) if idempotency_key is not None: - self._idempotency_keys[idempotency_key] = run.id + self._idempotency_keys[scope] = run.id _log_event( "info", "generation_run_store.create_run", @@ -162,40 +206,37 @@ async def list_runs( offset=offset, ) - # pylint: disable-next=too-many-arguments # Port signature is fixed. async def update_run_status( self, run_id: uuid.UUID, *, - status: GenerationRunStatus, - current_node: str | None, - ended_at: dt.datetime | None, + update: GenerationRunStatusUpdate, ) -> GenerationRun: """Update lifecycle fields for a run.""" async with self._lock: - run = self._runs.get(run_id) - if run is None: - _log_event( + run = self._require_mutable_run( + run_id, + on_missing=lambda: _log_event( "warning", "generation_run_store.update_run_missing", run_id=str(run_id), - status=status.value, - ) - raise RunNotFound(run_id) - if run.status.is_terminal(): - _log_event( + status=update.status.value, + ), + on_terminal=lambda run: _log_event( "warning", "generation_run_store.update_run_terminal", run_id=str(run_id), current_status=run.status.value, - requested_status=status.value, - ) - raise RunAlreadyTerminal(run_id) + requested_status=update.status.value, + ), + ) updated = dc.replace( run, - status=status, - current_node=current_node, - ended_at=ended_at, + status=update.status, + current_node=update.current_node, + ended_at=update.ended_at, + error_message=update.error_message, + error_category=update.error_category, updated_at=self.time_provider(), ) self._runs[run_id] = updated @@ -204,8 +245,70 @@ async def update_run_status( "generation_run_store.update_run_status", run_id=str(run_id), previous_status=run.status.value, - status=status.value, + status=update.status.value, + current_node=update.current_node, + ) + return updated + + # pylint: disable-next=too-many-arguments # Port signature is fixed. + async def claim_run_for_execution( + self, + run_id: uuid.UUID, + *, + current_node: str | None, + started_at: dt.datetime, + lease_expires_at: dt.datetime | None, + ) -> GenerationRun | None: + """Atomically claim a pending run for execution. + + The reference adapter logs ``lease_expires_at`` for protocol parity but + does not retain it: ``GenerationRun`` has no lease field and no + in-memory consumer reads lease state. + + Returns + ------- + GenerationRun | None + The claimed run, or ``None`` when another claimant already won. + """ + async with self._lock: + run = self._require_mutable_run( + run_id, + on_missing=lambda: _log_event( + "warning", + "generation_run_store.claim_run_missing", + run_id=str(run_id), + ), + on_terminal=lambda run: _log_event( + "warning", + "generation_run_store.claim_run_terminal", + run_id=str(run_id), + status=run.status.value, + ), + ) + if run.status is not GenerationRunStatus.PENDING: + _log_event( + "info", + "generation_run_store.claim_run_lost", + run_id=str(run_id), + status=run.status.value, + ) + return None + updated = dc.replace( + run, + status=GenerationRunStatus.RUNNING, + current_node=current_node, + started_at=started_at, + updated_at=self.time_provider(), + ) + self._runs[run_id] = updated + _log_event( + "info", + "generation_run_store.claim_run", + run_id=str(run_id), current_node=current_node, + lease_expires_at=lease_expires_at.isoformat() + if lease_expires_at is not None + else None, ) return updated @@ -220,24 +323,22 @@ async def append_event( ) -> GenerationEvent: """Append an event with an adapter-allocated sequence.""" async with self._lock: - run = self._runs.get(run_id) - if run is None: - _log_event( + self._require_mutable_run( + run_id, + on_missing=lambda: _log_event( "warning", "generation_run_store.append_event_missing_run", run_id=str(run_id), kind=kind, - ) - raise RunNotFound(run_id) - if run.status.is_terminal(): - _log_event( + ), + on_terminal=lambda run: _log_event( "warning", "generation_run_store.append_event_terminal_run", run_id=str(run_id), status=run.status.value, kind=kind, - ) - raise RunAlreadyTerminal(run_id) + ), + ) events = self._events.setdefault(run_id, []) now = self.time_provider() event = GenerationEvent( @@ -266,136 +367,33 @@ async def list_events( *, after_seq: EventSeq | None = None, limit: int = 100, + offset: int = 0, ) -> tuple[GenerationEvent, ...]: """List events for a run after an optional sequence cursor.""" - if limit < 0: - msg = "limit must be non-negative." - raise ValueError(msg) - async with self._lock: - if run_id not in self._runs: - raise RunNotFound(run_id) - minimum_seq = int(after_seq) if after_seq is not None else 0 - events = [ - event - for event in self._events.get(run_id, []) - if event.seq > minimum_seq - ] - return tuple(events[:limit]) - - async def create_checkpoint(self, checkpoint: Checkpoint) -> Checkpoint: - """Persist a checkpoint.""" + minimum_seq = _event_page_minimum_seq( + after_seq=after_seq, + limit=limit, + offset=offset, + ) async with self._lock: - if checkpoint.generation_run_id not in self._runs: - _log_event( - "warning", - "generation_run_store.create_checkpoint_missing_run", - checkpoint_id=str(checkpoint.id), - run_id=str(checkpoint.generation_run_id), - ) - raise RunNotFound(checkpoint.generation_run_id) - self._checkpoints[checkpoint.id] = checkpoint - _log_event( - "info", - "generation_run_store.create_checkpoint", - checkpoint_id=str(checkpoint.id), - run_id=str(checkpoint.generation_run_id), - status=checkpoint.status.value, + return self._event_page_for_run( + run_id, + minimum_seq=minimum_seq, + limit=limit, + offset=offset, ) - return checkpoint - - async def get_checkpoint( - self, - checkpoint_id: uuid.UUID, - ) -> Checkpoint | None: - """Return a checkpoint by identifier.""" - async with self._lock: - return self._checkpoints.get(checkpoint_id) - async def _apply_checkpoint_transition( + async def count_events( self, - checkpoint_id: uuid.UUID, - transition: cabc.Callable[[Checkpoint], Checkpoint], - spec: _CheckpointTransitionSpec, - ) -> Checkpoint: - """Apply a domain transition to a stored checkpoint under the lock.""" + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + ) -> int: + """Count events for a run after an optional sequence cursor.""" async with self._lock: - checkpoint = self._checkpoints.get(checkpoint_id) - if checkpoint is None: - _log_event( - "warning", - spec.missing_event, - checkpoint_id=str(checkpoint_id), - **spec.extra_fields, - ) - raise CheckpointNotFound(checkpoint_id) - try: - updated = transition(checkpoint) - except CheckpointAlreadyTerminal: - _log_event( - "warning", - f"{spec.done_event}_already_terminal", - checkpoint_id=str(checkpoint_id), - run_id=str(checkpoint.generation_run_id), - status=checkpoint.status.value, - **spec.extra_fields, - ) - raise - self._checkpoints[checkpoint_id] = updated - _log_event( - "info", - spec.done_event, - checkpoint_id=str(checkpoint_id), - run_id=str(updated.generation_run_id), - status=updated.status.value, - **spec.extra_fields, + if run_id not in self._runs: + raise RunNotFound(run_id) + minimum_seq = int(after_seq) if after_seq is not None else 0 + return sum( + event.seq > minimum_seq for event in self._events.get(run_id, []) ) - return updated - - async def respond_to_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - response: CheckpointResponse, - ) -> Checkpoint: - """Record a reviewer response using the checkpoint domain transition.""" - return await self._apply_checkpoint_transition( - checkpoint_id, - lambda cp: cp.respond(response), - _CheckpointTransitionSpec( - missing_event="generation_run_store.respond_checkpoint_missing", - done_event="generation_run_store.respond_checkpoint", - extra_fields={"action": response.action.value}, - ), - ) - - async def time_out_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - at: dt.datetime, - ) -> Checkpoint: - """Record a checkpoint timeout using the domain transition.""" - return await self._apply_checkpoint_transition( - checkpoint_id, - lambda cp: cp.time_out(at), - _CheckpointTransitionSpec( - missing_event="generation_run_store.timeout_checkpoint_missing", - done_event="generation_run_store.timeout_checkpoint", - ), - ) - - async def cancel_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - at: dt.datetime, - ) -> Checkpoint: - """Record checkpoint cancellation using the domain transition.""" - return await self._apply_checkpoint_transition( - checkpoint_id, - lambda cp: cp.cancel(at), - _CheckpointTransitionSpec( - missing_event="generation_run_store.cancel_checkpoint_missing", - done_event="generation_run_store.cancel_checkpoint", - ), - ) diff --git a/episodic/canonical/domain.py b/episodic/canonical/domain.py index 4eddcdf1..9ac6e2c6 100644 --- a/episodic/canonical/domain.py +++ b/episodic/canonical/domain.py @@ -4,6 +4,7 @@ import enum import typing as typ +from .generation_quality import QaStatus, QualityMode from .generation_run_errors import CheckpointAlreadyTerminal if typ.TYPE_CHECKING: @@ -147,12 +148,22 @@ class GenerationRun: started_at: dt.datetime | None ended_at: dt.datetime | None error_message: str | None + error_category: str | None = None + quality_mode: QualityMode = QualityMode.DRAFT_WITHOUT_QA + qa_status: QaStatus | None = None + skip_qa_rationale: str | None = None def __post_init__(self) -> None: """Validate generation-run invariants.""" _validate_non_empty_text(self.actor, "actor") _validate_optional_text(self.current_node, "current_node") _validate_optional_text(self.error_message, "error_message") + _validate_optional_text(self.error_category, "error_category") + _validate_draft_without_qa_metadata( + quality_mode=self.quality_mode, + qa_status=self.qa_status, + skip_qa_rationale=self.skip_qa_rationale, + ) _copy_json_mapping(self, "budget_snapshot") _copy_json_mapping(self, "configuration") @@ -305,6 +316,20 @@ class TeiHeader: updated_at: dt.datetime +def _require_positive_integer(value: object, field_name: str) -> None: + """Require an exact positive integer, excluding boolean values.""" + if type(value) is not int or value < 1: + msg = f"{field_name} must be a positive integer." + raise ValueError(msg) + + +def _require_value(value: object, field_name: str) -> None: + """Require a non-null provenance value.""" + if value is None: + msg = f"{field_name} must be set." + raise ValueError(msg) + + @dc.dataclass(frozen=True, slots=True) class CanonicalEpisode: """Canonical episode representation.""" @@ -318,6 +343,37 @@ class CanonicalEpisode: approval_state: ApprovalState created_at: dt.datetime updated_at: dt.datetime + tei_revision: int = 1 + tei_content_hash: str | None = None + qa_status: QaStatus | None = None + last_generation_run_id: uuid.UUID | None = None + + def __post_init__(self) -> None: + """Validate TEI revision metadata.""" + _validate_non_empty_text(self.tei_xml, "tei_xml") + _require_positive_integer(self.tei_revision, "tei_revision") + _validate_optional_text( + self.tei_content_hash, + "tei_content_hash", + ) + + +@dc.dataclass(frozen=True, slots=True) +class EpisodeTeiUpdate: + """Optimistic TEI update request for a canonical episode.""" + + tei_xml: str + qa_status: QaStatus + last_generation_run_id: uuid.UUID + expected_revision: int + updated_at: dt.datetime + + def __post_init__(self) -> None: + """Validate optimistic TEI update invariants.""" + _validate_non_empty_text(self.tei_xml, "tei_xml") + _require_value(self.qa_status, "qa_status") + _require_value(self.last_generation_run_id, "last_generation_run_id") + _require_positive_integer(self.expected_revision, "expected_revision") @dc.dataclass(frozen=True, slots=True) @@ -335,6 +391,7 @@ class IngestionJob: created_at: dt.datetime updated_at: dt.datetime intake_state: IntakeState = IntakeState.AWAITING_SOURCES + owner_principal_id: str | None = None @dc.dataclass(frozen=True, slots=True) @@ -343,6 +400,7 @@ class IngestionJobListFilters: series_profile_id: uuid.UUID | None intake_state: IntakeState | None + owner_principal_id: str | None = None @dc.dataclass(frozen=True, slots=True) @@ -618,6 +676,25 @@ def _validate_optional_text(value: str | None, field_name: str) -> None: _validate_non_empty_text(value, field_name) +def _validate_draft_without_qa_metadata( + *, + quality_mode: QualityMode, + qa_status: QaStatus | None, + skip_qa_rationale: str | None, +) -> None: + """Validate the no-QA slice's required audit metadata.""" + if quality_mode is not QualityMode.DRAFT_WITHOUT_QA: + msg = f"Unsupported quality_mode: {quality_mode!s}." + raise ValueError(msg) + if qa_status is not QaStatus.SKIPPED: + msg = "qa_status must be skipped for draft_without_qa runs." + raise ValueError(msg) + if skip_qa_rationale is None: + msg = "skip_qa_rationale must be a non-empty string." + raise ValueError(msg) + _validate_non_empty_text(skip_qa_rationale, "skip_qa_rationale") + + def _copy_json_mapping(owner: object, field_name: str) -> None: """Validate and defensively copy a JSON mapping field.""" value = getattr(owner, field_name) diff --git a/episodic/canonical/entity_protocols.py b/episodic/canonical/entity_protocols.py index 8ba9105b..22b15b02 100644 --- a/episodic/canonical/entity_protocols.py +++ b/episodic/canonical/entity_protocols.py @@ -1,14 +1,17 @@ """Repository protocols for core canonical entities.""" +import enum import typing as typ if typ.TYPE_CHECKING: import collections.abc as cabc + import datetime as dt import uuid from .domain import ( ApprovalEvent, CanonicalEpisode, + EpisodeTeiUpdate, EpisodeTemplate, IngestionJob, IngestionJobListFilters, @@ -19,6 +22,13 @@ ) +class SourceDocumentProjectionResult(enum.StrEnum): + """Outcome of a deterministic source-document projection write.""" + + ADDED = "added" + DUPLICATE = "duplicate" + + class SeriesProfileRepository(typ.Protocol): """Persistence interface for series profiles.""" @@ -72,6 +82,23 @@ async def list_by_ids( """Fetch canonical episodes by identifiers.""" raise NotImplementedError + async def update( + self, + episode_id: uuid.UUID, + *, + update: EpisodeTeiUpdate, + ) -> CanonicalEpisode: + """Update episode TEI with an optimistic revision precondition. + + Raises + ------ + EpisodeNotFoundError + If no episode exists for ``episode_id``. + EpisodeRevisionConflictError + If ``update.expected_revision`` differs from the stored revision. + """ + raise NotImplementedError + class IngestionJobRepository(typ.Protocol): """Persistence interface for ingestion jobs.""" @@ -84,6 +111,35 @@ async def get(self, job_id: uuid.UUID) -> IngestionJob | None: """Fetch an ingestion job by identifier.""" raise NotImplementedError + async def get_for_update(self, job_id: uuid.UUID) -> IngestionJob | None: + """Fetch and lock an ingestion job for transactional mutation.""" + raise NotImplementedError + + async def set_target_episode( + self, + job_id: uuid.UUID, + *, + episode_id: uuid.UUID, + updated_at: dt.datetime, + ) -> None: + """Associate an ingestion job with its materialised episode. + + Parameters + ---------- + job_id + Ingestion-job identifier to update. + episode_id + Canonical episode identifier reserved for the job. + updated_at + Timestamp supplied by the caller for the durable job update. + + Notes + ----- + The caller owns transaction boundaries. Unknown ``job_id`` values + affect zero rows and do not raise. + """ + raise NotImplementedError + async def list_paged( self, filters: IngestionJobListFilters, @@ -119,6 +175,35 @@ async def add(self, document: SourceDocument) -> None: """Persist a source document.""" raise NotImplementedError + async def add_projection( + self, + document: SourceDocument, + ) -> SourceDocumentProjectionResult: + """Persist a deterministic projection and report duplicate races. + + Parameters + ---------- + document : SourceDocument + Deterministically identified source document to persist. + + Returns + ------- + SourceDocumentProjectionResult + ``ADDED`` when inserted or ``DUPLICATE`` when an equivalent + projection won a concurrent insertion race. + + Raises + ------ + sqlalchemy.exc.IntegrityError + Unrecognised persistence failures propagate unchanged. + + Examples + -------- + ``await repository.add_projection(document)`` returns ``DUPLICATE`` + when the deterministic projection is already durable. + """ + raise NotImplementedError + async def list_for_job(self, job_id: uuid.UUID) -> list[SourceDocument]: """List source documents for an ingestion job.""" raise NotImplementedError diff --git a/episodic/canonical/episode_errors.py b/episodic/canonical/episode_errors.py new file mode 100644 index 00000000..5b8f049a --- /dev/null +++ b/episodic/canonical/episode_errors.py @@ -0,0 +1,103 @@ +"""Episode persistence errors.""" + +import typing as typ + +if typ.TYPE_CHECKING: + import uuid + + +class EpisodeError(Exception): + """Base exception for failures in the canonical episode lifecycle. + + Canonical repositories and generation services use this exception family + when an episode cannot be loaded or its optimistic persistence precondition + is not satisfied. + + Notes + ----- + Application boundaries can catch this base class while preserving the more + specific subclass and its diagnostic attributes for stable failure + categorisation. + """ + + +class EpisodeNotFoundError(EpisodeError): + """Signal that a requested canonical episode does not exist. + + Raised when a repository or generation service cannot load the episode + required for a lifecycle operation such as draft generation or TEI + retrieval. + + Parameters + ---------- + episode_id : uuid.UUID + Identifier of the episode that could not be loaded. + + Attributes + ---------- + episode_id : uuid.UUID + Identifier of the missing episode, retained for error mapping and + diagnostics. + + Notes + ----- + This error represents a lookup failure before the requested generation or + TEI persistence transition can begin. + """ + + def __init__(self, episode_id: uuid.UUID) -> None: + """Initialize an error for a missing canonical episode. + + Parameters + ---------- + episode_id : uuid.UUID + Identifier of the episode that could not be loaded. + """ + self.episode_id = episode_id + message = f"Episode {episode_id} was not found." + super().__init__(message) + + +class EpisodeRevisionConflictError(EpisodeError): + """Signal that an episode TEI revision precondition failed. + + Raised when a generation result attempts to persist against an episode + whose stored ``tei_revision`` no longer matches the caller's expectation. + + Parameters + ---------- + episode_id : uuid.UUID + Identifier of the episode whose TEI revision changed. + expected_revision : int + Revision the caller required before applying its update. + + Attributes + ---------- + episode_id : uuid.UUID + Identifier of the conflicting episode. + expected_revision : int + Revision supplied by the caller for the optimistic concurrency check. + + Notes + ----- + This error occurs at the TEI persistence boundary after generation has + produced output but before that output is committed to the episode. + """ + + def __init__(self, episode_id: uuid.UUID, expected_revision: int) -> None: + """Initialize an error for a failed optimistic revision check. + + Parameters + ---------- + episode_id : uuid.UUID + Identifier of the episode whose TEI revision changed. + expected_revision : int + Revision the caller required before applying its update. + """ + self.episode_id = episode_id + self.expected_revision = expected_revision + message = ( + f"Episode {episode_id} revision did not match expected revision " + f"{expected_revision}." + ) + super().__init__(message) diff --git a/episodic/canonical/generation_persistence.py b/episodic/canonical/generation_persistence.py new file mode 100644 index 00000000..a4ac3b01 --- /dev/null +++ b/episodic/canonical/generation_persistence.py @@ -0,0 +1,371 @@ +"""Persist canonical state at no-QA draft-generation boundaries. + +Services ``materialise_episode_from_ingestion`` and ``persist_draft_script``, +their request types, and typed persistence errors are exported. Callers provide +a ``CanonicalUnitOfWork``; this module never creates or disposes one. A ready +:class:`IngestionJob` becomes a deterministic placeholder: pages load before +locking, reservation and projection commit atomically, and repository results +verify duplicates. ``persist_draft_script`` validates a :class:`DraftScriptResult`, +carries :class:`GenerationRun` provenance through :class:`EpisodeTeiUpdate`, +and persists an optimistic TEI revision without commit or rollback. +""" + +import typing as typ +import uuid + +import tei_rapporteur as tei + +from episodic.canonical.domain import ( + ApprovalState, + CanonicalEpisode, + EpisodeStatus, + EpisodeTeiUpdate, + IngestionJob, + IntakeState, + TeiHeader, +) +from episodic.canonical.entity_protocols import SourceDocumentProjectionResult +from episodic.canonical.generation_persistence_projection import ( + source_document_from_attachment, +) +from episodic.canonical.generation_persistence_types import ( + DraftContentHashMismatchError, + DraftScriptPersistenceError, # noqa: F401 # Re-exported service contract. + DraftScriptPersistenceRequest, + EpisodeMaterialisationRequest, + GenerationSourceUploadNotFoundError, + IngestionJobNotReadyError, + InvalidDraftTeiError, + MissingAttachedSourcesError, + SourceCountLimitExceededError, + SourceDocumentProjectionError, + _SourceDocumentProjection, +) +from episodic.canonical.generation_quality import QaStatus +from episodic.canonical.hashing import sha256_text +from episodic.canonical.source_intake_errors import IngestionJobNotFoundError +from episodic.canonical.tei import parse_tei_header + +if typ.TYPE_CHECKING: + import datetime as dt + + from episodic.canonical.ingestion_sources import IngestionJobSource + from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + from episodic.canonical.uploads import Upload + from episodic.generation.draft_script import DraftScriptResult + + +async def materialise_episode_from_ingestion( + uow: CanonicalUnitOfWork, + request: EpisodeMaterialisationRequest, +) -> CanonicalEpisode: + """Materialise a ready ingestion job as a placeholder canonical episode. + + Parameters + ---------- + uow + Open unit of work; commits a successful reservation and projection. + request + Materialisation command describing the job and deterministic seams. + + Returns + ------- + CanonicalEpisode + Placeholder episode and its deterministic source projections. + + Raises + ------ + MissingAttachedSourcesError + If the ready job has no attached sources. + SourceCountLimitExceededError + If attached sources exceed ``request.max_source_count``. + GenerationSourceUploadNotFoundError + If a source projection references an unavailable upload. + SourceDocumentProjectionError + If a duplicate projection cannot be verified after persistence. + + Notes + ----- + Propagates :class:`IngestionJobNotFoundError` if no ingestion job exists, + and :class:`IngestionJobNotReadyError` if the job is not ready for + generation. + """ # noqa: DOC502 # Typed projection failures propagate through helpers. + sources = await _list_all_sources( + uow, + request.ingestion_job_id, + max_source_count=request.max_source_count, + ) + if len(sources) == 0: + await _load_ready_ingestion_job( + uow, request.ingestion_job_id, should_lock=False + ) + raise MissingAttachedSourcesError(request.ingestion_job_id) + + job = await _load_ready_ingestion_job( + uow, request.ingestion_job_id, should_lock=True + ) + + now = request.clock() + episode = await _materialise_or_reuse_episode(uow, job, request, now) + + has_duplicate = await _project_source_documents(uow, sources, episode.id, now) + if has_duplicate: + await _require_projected_source_documents(uow, sources, episode.id) + await uow.commit() + return episode + + +async def _materialise_or_reuse_episode( + uow: CanonicalUnitOfWork, + job: IngestionJob, + request: EpisodeMaterialisationRequest, + now: dt.datetime, +) -> CanonicalEpisode: + """Reserve and return the ingestion job's canonical episode.""" + episode_id = job.target_episode_id or request.uuid_factory() + existing_episode = await uow.episodes.get(episode_id) + if existing_episode is not None: + return existing_episode + + header = _build_placeholder_header( + header_id=request.uuid_factory(), + title=request.title, + now=now, + ) + episode = _build_placeholder_episode( + episode_id=episode_id, + job=job, + header=header, + now=now, + ) + await uow.tei_headers.add(header) + await uow.flush() + await uow.episodes.add(episode) + await uow.flush() + await uow.ingestion_jobs.set_target_episode( + job.id, + episode_id=episode.id, + updated_at=now, + ) + return episode + + +async def persist_draft_script( + uow: CanonicalUnitOfWork, + request: DraftScriptPersistenceRequest, +) -> CanonicalEpisode: + """Persist generated TEI and no-QA provenance onto an episode. + + Parameters + ---------- + uow + Open canonical unit of work that owns the revision-guarded update. + request + Draft result, episode provenance, expected revision, and clock. + + Returns + ------- + CanonicalEpisode + Episode returned by the revision-guarded repository update. + + Raises + ------ + InvalidDraftTeiError + If the generated TEI cannot be parsed as a TEI header. + + Notes + ----- + Hash mismatches and revision conflicts retain their typed domain errors. + The service does not commit or roll back the unit of work; callers compose + its episode update with generation-run events and terminal status writes. + """ + _validate_draft_result(request.result) + try: + parse_tei_header(request.result.tei_xml) + except (TypeError, ValueError) as exc: + raise InvalidDraftTeiError(str(exc)) from exc + return await uow.episodes.update( + request.episode_id, + update=EpisodeTeiUpdate( + tei_xml=request.result.tei_xml, + qa_status=QaStatus.SKIPPED, + last_generation_run_id=request.generation_run_id, + expected_revision=request.expected_revision, + updated_at=request.clock(), + ), + ) + + +async def _load_ready_ingestion_job( + uow: CanonicalUnitOfWork, + ingestion_job_id: uuid.UUID, + *, + should_lock: bool, +) -> IngestionJob: + """Load a ready ingestion job, optionally retaining its row lock.""" + if should_lock: + job = await uow.ingestion_jobs.get_for_update(ingestion_job_id) + else: + job = await uow.ingestion_jobs.get(ingestion_job_id) + if job is None: + raise IngestionJobNotFoundError(str(ingestion_job_id)) + if job.intake_state is not IntakeState.READY_FOR_GENERATION: + raise IngestionJobNotReadyError(ingestion_job_id) + return job + + +async def _list_all_sources( + uow: CanonicalUnitOfWork, + ingestion_job_id: uuid.UUID, + *, + max_source_count: int, +) -> list[IngestionJobSource]: + """Read at most one more source than the configured materialisation limit.""" + sources = list( + await uow.ingestion_job_sources.list_for_job_paged( + ingestion_job_id, + limit=max_source_count + 1, + offset=0, + ) + ) + if len(sources) > max_source_count: + raise SourceCountLimitExceededError(ingestion_job_id) + return sources + + +async def _projected_source_document_ids( + uow: CanonicalUnitOfWork, + ingestion_job_id: uuid.UUID, +) -> set[uuid.UUID]: + """Return persisted source-document IDs for one ingestion job.""" + documents = await uow.source_documents.list_for_job(ingestion_job_id) + return {document.id for document in documents} + + +async def _project_source_documents( + uow: CanonicalUnitOfWork, + sources: list[IngestionJobSource], + episode_id: uuid.UUID, + now: dt.datetime, +) -> bool: + """Persist source projections absent from this episode's job.""" + existing_document_ids = await _projected_source_document_ids( + uow, sources[0].ingestion_job_id + ) + had_duplicate = False + for source in sources: + document_id = uuid.uuid5(episode_id, str(source.id)) + if document_id in existing_document_ids: + continue + upload = await _upload_for_source(uow, source) + result = await uow.source_documents.add_projection( + source_document_from_attachment( + _SourceDocumentProjection( + source=source, + upload=upload, + episode_id=episode_id, + document_id=document_id, + now=now, + ) + ) + ) + had_duplicate |= result is SourceDocumentProjectionResult.DUPLICATE + return had_duplicate + + +async def _require_projected_source_documents( + uow: CanonicalUnitOfWork, + sources: list[IngestionJobSource], + episode_id: uuid.UUID, +) -> None: + """Verify a duplicate race left every deterministic projection durable.""" + projected_ids = await _projected_source_document_ids( + uow, sources[0].ingestion_job_id + ) + expected_ids = {uuid.uuid5(episode_id, str(source.id)) for source in sources} + missing_ids = expected_ids - projected_ids + if missing_ids: + raise SourceDocumentProjectionError(missing_ids) + + +def _build_placeholder_header( + *, + header_id: uuid.UUID, + title: str, + now: dt.datetime, +) -> TeiHeader: + """Build a validated placeholder TEI header.""" + tei_xml = _placeholder_tei_xml(title) + header_payload = parse_tei_header(tei_xml) + return TeiHeader( + id=header_id, + title=header_payload.title, + payload=header_payload.payload, + raw_xml=tei_xml, + created_at=now, + updated_at=now, + ) + + +def _placeholder_tei_xml(title: str) -> str: + """Return minimal valid TEI used before draft generation completes.""" + payload = { + "header": {"file_desc": {"title": title}}, + "text": { + "body": { + "blocks": [ + { + "type": "paragraph", + "xml_id": "p-placeholder", + "content": [ + {"type": "text", "value": "Draft generation pending."} + ], + } + ] + } + }, + } + document = tei.from_dict(payload) + document.validate() + return tei.emit_xml(document) + + +def _build_placeholder_episode( + *, + episode_id: uuid.UUID, + job: IngestionJob, + header: TeiHeader, + now: dt.datetime, +) -> CanonicalEpisode: + """Build a placeholder canonical episode.""" + return CanonicalEpisode( + id=episode_id, + series_profile_id=job.series_profile_id, + tei_header_id=header.id, + title=header.title, + tei_xml=header.raw_xml, + status=EpisodeStatus.DRAFT, + approval_state=ApprovalState.DRAFT, + created_at=now, + updated_at=now, + ) + + +async def _upload_for_source( + uow: CanonicalUnitOfWork, + source: IngestionJobSource, +) -> Upload | None: + """Return upload metadata when an attachment points at an upload.""" + if source.upload_id is None: + return None + upload = await uow.uploads.get(source.upload_id) + if upload is None: + raise GenerationSourceUploadNotFoundError(source.upload_id) + return upload + + +def _validate_draft_result(result: DraftScriptResult) -> None: + """Validate draft result metadata before writing episode TEI.""" + expected_hash = sha256_text(result.tei_xml) + if result.content_hash != expected_hash: + raise DraftContentHashMismatchError(expected_hash, result.content_hash) diff --git a/episodic/canonical/generation_persistence_projection.py b/episodic/canonical/generation_persistence_projection.py new file mode 100644 index 00000000..4d2da295 --- /dev/null +++ b/episodic/canonical/generation_persistence_projection.py @@ -0,0 +1,63 @@ +"""Private source-document projection helpers for generation persistence.""" + +import typing as typ + +from episodic.canonical.domain import SourceDocument +from episodic.canonical.hashing import sha256_text + +if typ.TYPE_CHECKING: + from episodic.canonical.generation_persistence_types import ( + _SourceDocumentProjection, + ) + from episodic.canonical.ingestion_sources import IngestionJobSource + from episodic.canonical.uploads import Upload + + +def source_document_from_attachment( + projection: _SourceDocumentProjection, +) -> SourceDocument: + """Project an intake source attachment into canonical source metadata. + + Parameters + ---------- + projection + Deterministic source-document projection input containing the intake + attachment, optional resolved upload, target episode, identifier, and + creation timestamp. + + Returns + ------- + SourceDocument + Canonical source-document metadata ready for repository persistence. + """ + return SourceDocument( + id=projection.document_id, + ingestion_job_id=projection.source.ingestion_job_id, + canonical_episode_id=projection.episode_id, + reference_document_revision_id=None, + source_type=projection.source.source_type, + source_uri=_source_uri(projection.source, projection.upload), + weight=projection.source.weight, + content_hash=_source_content_hash(projection.source, projection.upload), + metadata=projection.source.metadata, + created_at=projection.now, + ) + + +def _source_uri(source: IngestionJobSource, upload: Upload | None) -> str: + """Return a stable source URI for canonical provenance.""" + if source.source_uri is not None: + return source.source_uri + if upload is not None: + return f"upload:{upload.storage_key}" + return f"upload:{source.upload_id}" + + +def _source_content_hash(source: IngestionJobSource, upload: Upload | None) -> str: + """Return the best available source content hash.""" + match source.metadata.get("content_hash"): + case str() as metadata_hash if metadata_hash.strip(): + return metadata_hash + if upload is not None and upload.content_hash: + return upload.content_hash + return sha256_text(f"{source.source_type}:{_source_uri(source, upload)}") diff --git a/episodic/canonical/generation_persistence_types.py b/episodic/canonical/generation_persistence_types.py new file mode 100644 index 00000000..f1a1fe59 --- /dev/null +++ b/episodic/canonical/generation_persistence_types.py @@ -0,0 +1,257 @@ +"""Commands, typed errors, and projections for draft persistence services.""" + +import collections.abc as cabc +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +if typ.TYPE_CHECKING: + from episodic.canonical.ingestion_sources import IngestionJobSource + from episodic.canonical.uploads import Upload + from episodic.generation.draft_script import DraftScriptResult + +type Clock = cabc.Callable[[], dt.datetime] +type UuidFactory = cabc.Callable[[], uuid.UUID] + + +def _utc_now() -> dt.datetime: + """Return a timezone-aware UTC timestamp.""" + return dt.datetime.now(dt.UTC) + + +def _uuid7() -> uuid.UUID: + """Return a monotonic storage UUID.""" + return uuid.uuid7() + + +class DraftScriptPersistenceError(Exception): + """Base class for draft persistence failures. + + Catch this exception to handle failures raised while materialising an + episode or persisting its generated draft. + """ + + +class _IngestionJobPersistenceError(DraftScriptPersistenceError): + """Base error that retains an ingestion-job identifier for diagnostics.""" + + message_template: typ.ClassVar[str] + + def __init__(self, ingestion_job_id: uuid.UUID) -> None: + """Initialize an ingestion-job persistence error.""" + self.ingestion_job_id = ingestion_job_id + message = self.message_template.format(ingestion_job_id=ingestion_job_id) + super().__init__(message) + + +class IngestionJobNotReadyError(_IngestionJobPersistenceError): + """Raised when materialisation is requested before an intake job is ready. + + Parameters + ---------- + ingestion_job_id : uuid.UUID + Identifier of the ingestion job that is not ready for generation. + + Attributes + ---------- + ingestion_job_id : uuid.UUID + Identifier of the ingestion job retained for diagnostics. + """ + + message_template: typ.ClassVar[str] = ( + "Ingestion job {ingestion_job_id} is not ready for generation." + ) + + +class MissingAttachedSourcesError(_IngestionJobPersistenceError): + """Raised when a ready ingestion job has no source attachments. + + Parameters + ---------- + ingestion_job_id : uuid.UUID + Identifier of the ready ingestion job without source attachments. + + Attributes + ---------- + ingestion_job_id : uuid.UUID + Identifier of the ingestion job retained for diagnostics. + """ + + message_template: typ.ClassVar[str] = ( + "Ingestion job {ingestion_job_id} has no attached sources." + ) + + +class SourceCountLimitExceededError(_IngestionJobPersistenceError): + """Raised when an ingestion job exceeds its configured source limit.""" + + message_template: typ.ClassVar[str] = ( + "Ingestion job {ingestion_job_id} exceeds the generation source limit." + ) + + +class GenerationSourceUploadNotFoundError(DraftScriptPersistenceError): + """Raised when a generation source refers to an absent upload. + + Parameters + ---------- + upload_id : uuid.UUID + Identifier of the upload referenced by the generation source. + + Attributes + ---------- + upload_id : uuid.UUID + Identifier of the missing upload retained for diagnostics. + """ + + def __init__(self, upload_id: uuid.UUID) -> None: + """Initialize an error for a missing generation-source upload. + + Parameters + ---------- + upload_id : uuid.UUID + Identifier of the upload referenced by the generation source. + """ + self.upload_id = upload_id + message = f"Upload {upload_id} was not found for ingestion source." + super().__init__(message) + + +class DraftContentHashMismatchError(DraftScriptPersistenceError): + """Raised when generated TEI and its declared hash disagree. + + Parameters + ---------- + expected_hash : str + SHA-256 hash calculated from the generated TEI. + actual_hash : str + Hash declared by the generated draft result. + + Attributes + ---------- + expected_hash : str + SHA-256 hash calculated from the generated TEI. + actual_hash : str + Hash declared by the generated draft result. + """ + + def __init__(self, expected_hash: str, actual_hash: str) -> None: + """Initialize an error for mismatched generated-content hashes. + + Parameters + ---------- + expected_hash : str + SHA-256 hash calculated from the generated TEI. + actual_hash : str + Hash declared by the generated draft result. + """ + self.expected_hash = expected_hash + self.actual_hash = actual_hash + message = "Draft script content_hash does not match tei_xml." + super().__init__(message) + + +class InvalidDraftTeiError(DraftScriptPersistenceError, ValueError): + """Raised when generated TEI cannot be validated. + + The validation message is supplied to the inherited :class:`ValueError` + constructor and is available through the standard exception arguments. + """ + + +class SourceDocumentProjectionError(DraftScriptPersistenceError): + """Raised when a duplicate projection does not contain every source row. + + Parameters + ---------- + missing_document_ids : collections.abc.Collection[uuid.UUID] + Identifiers expected from the projection but absent after the + duplicate-write race. + + Attributes + ---------- + missing_document_ids : tuple[uuid.UUID, ...] + Missing document identifiers, sorted by their string representation + and retained as an immutable tuple for diagnostics. + """ + + def __init__(self, missing_document_ids: cabc.Collection[uuid.UUID]) -> None: + """Initialize an error for an incomplete source-document projection. + + Parameters + ---------- + missing_document_ids : collections.abc.Collection[uuid.UUID] + Identifiers expected from the projection but absent after the + duplicate-write race. + """ + self.missing_document_ids = tuple(sorted(missing_document_ids, key=str)) + missing_ids = ", ".join( + str(document_id) for document_id in self.missing_document_ids + ) + message = ( + "Source document projection did not complete after a duplicate race: " + f"missing {missing_ids}." + ) + super().__init__(message) + + +@dc.dataclass(frozen=True, slots=True) +class EpisodeMaterialisationRequest: + """Command for materialising an episode from an ingestion job. + + Parameters + ---------- + ingestion_job_id + Ready ingestion job whose attached sources become canonical documents. + title + Initial title for a newly materialised placeholder episode. + clock + UTC clock used for durable placeholder timestamps. + uuid_factory + Identifier factory used only when the job has no target episode. + max_source_count + Maximum attached sources materialised for one generation run. + """ + + ingestion_job_id: uuid.UUID + title: str + clock: Clock = _utc_now + uuid_factory: UuidFactory = _uuid7 + max_source_count: int = 32 + + +@dc.dataclass(frozen=True, slots=True) +class DraftScriptPersistenceRequest: + """Command for writing generated draft TEI to an episode. + + Parameters + ---------- + episode_id + Canonical episode receiving the generated TEI revision. + generation_run_id + Run recorded as provenance for the no-QA update. + result + Validated draft output from :class:`DraftScriptGenerator`. + expected_revision + Revision that must still be current when the TEI is written. + clock + UTC clock used for the persisted update timestamp. + """ + + episode_id: uuid.UUID + generation_run_id: uuid.UUID + result: DraftScriptResult + expected_revision: int + clock: Clock = _utc_now + + +@dc.dataclass(frozen=True, slots=True) +class _SourceDocumentProjection: + """Values needed to turn an intake attachment into a source document.""" + + source: IngestionJobSource + upload: Upload | None + episode_id: uuid.UUID + document_id: uuid.UUID + now: dt.datetime diff --git a/episodic/canonical/generation_quality.py b/episodic/canonical/generation_quality.py new file mode 100644 index 00000000..10c41598 --- /dev/null +++ b/episodic/canonical/generation_quality.py @@ -0,0 +1,15 @@ +"""Generation quality policy and QA outcome values.""" + +import enum + + +class QualityMode(enum.StrEnum): + """Requested generation quality policy for a run.""" + + DRAFT_WITHOUT_QA = "draft_without_qa" + + +class QaStatus(enum.StrEnum): + """Recorded QA outcome for a run and the TEI it produced.""" + + SKIPPED = "skipped" diff --git a/episodic/canonical/generation_run_ports.py b/episodic/canonical/generation_run_ports.py index 35632f68..262110f9 100644 --- a/episodic/canonical/generation_run_ports.py +++ b/episodic/canonical/generation_run_ports.py @@ -12,6 +12,7 @@ async def use_runs(port: GenerationRunPort) -> None: ``` """ +import dataclasses as dc import typing as typ if typ.TYPE_CHECKING: @@ -30,6 +31,32 @@ async def use_runs(port: GenerationRunPort) -> None: EventSeq = typ.NewType("EventSeq", int) +@dc.dataclass(frozen=True, slots=True) +class GenerationRunStatusUpdate: + """Lifecycle fields to apply to a generation run. + + Attributes + ---------- + status : GenerationRunStatus + New durable lifecycle status. + current_node : str | None + Current launcher node, or ``None`` for a terminal run. + ended_at : dt.datetime | None + Terminal timestamp, when the lifecycle transition ends a run. + error_message : str | None + Failure detail. An omitted value clears the persisted field. + error_category : str | None + Stable failure classification. An omitted value clears the persisted + field. + """ + + status: GenerationRunStatus + current_node: str | None + ended_at: dt.datetime | None + error_message: str | None = None + error_category: str | None = None + + def event_seq(value: int) -> EventSeq: """Validate and convert a positive event sequence integer.""" if not isinstance(value, int) or value < 1: @@ -47,8 +74,9 @@ async def create_run( run: GenerationRun, *, idempotency_key: str | None = None, + idempotency_principal_id: str | None = None, ) -> GenerationRun: - """Create a run or return the first run for an idempotency key.""" + """Create a run or return the first run for a principal-scoped key.""" raise NotImplementedError async def get_run(self, run_id: uuid.UUID) -> GenerationRun | None: @@ -67,18 +95,27 @@ async def list_runs( """List runs for an episode, ordered by creation time.""" raise NotImplementedError - # pylint: disable-next=too-many-arguments # Port signature is fixed. async def update_run_status( self, run_id: uuid.UUID, *, - status: GenerationRunStatus, - current_node: str | None, - ended_at: dt.datetime | None, + update: GenerationRunStatusUpdate, ) -> GenerationRun: """Update the run lifecycle state and return the stored run.""" raise NotImplementedError + # pylint: disable-next=too-many-arguments # Port signature is fixed. + async def claim_run_for_execution( + self, + run_id: uuid.UUID, + *, + current_node: str | None, + started_at: dt.datetime, + lease_expires_at: dt.datetime | None, + ) -> GenerationRun | None: + """Atomically move a pending run to running, or return None if lost.""" + raise NotImplementedError + @typ.runtime_checkable class GenerationEventLog(typ.Protocol): @@ -86,7 +123,9 @@ class GenerationEventLog(typ.Protocol): `list_events` returns records ordered ascending by `seq`. When `after_seq` is supplied, the result range is half-open `(after_seq, ...]`; - otherwise it starts from sequence 1. `limit` is a hard cap. + otherwise it starts from sequence 1. `offset` skips records within the + range selected by `after_seq`, and `limit` is a hard cap. Callers must use + either `after_seq` or `offset`, not both. """ # pylint: disable-next=too-many-arguments # Port signature is fixed. @@ -107,10 +146,36 @@ async def list_events( *, after_seq: EventSeq | None = None, limit: int = 100, + offset: int = 0, ) -> tuple[GenerationEvent, ...]: - """List events for a run.""" + """List events for a run. + + Raises + ------ + ValueError + If ``after_seq`` is supplied with a non-zero ``offset``. These + pagination mechanisms are mutually exclusive. + """ raise NotImplementedError + async def count_events( + self, + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + ) -> int: + """Count events for a run after an optional sequence cursor.""" + raise NotImplementedError + + +@typ.runtime_checkable +class GenerationRunEventStore( + GenerationRunRepository, + GenerationEventLog, + typ.Protocol, +): + """Composite port for run persistence plus append-only event logging.""" + @typ.runtime_checkable class GenerationCheckpointPort(typ.Protocol): diff --git a/episodic/canonical/hashing.py b/episodic/canonical/hashing.py new file mode 100644 index 00000000..65592225 --- /dev/null +++ b/episodic/canonical/hashing.py @@ -0,0 +1,21 @@ +"""Canonical content-hash helpers.""" + +import hashlib + + +def sha256_text(value: str) -> str: + """Return the prefixed SHA-256 digest of UTF-8 encoded text. + + Parameters + ---------- + value : str + Text to encode as UTF-8 and hash. The value is accepted as supplied, + including empty text; it is not stripped or otherwise normalised. + + Returns + ------- + str + A string in the exact form ``sha256:<64 lowercase hexadecimal + characters>``. + """ + return f"sha256:{hashlib.sha256(value.encode()).hexdigest()}" diff --git a/episodic/canonical/source_intake_service.py b/episodic/canonical/source_intake_service.py index 9f5a2a90..b46dbd38 100644 --- a/episodic/canonical/source_intake_service.py +++ b/episodic/canonical/source_intake_service.py @@ -243,6 +243,7 @@ async def create_ingestion_job( created_at=now, updated_at=now, intake_state=IntakeState.AWAITING_SOURCES, + owner_principal_id=request.owner_principal_id, ) await uow.ingestion_jobs.add(job) await uow.commit() @@ -260,7 +261,11 @@ async def attach_source_to_ingestion_job( if job is None: raise IngestionJobNotFoundError(str(request.ingestion_job_id)) if request.attachment_kind is AttachmentKind.UPLOAD: - await _require_ready_upload(uow, request.upload_id) + await _require_ready_upload( + uow, + request.upload_id, + owner_principal_id=job.owner_principal_id, + ) source_uri = None else: source_uri = request.source_uri @@ -345,6 +350,8 @@ async def list_ingestion_job_sources( async def _require_ready_upload( uow: CanonicalUnitOfWork, upload_id: uuid.UUID | None, + *, + owner_principal_id: str | None, ) -> Upload: """Return a ready upload or raise the correct source-intake error.""" if upload_id is None: @@ -352,6 +359,11 @@ async def _require_ready_upload( upload = await uow.uploads.get(upload_id) if upload is None: raise UploadNotFoundError(str(upload_id)) + if ( + owner_principal_id is not None + and upload.owner_principal_id != owner_principal_id + ): + raise UploadNotFoundError(str(upload_id)) if upload.state is not UploadState.READY: raise UploadNotReadyError(str(upload_id)) return upload diff --git a/episodic/canonical/source_intake_types.py b/episodic/canonical/source_intake_types.py index 0b7ca266..997a557e 100644 --- a/episodic/canonical/source_intake_types.py +++ b/episodic/canonical/source_intake_types.py @@ -31,6 +31,7 @@ class CreateIngestionJobRequest: series_profile_id: uuid.UUID target_episode_id: uuid.UUID | None + owner_principal_id: str | None = None @dc.dataclass(frozen=True, slots=True) diff --git a/episodic/canonical/storage/__init__.py b/episodic/canonical/storage/__init__.py index b10db236..8faf42f0 100644 --- a/episodic/canonical/storage/__init__.py +++ b/episodic/canonical/storage/__init__.py @@ -12,7 +12,9 @@ ... episode = await uow.episodes.get(episode_id) """ +from .episode_repository import SqlAlchemyEpisodeRepository from .filesystem_object_store import FilesystemObjectStore +from .generation_runs import SqlAlchemyGenerationRunStore from .ingestion_job_repositories import SqlAlchemyIngestionJobRepository from .migration_check import detect_schema_drift from .models import ( @@ -21,6 +23,8 @@ EpisodeRecord, EpisodeTemplateHistoryRecord, EpisodeTemplateRecord, + GenerationEventRecord, + GenerationRunRecord, IdempotencyRecordModel, IngestionJobRecord, IngestionJobSourceRecord, @@ -36,7 +40,6 @@ ) from .repositories import ( SqlAlchemyApprovalEventRepository, - SqlAlchemyEpisodeRepository, SqlAlchemyEpisodeTemplateHistoryRepository, SqlAlchemyEpisodeTemplateRepository, SqlAlchemyReferenceBindingRepository, @@ -62,6 +65,8 @@ "EpisodeTemplateHistoryRecord", "EpisodeTemplateRecord", "FilesystemObjectStore", + "GenerationEventRecord", + "GenerationRunRecord", "IdempotencyRecordModel", "IngestionJobRecord", "IngestionJobSourceRecord", @@ -75,6 +80,7 @@ "SqlAlchemyEpisodeRepository", "SqlAlchemyEpisodeTemplateHistoryRepository", "SqlAlchemyEpisodeTemplateRepository", + "SqlAlchemyGenerationRunStore", "SqlAlchemyIdempotencyStore", "SqlAlchemyIngestionJobRepository", "SqlAlchemyIngestionJobSourceRepository", diff --git a/episodic/canonical/storage/entity_mappers.py b/episodic/canonical/storage/entity_mappers.py index a56fcdeb..70084781 100644 --- a/episodic/canonical/storage/entity_mappers.py +++ b/episodic/canonical/storage/entity_mappers.py @@ -33,6 +33,7 @@ SourceDocument, TeiHeader, ) +from episodic.canonical.hashing import sha256_text from .compression import decode_text_from_storage, encode_text_for_storage from .entity_models import ( @@ -119,6 +120,10 @@ def _episode_from_record(record: EpisodeRecord) -> CanonicalEpisode: approval_state=record.approval_state, created_at=record.created_at, updated_at=record.updated_at, + tei_revision=record.tei_revision, + tei_content_hash=record.tei_content_hash, + qa_status=record.qa_status, + last_generation_run_id=record.last_generation_run_id, ) @@ -132,6 +137,10 @@ def _episode_to_record(episode: CanonicalEpisode) -> EpisodeRecord: title=episode.title, tei_xml=tei_xml, tei_xml_zstd=tei_xml_zstd, + tei_revision=episode.tei_revision, + tei_content_hash=sha256_text(episode.tei_xml), + qa_status=episode.qa_status, + last_generation_run_id=episode.last_generation_run_id, status=episode.status, approval_state=episode.approval_state, created_at=episode.created_at, @@ -153,6 +162,7 @@ def _ingestion_job_from_record(record: IngestionJobRecord) -> IngestionJob: created_at=record.created_at, updated_at=record.updated_at, intake_state=record.intake_state, + owner_principal_id=record.owner_principal_id, ) @@ -170,6 +180,7 @@ def _ingestion_job_to_record(job: IngestionJob) -> IngestionJobRecord: intake_state=job.intake_state, created_at=job.created_at, updated_at=job.updated_at, + owner_principal_id=job.owner_principal_id, ) diff --git a/episodic/canonical/storage/entity_models.py b/episodic/canonical/storage/entity_models.py index a9209d99..4b7ca075 100644 --- a/episodic/canonical/storage/entity_models.py +++ b/episodic/canonical/storage/entity_models.py @@ -13,12 +13,16 @@ IngestionStatus, IntakeState, ) +from episodic.canonical.generation_quality import ( # noqa: TC001 # SQLAlchemy evaluates annotations at runtime. + QaStatus, +) from .models_base import ( APPROVAL_STATE, EPISODE_STATUS, INGESTION_STATUS, INTAKE_STATE, + QA_STATUS, Base, ) @@ -52,13 +56,11 @@ class TeiHeaderRecord(Base): ) title: orm.Mapped[str] = orm.mapped_column(sa.String(240), nullable=False) payload: orm.Mapped[dict[str, object]] = orm.mapped_column( - postgresql.JSONB, - nullable=False, + postgresql.JSONB, nullable=False ) raw_xml: orm.Mapped[str] = orm.mapped_column(sa.Text, nullable=False) raw_xml_zstd: orm.Mapped[bytes | None] = orm.mapped_column( - postgresql.BYTEA, - nullable=True, + postgresql.BYTEA, nullable=True ) created_at: orm.Mapped[dt.datetime] = orm.mapped_column( sa.DateTime(timezone=True), @@ -90,6 +92,14 @@ class EpisodeRecord(Base): Raw TEI XML associated with the episode. tei_xml_zstd : bytes | None Zstandard-compressed TEI XML associated with the episode. + tei_revision : int + Monotonic TEI revision number for the episode. + tei_content_hash : str | None + Content hash for the current TEI payload, when available. + qa_status : QaStatus | None + Quality-assurance status for the current episode content. + last_generation_run_id : uuid.UUID | None + Most recent generation run associated with the episode. status : EpisodeStatus Episode status enum. approval_state : ApprovalState @@ -101,6 +111,12 @@ class EpisodeRecord(Base): """ __tablename__ = "episodes" + __table_args__ = ( + sa.CheckConstraint( + "tei_revision >= 1", + name="ck_episodes_tei_revision_positive", + ), + ) id: orm.Mapped[uuid.UUID] = orm.mapped_column( postgresql.UUID(as_uuid=True), @@ -123,6 +139,24 @@ class EpisodeRecord(Base): postgresql.BYTEA, nullable=True, ) + tei_revision: orm.Mapped[int] = orm.mapped_column( + sa.Integer, + nullable=False, + server_default="1", + ) + tei_content_hash: orm.Mapped[str | None] = orm.mapped_column( + sa.String(128), + nullable=True, + ) + qa_status: orm.Mapped[QaStatus | None] = orm.mapped_column( + QA_STATUS, + nullable=True, + ) + last_generation_run_id: orm.Mapped[uuid.UUID | None] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("generation_runs.id", ondelete="SET NULL"), + nullable=True, + ) status: orm.Mapped[EpisodeStatus] = orm.mapped_column( EPISODE_STATUS, nullable=False, @@ -210,6 +244,9 @@ class IngestionJobRecord(Base): nullable=False, server_default=IntakeState.AWAITING_SOURCES.value, ) + owner_principal_id: orm.Mapped[str | None] = orm.mapped_column( + sa.String(240), nullable=True + ) created_at: orm.Mapped[dt.datetime] = orm.mapped_column( sa.DateTime(timezone=True), nullable=False, @@ -229,6 +266,7 @@ class IngestionJobRecord(Base): "intake_state", sa.desc("created_at"), ), + sa.Index("ix_ij_owner_created", "owner_principal_id", "created_at", "id"), ) @@ -353,9 +391,7 @@ class ApprovalEventRecord(Base): ) note: orm.Mapped[str | None] = orm.mapped_column(sa.Text, nullable=True) payload: orm.Mapped[dict[str, object]] = orm.mapped_column( - postgresql.JSONB, - default=dict, - nullable=False, + postgresql.JSONB, default=dict, nullable=False ) created_at: orm.Mapped[dt.datetime] = orm.mapped_column( sa.DateTime(timezone=True), diff --git a/episodic/canonical/storage/episode_repository.py b/episodic/canonical/storage/episode_repository.py new file mode 100644 index 00000000..59a8fb4e --- /dev/null +++ b/episodic/canonical/storage/episode_repository.py @@ -0,0 +1,119 @@ +"""SQLAlchemy repository for canonical episodes.""" + +import typing as typ + +import sqlalchemy as sa + +from episodic.canonical.entity_protocols import EpisodeRepository +from episodic.canonical.episode_errors import ( + EpisodeNotFoundError, + EpisodeRevisionConflictError, +) +from episodic.canonical.hashing import sha256_text + +from .compression import encode_text_for_storage +from .entity_mappers import _episode_from_record, _episode_to_record +from .entity_models import EpisodeRecord +from .repository_base import _RepositoryBase + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from sqlalchemy.engine import CursorResult + + from episodic.canonical.domain import CanonicalEpisode, EpisodeTeiUpdate + + +class SqlAlchemyEpisodeRepository(_RepositoryBase, EpisodeRepository): + """Persist canonical episodes using SQLAlchemy.""" + + async def add(self, episode: CanonicalEpisode) -> None: + """Add a canonical episode record. + + Parameters + ---------- + episode : CanonicalEpisode + Canonical episode domain entity to persist. + + """ + self._session.add(_episode_to_record(episode)) + + async def get(self, episode_id: uuid.UUID) -> CanonicalEpisode | None: + """Fetch a canonical episode by identifier.""" + return await self._get_one_or_none( + EpisodeRecord, + EpisodeRecord.id == episode_id, + _episode_from_record, + ) + + async def list_by_ids( + self, episode_ids: cabc.Collection[uuid.UUID] + ) -> list[CanonicalEpisode]: + """Fetch canonical episodes by identifiers.""" + if not episode_ids: + return [] + + return await self._get_many( + EpisodeRecord, + EpisodeRecord.id.in_(episode_ids), + _episode_from_record, + ) + + async def update( + self, + episode_id: uuid.UUID, + *, + update: EpisodeTeiUpdate, + ) -> CanonicalEpisode: + """Update episode TEI when the expected revision still matches. + + Parameters + ---------- + episode_id : uuid.UUID + Identifier of the episode to update. + update : EpisodeTeiUpdate + TEI content, provenance, QA metadata, and required revision. + + Returns + ------- + CanonicalEpisode + The stored episode after its revision increases by one. + + Raises + ------ + EpisodeNotFoundError + If no episode exists for ``episode_id``. + EpisodeRevisionConflictError + If the stored revision differs from ``update.expected_revision``. + """ + stored_tei_xml, tei_xml_zstd = encode_text_for_storage(update.tei_xml) + result = await self._session.execute( + sa + .update(EpisodeRecord) + .where( + EpisodeRecord.id == episode_id, + EpisodeRecord.tei_revision == update.expected_revision, + ) + .values( + tei_xml=stored_tei_xml, + tei_xml_zstd=tei_xml_zstd, + tei_revision=update.expected_revision + 1, + tei_content_hash=sha256_text(update.tei_xml), + qa_status=update.qa_status, + last_generation_run_id=update.last_generation_run_id, + updated_at=update.updated_at, + ) + ) + cursor_result = typ.cast("CursorResult[typ.Any]", result) + if cursor_result.rowcount != 1: + existing = await self.get(episode_id) + if existing is None: + raise EpisodeNotFoundError(episode_id) + raise EpisodeRevisionConflictError(episode_id, update.expected_revision) + + await self._session.flush() + updated = await self.get(episode_id) + if updated is None: # pragma: no cover - guarded by updated row. + raise EpisodeNotFoundError(episode_id) + return updated diff --git a/episodic/canonical/storage/filesystem_object_store.py b/episodic/canonical/storage/filesystem_object_store.py index bc6e95fc..8a0e7e86 100644 --- a/episodic/canonical/storage/filesystem_object_store.py +++ b/episodic/canonical/storage/filesystem_object_store.py @@ -68,9 +68,10 @@ def _resolve_sha256( class FilesystemObjectStore(ObjectStorePort): """Store object bytes under a configured filesystem root. - This local adapter intentionally uses blocking filesystem calls inside its - async port methods because it is the development and CI object store. A - production network adapter should provide non-blocking I/O at this port. + The async ``open`` path offloads filesystem open, read, and close calls to + worker threads so source hydration does not block the event loop. This + adapter remains intended for development and CI; production deployments may + use a network-backed adapter through the same port. """ def __init__(self, root: pathlib.Path) -> None: @@ -116,10 +117,19 @@ async def open(self, key: str) -> cabc.AsyncIterator[cabc.AsyncIterator[bytes]]: path = self._resolve_under_root(validate_object_key(key)) async def _chunks() -> cabc.AsyncIterator[bytes]: - with path.open("rb") as file_handle: - while chunk := file_handle.read(_OBJECT_STORE_READ_CHUNK_BYTES): + file_handle = typ.cast( + "typ.BinaryIO", + await asyncio.to_thread(path.open, "rb"), + ) + try: + while chunk := await asyncio.to_thread( + file_handle.read, + _OBJECT_STORE_READ_CHUNK_BYTES, + ): await _yield_checkpoint() yield chunk + finally: + await asyncio.to_thread(file_handle.close) yield _chunks() diff --git a/episodic/canonical/storage/generation_run_mappers.py b/episodic/canonical/storage/generation_run_mappers.py new file mode 100644 index 00000000..bf795a45 --- /dev/null +++ b/episodic/canonical/storage/generation_run_mappers.py @@ -0,0 +1,82 @@ +"""Record-to-domain mappings for generation runs and their events.""" + +from episodic.canonical.domain import GenerationEvent, GenerationRun +from episodic.canonical.generation_run_ports import event_seq + +from .generation_run_models import GenerationEventRecord, GenerationRunRecord + +_ANONYMOUS_IDEMPOTENCY_PRINCIPAL = "__anonymous__" + + +def normalise_idempotency_principal(principal_id: str | None) -> str: + """Return the persisted principal scope for an idempotency key.""" + return principal_id or _ANONYMOUS_IDEMPOTENCY_PRINCIPAL + + +def run_from_record(record: GenerationRunRecord) -> GenerationRun: + """Map a generation-run record to a domain entity.""" + return GenerationRun( + id=record.id, + episode_id=record.episode_id, + source_bundle_id=record.source_bundle_id, + actor=record.actor, + status=record.status, + current_node=record.current_node, + budget_snapshot=record.budget_snapshot, + configuration=record.configuration, + created_at=record.created_at, + updated_at=record.updated_at, + started_at=record.started_at, + ended_at=record.ended_at, + error_message=record.error_message, + error_category=record.error_category, + quality_mode=record.quality_mode, + qa_status=record.qa_status, + skip_qa_rationale=record.skip_qa_rationale, + ) + + +def run_to_record( + run: GenerationRun, + *, + idempotency_key: str | None, + idempotency_principal_id: str | None, +) -> GenerationRunRecord: + """Map a generation-run domain entity to a SQLAlchemy record.""" + return GenerationRunRecord( + id=run.id, + episode_id=run.episode_id, + source_bundle_id=run.source_bundle_id, + actor=run.actor, + status=run.status, + current_node=run.current_node, + budget_snapshot=run.budget_snapshot, + configuration=run.configuration, + quality_mode=run.quality_mode, + qa_status=run.qa_status, + skip_qa_rationale=run.skip_qa_rationale, + idempotency_principal_id=normalise_idempotency_principal( + idempotency_principal_id + ), + idempotency_key=idempotency_key, + error_message=run.error_message, + error_category=run.error_category, + lease_expires_at=None, + started_at=run.started_at, + ended_at=run.ended_at, + created_at=run.created_at, + updated_at=run.updated_at, + ) + + +def event_from_record(record: GenerationEventRecord) -> GenerationEvent: + """Map an event record to a domain event.""" + return GenerationEvent( + id=record.id, + generation_run_id=record.generation_run_id, + seq=event_seq(record.seq), + kind=record.kind, + payload=record.payload, + occurred_at=record.occurred_at, + created_at=record.created_at, + ) diff --git a/episodic/canonical/storage/generation_run_models.py b/episodic/canonical/storage/generation_run_models.py new file mode 100644 index 00000000..1fd4460e --- /dev/null +++ b/episodic/canonical/storage/generation_run_models.py @@ -0,0 +1,155 @@ +"""SQLAlchemy models for durable generation runs and event logs.""" + +import datetime as dt # noqa: TC003 # SQLAlchemy evaluates annotations at runtime. +import uuid # noqa: TC003 # SQLAlchemy evaluates annotations at runtime. + +import sqlalchemy as sa +from sqlalchemy import orm +from sqlalchemy.dialects import postgresql + +from episodic.canonical.domain import ( # noqa: TC001 # SQLAlchemy evaluates annotations at runtime. + GenerationRunStatus, + JsonMapping, +) +from episodic.canonical.generation_quality import ( # noqa: TC001 # SQLAlchemy evaluates annotations at runtime. + QaStatus, + QualityMode, +) + +from .models_base import GENERATION_RUN_STATUS, QA_STATUS, QUALITY_MODE, Base + + +class GenerationRunRecord(Base): + """SQLAlchemy model for first-class generation-run resources.""" + + __tablename__ = "generation_runs" + + id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + primary_key=True, + ) + episode_id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("episodes.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + source_bundle_id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("ingestion_jobs.id"), + nullable=False, + ) + actor: orm.Mapped[str] = orm.mapped_column(sa.String(240), nullable=False) + status: orm.Mapped[GenerationRunStatus] = orm.mapped_column( + GENERATION_RUN_STATUS, + nullable=False, + index=True, + ) + current_node: orm.Mapped[str | None] = orm.mapped_column( + sa.String(160), + nullable=True, + ) + budget_snapshot: orm.Mapped[JsonMapping] = orm.mapped_column( + postgresql.JSONB, + nullable=False, + ) + configuration: orm.Mapped[JsonMapping] = orm.mapped_column( + postgresql.JSONB, + nullable=False, + ) + quality_mode: orm.Mapped[QualityMode] = orm.mapped_column( + QUALITY_MODE, + nullable=False, + ) + qa_status: orm.Mapped[QaStatus | None] = orm.mapped_column( + QA_STATUS, + nullable=True, + ) + skip_qa_rationale: orm.Mapped[str | None] = orm.mapped_column( + sa.Text, + nullable=True, + ) + idempotency_principal_id: orm.Mapped[str] = orm.mapped_column( + sa.String(200), + nullable=False, + ) + idempotency_key: orm.Mapped[str | None] = orm.mapped_column( + sa.String(512), + nullable=True, + ) + error_message: orm.Mapped[str | None] = orm.mapped_column(sa.Text, nullable=True) + error_category: orm.Mapped[str | None] = orm.mapped_column( + sa.String(120), + nullable=True, + ) + lease_expires_at: orm.Mapped[dt.datetime | None] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=True, + ) + started_at: orm.Mapped[dt.datetime | None] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=True, + index=True, + ) + ended_at: orm.Mapped[dt.datetime | None] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=True, + ) + created_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ) + updated_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ) + + __table_args__ = ( + sa.UniqueConstraint( + "idempotency_principal_id", + "idempotency_key", + name="uq_generation_runs_idempotency_principal_key", + ), + ) + + +class GenerationEventRecord(Base): + """SQLAlchemy model for append-only generation-run events.""" + + __tablename__ = "generation_events" + __table_args__ = ( + sa.UniqueConstraint( + "generation_run_id", + "seq", + name="uq_generation_events_run_seq", + ), + ) + + id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + primary_key=True, + ) + generation_run_id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("generation_runs.id"), + nullable=False, + index=True, + ) + seq: orm.Mapped[int] = orm.mapped_column(sa.Integer, nullable=False) + kind: orm.Mapped[str] = orm.mapped_column(sa.String(160), nullable=False) + payload: orm.Mapped[JsonMapping] = orm.mapped_column( + postgresql.JSONB, + nullable=False, + ) + occurred_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + ) + created_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ) diff --git a/episodic/canonical/storage/generation_run_storage_runtime.py b/episodic/canonical/storage/generation_run_storage_runtime.py new file mode 100644 index 00000000..ac37db00 --- /dev/null +++ b/episodic/canonical/storage/generation_run_storage_runtime.py @@ -0,0 +1,54 @@ +"""Provide injectable runtime dependencies for generation-run storage. + +The provider bundle isolates adapter-owned UTC timestamps and event identifiers +from SQLAlchemy persistence. Production callers use the default bundle, while +tests may provide deterministic collaborators through the unit of work. +""" + +import collections.abc as cabc +import dataclasses as dc +import datetime as dt +import uuid + +type Clock = cabc.Callable[[], dt.datetime] +type UuidFactory = cabc.Callable[[], uuid.UUID] + + +@dc.dataclass(frozen=True, slots=True) +class GenerationRunStorageRuntime: + """Runtime providers used by the generation-run SQLAlchemy adapter. + + Attributes + ---------- + clock : Clock + UTC timestamp provider for adapter-owned lifecycle writes. + uuid_factory : UuidFactory + Identifier provider for appended durable events. + """ + + clock: Clock + uuid_factory: UuidFactory + + +def generation_run_storage_runtime( + runtime: GenerationRunStorageRuntime | None, +) -> GenerationRunStorageRuntime: + """Return generation-run providers with production defaults. + + Parameters + ---------- + runtime + Optional injected providers. When omitted, production UTC-clock and + UUIDv7 providers are created. + + Returns + ------- + GenerationRunStorageRuntime + The supplied runtime, or the production-default provider bundle. + """ + if runtime is not None: + return runtime + return GenerationRunStorageRuntime( + clock=lambda: dt.datetime.now(dt.UTC), + uuid_factory=uuid.uuid7, + ) diff --git a/episodic/canonical/storage/generation_runs.py b/episodic/canonical/storage/generation_runs.py new file mode 100644 index 00000000..98c4d99c --- /dev/null +++ b/episodic/canonical/storage/generation_runs.py @@ -0,0 +1,367 @@ +"""SQLAlchemy adapter for durable generation runs and event logs.""" + +import typing as typ + +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError + +from episodic.canonical.domain import ( + GenerationEvent, + GenerationRun, + GenerationRunStatus, + JsonMapping, +) +from episodic.canonical.generation_run_errors import RunAlreadyTerminal, RunNotFound +from episodic.orchestration._types import _log_event + +from .generation_run_mappers import ( + event_from_record, + normalise_idempotency_principal, + run_from_record, + run_to_record, +) +from .generation_run_models import GenerationEventRecord, GenerationRunRecord +from .generation_run_storage_runtime import generation_run_storage_runtime + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + + from sqlalchemy.engine import CursorResult + from sqlalchemy.ext.asyncio import AsyncSession + + from episodic.canonical.generation_run_ports import ( + EventSeq, + GenerationRunStatusUpdate, + ) + from episodic.canonical.storage.generation_run_storage_runtime import ( + GenerationRunStorageRuntime, + ) + + +def _minimum_event_sequence( + after_seq: EventSeq | None, + *, + limit: int, + offset: int, +) -> int: + """Validate event-page arguments and return the exclusive sequence bound.""" + if limit < 0 or offset < 0: + msg = "limit and offset must be non-negative." + raise ValueError(msg) + if after_seq is not None and offset != 0: + msg = "after_seq and offset cannot be combined." + raise ValueError(msg) + return int(after_seq) if after_seq is not None else 0 + + +class SqlAlchemyGenerationRunStore: + """Durable generation-run repository and event-log adapter.""" + + def __init__( + self, + session: "AsyncSession", # noqa: UP037 # Imported only during type checking. + *, + runtime: GenerationRunStorageRuntime | None = None, + ) -> None: + self._session = session + self._runtime = generation_run_storage_runtime(runtime) + + async def _get_record(self, run_id: uuid.UUID) -> GenerationRunRecord | None: + """Return the storage record for a run id.""" + return await self._session.get(GenerationRunRecord, run_id) + + async def _require_mutable_run( + self, + run_id: uuid.UUID, + *, + lock: bool = False, + ) -> GenerationRunRecord: + """Return a non-terminal run record or raise the domain error.""" + if lock: + result = await self._session.execute( + sa + .select(GenerationRunRecord) + .where(GenerationRunRecord.id == run_id) + .with_for_update() + ) + record = result.scalar_one_or_none() + else: + record = await self._get_record(run_id) + if record is None: + raise RunNotFound(run_id) + if record.status.is_terminal(): + raise RunAlreadyTerminal(run_id) + return record + + async def _get_by_idempotency_key( + self, + idempotency_principal_id: str | None, + idempotency_key: str, + ) -> GenerationRun | None: + """Return the first run for an idempotency key.""" + result = await self._session.execute( + sa.select(GenerationRunRecord).where( + GenerationRunRecord.idempotency_principal_id + == normalise_idempotency_principal(idempotency_principal_id), + GenerationRunRecord.idempotency_key == idempotency_key, + ) + ) + record = result.scalar_one_or_none() + if record is None: + return None + return run_from_record(record) + + async def create_run( + self, + run: GenerationRun, + *, + idempotency_key: str | None = None, + idempotency_principal_id: str | None = None, + ) -> GenerationRun: + """Create a run, reusing the first run for an idempotency key.""" + if idempotency_key is not None: + existing = await self._get_by_idempotency_key( + idempotency_principal_id, + idempotency_key, + ) + if existing is not None: + return existing + + record = run_to_record( + run, + idempotency_key=idempotency_key, + idempotency_principal_id=idempotency_principal_id, + ) + try: + async with self._session.begin_nested(): + self._session.add(record) + await self._session.flush() + except IntegrityError: + if idempotency_key is None: + raise + existing = await self._get_by_idempotency_key( + idempotency_principal_id, + idempotency_key, + ) + if existing is not None: + return existing + raise + _log_event( + "info", + "sql_generation_run_store.create_run", + run_id=str(run.id), + episode_id=str(run.episode_id), + idempotent=idempotency_key is not None, + ) + return run_from_record(record) + + async def get_run(self, run_id: uuid.UUID) -> GenerationRun | None: + """Return a run by identifier.""" + record = await self._get_record(run_id) + if record is None: + return None + return run_from_record(record) + + async def list_runs( + self, + episode_id: uuid.UUID, + *, + status: GenerationRunStatus | None = None, + limit: int = 50, + offset: int = 0, + ) -> tuple[GenerationRun, ...]: + """List runs for one episode in creation order.""" + if limit < 0 or offset < 0: + msg = "limit and offset must be non-negative." + raise ValueError(msg) + statement = ( + sa + .select(GenerationRunRecord) + .where(GenerationRunRecord.episode_id == episode_id) + .order_by(GenerationRunRecord.created_at, GenerationRunRecord.id) + .limit(limit) + .offset(offset) + ) + if status is not None: + statement = statement.where(GenerationRunRecord.status == status) + result = await self._session.execute(statement) + return tuple(run_from_record(record) for record in result.scalars()) + + async def update_run_status( + self, + run_id: uuid.UUID, + *, + update: GenerationRunStatusUpdate, + ) -> GenerationRun: + """Update lifecycle fields for a run.""" + record = await self._require_mutable_run(run_id, lock=True) + record.status = update.status + record.current_node = update.current_node + record.ended_at = update.ended_at + record.error_message = update.error_message + record.error_category = update.error_category + record.updated_at = self._runtime.clock() + await self._session.flush() + await self._session.refresh(record) + _log_event( + "info", + "sql_generation_run_store.update_run_status", + run_id=str(run_id), + status=update.status.value, + current_node=update.current_node, + ) + return run_from_record(record) + + async def claim_run_for_execution( + self, + run_id: uuid.UUID, + *, + current_node: str | None, + started_at: dt.datetime, + lease_expires_at: dt.datetime | None, + ) -> GenerationRun | None: + """Atomically move a pending run to running, or return None if lost.""" + now = self._runtime.clock() + result = await self._session.execute( + sa + .update(GenerationRunRecord) + .where( + GenerationRunRecord.id == run_id, + GenerationRunRecord.status == GenerationRunStatus.PENDING, + ) + .values( + status=GenerationRunStatus.RUNNING, + current_node=current_node, + started_at=started_at, + lease_expires_at=lease_expires_at, + updated_at=now, + ) + ) + cursor_result = typ.cast("CursorResult[typ.Any]", result) + if cursor_result.rowcount == 1: + record = await self._get_record(run_id) + if record is None: # pragma: no cover - guarded by updated row. + raise RunNotFound(run_id) + _log_event( + "info", + "sql_generation_run_store.claim_run", + run_id=str(run_id), + current_node=current_node, + lease_expires_at=lease_expires_at.isoformat() + if lease_expires_at is not None + else None, + ) + return run_from_record(record) + + record = await self._get_record(run_id) + if record is None: + _log_event( + "warning", + "sql_generation_run_store.claim_run_missing", + run_id=str(run_id), + ) + raise RunNotFound(run_id) + if record.status.is_terminal(): + _log_event( + "warning", + "sql_generation_run_store.claim_run_terminal", + run_id=str(run_id), + status=record.status.value, + ) + raise RunAlreadyTerminal(run_id) + _log_event( + "info", + "sql_generation_run_store.claim_run_lost", + run_id=str(run_id), + status=record.status.value, + ) + return None + + async def append_event( + self, + run_id: uuid.UUID, + *, + kind: str, + payload: JsonMapping, + occurred_at: dt.datetime | None = None, + ) -> GenerationEvent: + """Append an event with an adapter-allocated sequence.""" + await self._require_mutable_run(run_id, lock=True) + now = self._runtime.clock() + max_seq = await self._session.scalar( + sa.select(sa.func.max(GenerationEventRecord.seq)).where( + GenerationEventRecord.generation_run_id == run_id + ) + ) + record = GenerationEventRecord( + id=self._runtime.uuid_factory(), + generation_run_id=run_id, + seq=(max_seq or 0) + 1, + kind=kind, + payload=payload, + occurred_at=occurred_at or now, + created_at=now, + ) + self._session.add(record) + await self._session.flush() + await self._session.refresh(record) + _log_event( + "info", + "sql_generation_run_store.append_event", + run_id=str(run_id), + event_id=str(record.id), + seq=record.seq, + kind=kind, + ) + return event_from_record(record) + + async def list_events( + self, + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + limit: int = 100, + offset: int = 0, + ) -> tuple[GenerationEvent, ...]: + """List events for a run after an optional sequence cursor.""" + minimum_seq = _minimum_event_sequence( + after_seq, + limit=limit, + offset=offset, + ) + if await self._get_record(run_id) is None: + raise RunNotFound(run_id) + result = await self._session.execute( + sa + .select(GenerationEventRecord) + .where( + GenerationEventRecord.generation_run_id == run_id, + GenerationEventRecord.seq > minimum_seq, + ) + .order_by(GenerationEventRecord.seq) + .limit(limit) + .offset(offset) + ) + return tuple(event_from_record(record) for record in result.scalars()) + + async def count_events( + self, + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + ) -> int: + """Count events for a run after an optional sequence cursor.""" + if await self._get_record(run_id) is None: + raise RunNotFound(run_id) + minimum_seq = int(after_seq) if after_seq is not None else 0 + count = await self._session.scalar( + sa + .select(sa.func.count()) + .select_from(GenerationEventRecord) + .where( + GenerationEventRecord.generation_run_id == run_id, + GenerationEventRecord.seq > minimum_seq, + ) + ) + return typ.cast("int", count) diff --git a/episodic/canonical/storage/ingestion_job_repositories.py b/episodic/canonical/storage/ingestion_job_repositories.py index c1fc5fb3..e9ec83be 100644 --- a/episodic/canonical/storage/ingestion_job_repositories.py +++ b/episodic/canonical/storage/ingestion_job_repositories.py @@ -12,6 +12,7 @@ if typ.TYPE_CHECKING: import collections.abc as cabc + import datetime as dt import uuid from episodic.canonical.domain import ( @@ -36,6 +37,47 @@ async def get(self, job_id: uuid.UUID) -> IngestionJob | None: _ingestion_job_from_record, ) + async def get_for_update(self, job_id: uuid.UUID) -> IngestionJob | None: + """Fetch and lock an ingestion job for transactional mutation.""" + result = await self._session.execute( + sa + .select(IngestionJobRecord) + .where(IngestionJobRecord.id == job_id) + .with_for_update() + ) + record = result.scalar_one_or_none() + return None if record is None else _ingestion_job_from_record(record) + + async def set_target_episode( + self, + job_id: uuid.UUID, + *, + episode_id: uuid.UUID, + updated_at: dt.datetime, + ) -> None: + """Associate an ingestion job with its materialised episode. + + Parameters + ---------- + job_id + Ingestion-job identifier for the SQL ``UPDATE``. + episode_id + Canonical episode identifier to persist. + updated_at + Caller-supplied durable update timestamp. + + Notes + ----- + The caller owns transaction boundaries. The SQL ``UPDATE`` does not + validate rowcount, so an unknown ``job_id`` affects zero rows. + """ + await self._session.execute( + sa + .update(IngestionJobRecord) + .where(IngestionJobRecord.id == job_id) + .values(target_episode_id=episode_id, updated_at=updated_at) + ) + async def list_paged( self, filters: IngestionJobListFilters, @@ -102,4 +144,8 @@ def _ingestion_job_filter_clause( ) if filters.intake_state is not None: clauses.append(IngestionJobRecord.intake_state == filters.intake_state) + if filters.owner_principal_id is not None: + clauses.append( + IngestionJobRecord.owner_principal_id == filters.owner_principal_id + ) return sa.and_(*clauses) if clauses else sa.true() diff --git a/episodic/canonical/storage/integrity_helpers.py b/episodic/canonical/storage/integrity_helpers.py index 8ec41a27..a173dcc9 100644 --- a/episodic/canonical/storage/integrity_helpers.py +++ b/episodic/canonical/storage/integrity_helpers.py @@ -95,6 +95,13 @@ def is_revision_conflict_integrity_error( ) +def is_source_document_duplicate_integrity_error(exc: IntegrityError) -> bool: + """Return whether ``exc`` is the deterministic source-document ID conflict.""" + if constraint_name(exc) == "source_documents_pkey": + return True + return "UNIQUE constraint failed: source_documents.id" in str(exc.orig) + + async def insert_with_conflict_translation( session: AsyncSession, record: object, diff --git a/episodic/canonical/storage/models.py b/episodic/canonical/storage/models.py index 44c6d150..57e6895a 100644 --- a/episodic/canonical/storage/models.py +++ b/episodic/canonical/storage/models.py @@ -20,14 +20,18 @@ SourceDocumentRecord, TeiHeaderRecord, ) +from .generation_run_models import GenerationEventRecord, GenerationRunRecord from .history_models import EpisodeTemplateHistoryRecord, SeriesProfileHistoryRecord from .models_base import ( APPROVAL_STATE, ATTACHMENT_KIND, EPISODE_STATUS, + GENERATION_RUN_STATUS, IDEMPOTENCY_STATE, INGESTION_STATUS, INTAKE_STATE, + QA_STATUS, + QUALITY_MODE, REFERENCE_BINDING_TARGET_KIND, REFERENCE_DOCUMENT_KIND, REFERENCE_DOCUMENT_LIFECYCLE_STATE, @@ -52,9 +56,12 @@ "APPROVAL_STATE", "ATTACHMENT_KIND", "EPISODE_STATUS", + "GENERATION_RUN_STATUS", "IDEMPOTENCY_STATE", "INGESTION_STATUS", "INTAKE_STATE", + "QA_STATUS", + "QUALITY_MODE", "REFERENCE_BINDING_TARGET_KIND", "REFERENCE_DOCUMENT_KIND", "REFERENCE_DOCUMENT_LIFECYCLE_STATE", @@ -66,6 +73,8 @@ "EpisodeRecord", "EpisodeTemplateHistoryRecord", "EpisodeTemplateRecord", + "GenerationEventRecord", + "GenerationRunRecord", "IdempotencyRecordModel", "IngestionJobRecord", "IngestionJobSourceRecord", diff --git a/episodic/canonical/storage/models_base.py b/episodic/canonical/storage/models_base.py index d3974b55..0852d624 100644 --- a/episodic/canonical/storage/models_base.py +++ b/episodic/canonical/storage/models_base.py @@ -6,6 +6,7 @@ from episodic.canonical.domain import ( ApprovalState, EpisodeStatus, + GenerationRunStatus, IngestionStatus, IntakeState, ReferenceBindingTargetKind, @@ -13,6 +14,7 @@ ReferenceDocumentLifecycleState, WorkflowCheckpointStatus, ) +from episodic.canonical.generation_quality import QaStatus, QualityMode from episodic.canonical.idempotency import IdempotencyState from episodic.canonical.ingestion_sources import AttachmentKind from episodic.canonical.uploads import UploadState @@ -83,3 +85,18 @@ class Base(orm.DeclarativeBase): name="workflow_checkpoint_status", values_callable=lambda enum_cls: [item.value for item in enum_cls], ) +GENERATION_RUN_STATUS = sa.Enum( + GenerationRunStatus, + name="generation_run_status", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) +QUALITY_MODE = sa.Enum( + QualityMode, + name="quality_mode", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) +QA_STATUS = sa.Enum( + QaStatus, + name="qa_status", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) diff --git a/episodic/canonical/storage/repositories.py b/episodic/canonical/storage/repositories.py index 86b1ea85..ab026d61 100644 --- a/episodic/canonical/storage/repositories.py +++ b/episodic/canonical/storage/repositories.py @@ -20,10 +20,9 @@ from episodic.canonical.entity_protocols import ( ApprovalEventRepository, - EpisodeRepository, EpisodeTemplateRepository, - IngestionJobRepository, SeriesProfileRepository, + SourceDocumentProjectionResult, SourceDocumentRepository, TeiHeaderRepository, ) @@ -31,12 +30,8 @@ from .entity_mappers import ( _approval_event_from_record, _approval_event_to_record, - _episode_from_record, _episode_template_from_record, _episode_template_to_record, - _episode_to_record, - _ingestion_job_from_record, - _ingestion_job_to_record, _series_profile_from_record, _series_profile_to_record, _source_document_from_record, @@ -46,8 +41,6 @@ ) from .entity_models import ( ApprovalEventRecord, - EpisodeRecord, - IngestionJobRecord, SourceDocumentRecord, TeiHeaderRecord, ) @@ -55,6 +48,10 @@ SqlAlchemyEpisodeTemplateHistoryRepository, SqlAlchemySeriesProfileHistoryRepository, ) +from .integrity_helpers import ( + insert_with_conflict_translation, + is_source_document_duplicate_integrity_error, +) from .profile_models import EpisodeTemplateRecord, SeriesProfileRecord from .reference_repositories import ( SqlAlchemyReferenceBindingRepository, @@ -67,17 +64,21 @@ import collections.abc as cabc import uuid + from sqlalchemy.exc import IntegrityError + from episodic.canonical.domain import ( ApprovalEvent, - CanonicalEpisode, EpisodeTemplate, - IngestionJob, SeriesProfile, SourceDocument, TeiHeader, ) +class _DuplicateProjectionError(Exception): + """Signal a recognised source-document projection write race.""" + + class SqlAlchemySeriesProfileRepository(_RepositoryBase, SeriesProfileRepository): """Persist series profiles using SQLAlchemy.""" @@ -173,65 +174,6 @@ async def get(self, header_id: uuid.UUID) -> TeiHeader | None: ) -class SqlAlchemyEpisodeRepository(_RepositoryBase, EpisodeRepository): - """Persist canonical episodes using SQLAlchemy.""" - - async def add(self, episode: CanonicalEpisode) -> None: - """Add a canonical episode record. - - Parameters - ---------- - episode : CanonicalEpisode - Canonical episode domain entity to persist. - - """ - self._session.add(_episode_to_record(episode)) - - async def get(self, episode_id: uuid.UUID) -> CanonicalEpisode | None: - """Fetch a canonical episode by identifier.""" - return await self._get_one_or_none( - EpisodeRecord, - EpisodeRecord.id == episode_id, - _episode_from_record, - ) - - async def list_by_ids( - self, episode_ids: cabc.Collection[uuid.UUID] - ) -> list[CanonicalEpisode]: - """Fetch canonical episodes by identifiers.""" - if not episode_ids: - return [] - - return await self._get_many( - EpisodeRecord, - EpisodeRecord.id.in_(episode_ids), - _episode_from_record, - ) - - -class SqlAlchemyIngestionJobRepository(_RepositoryBase, IngestionJobRepository): - """Persist ingestion jobs using SQLAlchemy.""" - - async def add(self, job: IngestionJob) -> None: - """Add an ingestion job record. - - Parameters - ---------- - job : IngestionJob - Ingestion job domain entity to persist. - - """ - self._session.add(_ingestion_job_to_record(job)) - - async def get(self, job_id: uuid.UUID) -> IngestionJob | None: - """Fetch an ingestion job by identifier.""" - return await self._get_one_or_none( - IngestionJobRecord, - IngestionJobRecord.id == job_id, - _ingestion_job_from_record, - ) - - class SqlAlchemySourceDocumentRepository(_RepositoryBase, SourceDocumentRepository): """Persist source documents using SQLAlchemy.""" @@ -246,6 +188,43 @@ async def add(self, document: SourceDocument) -> None: """ self._session.add(_source_document_to_record(document)) + async def add_projection( + self, + document: SourceDocument, + ) -> SourceDocumentProjectionResult: + """Insert one projection in a savepoint and report a duplicate ID race. + + Parameters + ---------- + document : SourceDocument + Deterministically identified source document to persist. + + Returns + ------- + SourceDocumentProjectionResult + ``ADDED`` after insertion or ``DUPLICATE`` when a concurrent + projection with the same deterministic identifier already exists. + + Notes + ----- + Unrecognised ``IntegrityError`` failures propagate unchanged. + """ + + def _translate(error: IntegrityError) -> BaseException | None: + if is_source_document_duplicate_integrity_error(error): + return _DuplicateProjectionError() + return None + + try: + await insert_with_conflict_translation( + self._session, + _source_document_to_record(document), + translate=_translate, + ) + except _DuplicateProjectionError: + return SourceDocumentProjectionResult.DUPLICATE + return SourceDocumentProjectionResult.ADDED + async def list_for_job(self, job_id: uuid.UUID) -> list[SourceDocument]: """List source documents for an ingestion job. @@ -382,10 +361,8 @@ async def update(self, template: EpisodeTemplate) -> None: __all__ = ( "SqlAlchemyApprovalEventRepository", - "SqlAlchemyEpisodeRepository", "SqlAlchemyEpisodeTemplateHistoryRepository", "SqlAlchemyEpisodeTemplateRepository", - "SqlAlchemyIngestionJobRepository", "SqlAlchemyReferenceBindingRepository", "SqlAlchemyReferenceDocumentRepository", "SqlAlchemyReferenceDocumentRevisionRepository", diff --git a/episodic/canonical/storage/uow.py b/episodic/canonical/storage/uow.py index 4529d2c0..e49821a3 100644 --- a/episodic/canonical/storage/uow.py +++ b/episodic/canonical/storage/uow.py @@ -15,12 +15,14 @@ import typing as typ from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork +from episodic.cost.storage import SqlAlchemyCostLedgerStore from episodic.logging import get_logger +from .episode_repository import SqlAlchemyEpisodeRepository +from .generation_runs import SqlAlchemyGenerationRunStore from .ingestion_job_repositories import SqlAlchemyIngestionJobRepository from .repositories import ( SqlAlchemyApprovalEventRepository, - SqlAlchemyEpisodeRepository, SqlAlchemyEpisodeTemplateHistoryRepository, SqlAlchemyEpisodeTemplateRepository, SqlAlchemyReferenceBindingRepository, @@ -47,6 +49,8 @@ from episodic.observability import MetricsPort, MonotonicClockPort + from .generation_run_storage_runtime import GenerationRunStorageRuntime + logger = get_logger(__name__) @@ -90,6 +94,10 @@ class SqlAlchemyUnitOfWork(CanonicalUnitOfWork): Repository for immutable reusable reference document revisions. reference_bindings : SqlAlchemyReferenceBindingRepository Repository for reusable reference binding persistence. + generation_runs : SqlAlchemyGenerationRunStore + Repository and event log for durable generation runs. + cost_ledger : SqlAlchemyCostLedgerStore + Cost ledger adapter bound to the same session. """ def __init__( @@ -98,10 +106,12 @@ def __init__( *, metrics: MetricsPort | None = None, clock: MonotonicClockPort | None = None, + generation_run_runtime: GenerationRunStorageRuntime | None = None, ) -> None: self._session_factory = session_factory self._metrics = metrics self._clock = clock + self._generation_run_runtime = generation_run_runtime self._session: AsyncSession | None = None async def __aenter__(self) -> SqlAlchemyUnitOfWork: @@ -143,6 +153,11 @@ async def __aenter__(self) -> SqlAlchemyUnitOfWork: monotonic_clock=self._clock, ), ) + self.generation_runs = SqlAlchemyGenerationRunStore( + self._session, + runtime=self._generation_run_runtime, + ) + self.cost_ledger = SqlAlchemyCostLedgerStore(self._session) self.workflow_checkpoints = SqlAlchemyWorkflowCheckpointStore( self._session, metrics=self._metrics, diff --git a/episodic/canonical/unit_of_work_protocols.py b/episodic/canonical/unit_of_work_protocols.py index 4be3d673..766c4686 100644 --- a/episodic/canonical/unit_of_work_protocols.py +++ b/episodic/canonical/unit_of_work_protocols.py @@ -5,6 +5,8 @@ if typ.TYPE_CHECKING: from types import TracebackType + from episodic.cost.ports import CostLedgerPort + from .entity_protocols import ( ApprovalEventRepository, EpisodeRepository, @@ -14,6 +16,7 @@ SourceDocumentRepository, TeiHeaderRepository, ) + from .generation_run_ports import GenerationRunEventStore from .history_protocols import ( EpisodeTemplateHistoryRepository, SeriesProfileHistoryRepository, @@ -49,6 +52,8 @@ class CanonicalUnitOfWork(typ.Protocol): uploads: UploadRepository ingestion_job_sources: IngestionJobSourceRepository idempotency: IdempotencyStore + generation_runs: GenerationRunEventStore + cost_ledger: CostLedgerPort async def __aenter__(self) -> CanonicalUnitOfWork: """Enter the unit-of-work context.""" diff --git a/episodic/cost/__init__.py b/episodic/cost/__init__.py index 62232ca4..7d553db6 100644 --- a/episodic/cost/__init__.py +++ b/episodic/cost/__init__.py @@ -38,11 +38,13 @@ TaskRollupLedgerEntry, UsageSource, ) +from episodic.cost.recorder import CostRecorderPort __all__ = [ "BillingPeriodKey", "CostLedgerEntryId", "CostLedgerPort", + "CostRecorderPort", "CurrencyCode", "IdempotencyKey", "LedgerScope", diff --git a/episodic/cost/recorder.py b/episodic/cost/recorder.py index 608573f0..6aec699c 100644 --- a/episodic/cost/recorder.py +++ b/episodic/cost/recorder.py @@ -69,6 +69,35 @@ class CostProviderOperation: operation: str +@typ.runtime_checkable +class CostRecorderPort(typ.Protocol): + """Application port for recording the costs of one workflow run.""" + + async def pin_run_pricing( + self, + workflow_run_id: str, + providers: tuple[CostProviderOperation, ...], + billing_period_key: BillingPeriodKey, + ) -> None: + """Pin pricing snapshots before recording provider calls.""" + raise NotImplementedError + + async def record_provider_call( + self, + record: ProviderCallRecord, + ) -> CostLedgerEntryId: + """Record one priced provider call and return its ledger entry id.""" + raise NotImplementedError + + async def finalize_run( + self, + workflow_run_id: str, + workflow_node: str | None, + ) -> CostLedgerEntryId: + """Write the final run roll-up and return its ledger entry id.""" + raise NotImplementedError + + @dc.dataclass(frozen=True, slots=True) class CostRecorder: """Coordinate pricing and ledger writes for orchestration code.""" diff --git a/episodic/generation/__init__.py b/episodic/generation/__init__.py index ec06977d..a5854eaf 100644 --- a/episodic/generation/__init__.py +++ b/episodic/generation/__init__.py @@ -1,7 +1,10 @@ -"""Content generation services package. +"""Draft-generation, launch, source-limit, and TEI-enrichment services. -This package contains services for enriching canonical TEI content with generated -metadata: show notes, chapter markers, guest biographies, and sponsor reads. +``DraftScriptGenerator`` turns bounded canonical sources and presenter context +into a TEI-P5 draft. ``InProcessGenerationRunLauncher`` owns its detached +generation lifecycle and enforces ``GenerationSourceLimits`` before invoking a +provider. The package also exports TEI enrichment services for show notes, +chapter markers, and guest biographies that operate on persisted canonical TEI. """ from episodic.generation.chapter_markers import ( @@ -12,6 +15,22 @@ ChapterMarkersResult, enrich_tei_with_chapter_markers, ) +from episodic.generation.draft_script import ( + DraftPresenterProfile, + DraftScriptGenerationError, + DraftScriptGenerator, + DraftScriptProviderResponseError, + DraftScriptRequest, + DraftScriptResponseFormatError, + DraftScriptResult, + DraftScriptSource, + DraftScriptTeiError, + DraftScriptTokenBudgetError, + DraftScriptTransientProviderError, + DraftTurn, + LLMDraftScriptGenerator, + LLMDraftScriptGeneratorConfig, +) from episodic.generation.guest_bios import ( GuestBioEntry, GuestBiosEnrichmentRequest, @@ -25,6 +44,12 @@ generate_guest_bios_from_reference_bindings, project_guest_bio_sources, ) +from episodic.generation.launcher import ( + GenerationRunAdmissionError, + GenerationRunLauncher, + InProcessGenerationRunLauncher, +) +from episodic.generation.launcher_support import GenerationSourceLimits from episodic.generation.show_notes import ( ShowNotesEntry, ShowNotesGenerator, @@ -40,6 +65,21 @@ "ChapterMarkersGeneratorConfig", "ChapterMarkersResponseFormatError", "ChapterMarkersResult", + "DraftPresenterProfile", + "DraftScriptGenerationError", + "DraftScriptGenerator", + "DraftScriptProviderResponseError", + "DraftScriptRequest", + "DraftScriptResponseFormatError", + "DraftScriptResult", + "DraftScriptSource", + "DraftScriptTeiError", + "DraftScriptTokenBudgetError", + "DraftScriptTransientProviderError", + "DraftTurn", + "GenerationRunAdmissionError", + "GenerationRunLauncher", + "GenerationSourceLimits", "GuestBioEntry", "GuestBioSource", "GuestBiosEnrichmentRequest", @@ -48,6 +88,9 @@ "GuestBiosGeneratorConfig", "GuestBiosResponseFormatError", "GuestBiosResult", + "InProcessGenerationRunLauncher", + "LLMDraftScriptGenerator", + "LLMDraftScriptGeneratorConfig", "ShowNotesEntry", "ShowNotesGenerator", "ShowNotesGeneratorConfig", diff --git a/episodic/generation/draft_script.py b/episodic/generation/draft_script.py new file mode 100644 index 00000000..c9f2ad92 --- /dev/null +++ b/episodic/generation/draft_script.py @@ -0,0 +1,435 @@ +"""Define the port and implementation for one-pass draft script generation. + +The immutable request and result types carry canonical source and presenter +context. ``DraftScriptGenerator`` is the public seam; its LLM implementation +maps provider failures, parses deterministic JSON, and emits TEI-P5 XML with +content hashes and usage metadata. + +The launcher maps canonical documents and bindings into a request; generation +persistence stores the result on the canonical episode. +""" + +import collections.abc as cabc +import dataclasses as dc +import json +import typing as typ + +import tei_rapporteur as tei + +from episodic.canonical.hashing import sha256_text +from episodic.generation.tei_payload import ( + require_mapping, + require_non_empty_str_value, + require_sequence, +) +from episodic.llm import ( + LLMPort, + LLMProviderOperation, + LLMProviderResponseError, + LLMRequest, + LLMResponse, + LLMTokenBudget, + LLMTokenBudgetExceededError, + LLMTransientProviderError, + LLMUsage, + ProviderCallUsage, +) + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + +type JsonMapping = dict[str, object] +type DraftClock = cabc.Callable[[], dt.datetime] +type DraftIdFactory = cabc.Callable[[str], str] + +_DEFAULT_SYSTEM_PROMPT = ( + "The assistant writes concise podcast draft scripts from supplied source " + "material. Return JSON only with keys title and turns. Each turn must " + "contain text and may contain speaker. Do not invent facts beyond the " + "provided sources and presenter profiles." +) + + +class DraftScriptGenerationError(Exception): + """Base class for draft script generation failures.""" + + +class DraftScriptResponseFormatError(DraftScriptGenerationError, ValueError): + """Raised when an LLM response does not match the draft schema.""" + + +class DraftScriptTeiError(DraftScriptGenerationError, ValueError): + """Raised when draft payloads cannot be emitted as valid TEI.""" + + +class DraftScriptTokenBudgetError(DraftScriptGenerationError): + """Raised when the LLM adapter rejects the draft request budget.""" + + +class DraftScriptProviderResponseError(DraftScriptGenerationError): + """Raised when the LLM provider returns a non-retryable response error.""" + + +class DraftScriptTransientProviderError(DraftScriptGenerationError): + """Raised when transient provider failures are exhausted.""" + + +@dc.dataclass(frozen=True, slots=True) +class DraftScriptSource: + """Source identity, provenance, prompt text, and relative weight.""" + + source_id: str + source_type: str + source_uri: str + content: str + weight: float + + def __post_init__(self) -> None: + """Validate source fields.""" + _require_non_empty_text(self.source_id, "source_id") + _require_non_empty_text(self.source_type, "source_type") + _require_non_empty_text(self.source_uri, "source_uri") + _require_non_empty_text(self.content, "content") + if not 0 <= self.weight <= 1: + msg = "weight must be between 0 and 1." + raise ValueError(msg) + + +@dc.dataclass(frozen=True, slots=True) +class DraftPresenterProfile: + """Presenter identity, role, and reference text for generation context.""" + + display_name: str + role: str + source_content: str + + def __post_init__(self) -> None: + """Validate presenter profile fields.""" + _require_non_empty_text(self.display_name, "display_name") + _require_non_empty_text(self.role, "role") + _require_non_empty_text(self.source_content, "source_content") + + +@dc.dataclass(frozen=True, slots=True) +class DraftScriptRequest: + """Immutable input required to generate one draft TEI script. + + Attributes + ---------- + episode_id, series_profile_id, title + Canonical context and non-empty working title. + sources, presenter_profiles + Required sources and optional presenter context. + clock, id_factory + Deterministic timestamp and TEI-ID seams. + """ + + episode_id: uuid.UUID + series_profile_id: uuid.UUID + title: str + sources: tuple[DraftScriptSource, ...] + presenter_profiles: tuple[DraftPresenterProfile, ...] + clock: DraftClock + id_factory: DraftIdFactory + + def __post_init__(self) -> None: + """Validate draft request fields.""" + _require_non_empty_text(self.title, "title") + if len(self.sources) == 0: + msg = "sources must contain at least one source." + raise ValueError(msg) + + +@dc.dataclass(frozen=True, slots=True) +class DraftTurn: + """One ordered generated turn with text and an optional speaker.""" + + text: str + speaker: str | None = None + + def __post_init__(self) -> None: + """Validate turn text and speaker.""" + _require_non_empty_text(self.text, "text") + if self.speaker is not None: + _require_non_empty_text(self.speaker, "speaker") + + +@dc.dataclass(frozen=True, slots=True) +class DraftScriptResult: + """Generated draft TEI and the provider metadata needed downstream. + + Attributes + ---------- + tei_xml, content_hash, usage + TEI-P5 output, canonical hash, and normalised usage. + model, provider_response_id, finish_reason, provider_call_usage + Provider metadata for lifecycle and cost recording. + """ + + tei_xml: str + content_hash: str + usage: LLMUsage + model: str + provider_response_id: str + finish_reason: str | None + provider_call_usage: ProviderCallUsage | None = None + + +@dc.dataclass(frozen=True, slots=True) +class LLMDraftScriptGeneratorConfig: + """Configuration for one-pass draft-script generation. + + Attributes + ---------- + model : str + Provider model identifier. + provider_operation : LLMProviderOperation | str + Provider operation shape used for the request. + token_budget : LLMTokenBudget | None + Token budget constraints for the request, or ``None`` for no token + budget. + system_prompt : str + System prompt sent with the draft-generation request. + max_response_bytes : int + Positive maximum UTF-8 response size in bytes. The cap is checked + before the generated JSON is parsed. + """ + + model: str + provider_operation: LLMProviderOperation | str = ( + LLMProviderOperation.CHAT_COMPLETIONS + ) + token_budget: LLMTokenBudget | None = None + system_prompt: str = _DEFAULT_SYSTEM_PROMPT + max_response_bytes: int = 1_048_576 + + def __post_init__(self) -> None: + """Reject non-positive response-size limits.""" + if self.max_response_bytes < 1: + msg = "max_response_bytes must be positive." + raise ValueError(msg) + + +class DraftScriptGenerator(typ.Protocol): + """Protocol implemented by one-pass draft generators.""" + + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Generate one draft script from canonical generation context. + + Parameters + ---------- + request + Immutable canonical generation context. + + Returns + ------- + DraftScriptResult + Generated TEI-P5 XML, hash, and provider metadata. + + Raises + ------ + DraftScriptGenerationError + If generation cannot produce a valid draft. + """ + raise NotImplementedError + + +@dc.dataclass(frozen=True, slots=True) +class _ParsedDraft: + """Represent a parsed draft title and its ordered turns.""" + + title: str + turns: tuple[DraftTurn, ...] + + +@dc.dataclass(frozen=True, slots=True) +class LLMDraftScriptGenerator(DraftScriptGenerator): + """Generate draft TEI scripts through an LLM port and its configuration.""" + + llm: LLMPort + config: LLMDraftScriptGeneratorConfig + + @typ.override + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Generate and validate one TEI-P5 draft script. + + Parameters + ---------- + request + Canonical context used to build the deterministic prompt. + + Returns + ------- + DraftScriptResult + Validated TEI-P5 XML, hash, usage, and provider metadata. + + Raises + ------ + DraftScriptTokenBudgetError + If the LLM adapter rejects the requested token budget. + DraftScriptProviderResponseError + If the provider returns a non-retryable response error. + DraftScriptTransientProviderError + If the provider reports a transient failure. + DraftScriptResponseFormatError + If the provider response is not the expected JSON draft. + DraftScriptTeiError + If the parsed draft cannot be emitted as valid TEI-P5. + + Notes + ----- + Provider errors are translated before parsing and TEI emission. + """ # noqa: DOC502 # Parsing and TEI helpers raise these documented exceptions. + llm_request = LLMRequest( + model=self.config.model, + prompt=_build_prompt(request), + system_prompt=self.config.system_prompt, + provider_operation=self.config.provider_operation, + token_budget=self.config.token_budget, + ) + try: + response = await self.llm.generate(llm_request) + except LLMTokenBudgetExceededError as exc: + raise DraftScriptTokenBudgetError(str(exc)) from exc + except LLMProviderResponseError as exc: + raise DraftScriptProviderResponseError(str(exc)) from exc + except LLMTransientProviderError as exc: + raise DraftScriptTransientProviderError(str(exc)) from exc + + _require_response_size(response.text, self.config.max_response_bytes) + parsed = _parse_response(response) + tei_xml = _emit_tei(parsed, request.id_factory) + return DraftScriptResult( + tei_xml=tei_xml, + content_hash=sha256_text(tei_xml), + usage=response.usage, + model=response.model, + provider_response_id=response.provider_response_id, + finish_reason=response.finish_reason, + provider_call_usage=response.provider_call_usage, + ) + + +def _require_non_empty_text(value: str, field_name: str) -> None: + """Reject blank strings.""" + if not isinstance(value, str): + msg = f"{field_name} must be a string." + raise TypeError(msg) + if value.strip() == "": + msg = f"{field_name} must be a non-empty string." + raise ValueError(msg) + + +def _build_prompt(request: DraftScriptRequest) -> str: + """Build a deterministic JSON prompt payload.""" + payload: JsonMapping = { + "episode_id": str(request.episode_id), + "series_profile_id": str(request.series_profile_id), + "title": request.title, + "requested_at": request.clock().isoformat(), + "sources": [dc.asdict(source) for source in request.sources], + "presenter_profiles": [ + dc.asdict(profile) for profile in request.presenter_profiles + ], + } + return json.dumps(payload, indent=2, sort_keys=True) + + +def _require_response_size(response_text: str, maximum_bytes: int) -> None: + """Reject provider responses that exceed the configured byte limit.""" + if len(response_text.encode("utf-8")) > maximum_bytes: + msg = "LLM response exceeds the configured maximum size." + raise DraftScriptResponseFormatError(msg) + + +def _parse_response(response: LLMResponse) -> _ParsedDraft: + """Parse and validate the LLM response JSON.""" + try: + payload = json.loads(response.text) + except json.JSONDecodeError as exc: + msg = "LLM response is not valid JSON." + raise DraftScriptResponseFormatError(msg) from exc + + payload_dict = require_mapping( + payload, + "response", + error_cls=DraftScriptResponseFormatError, + ) + title = require_non_empty_str_value( + payload_dict.get("title"), + "title", + error_cls=DraftScriptResponseFormatError, + ).strip() + raw_turns = require_sequence( + payload_dict.get("turns"), + "turns", + error_cls=DraftScriptResponseFormatError, + ) + turns = tuple(_parse_turn(raw_turn) for raw_turn in raw_turns) + if len(turns) == 0: + msg = "turns must contain at least one turn." + raise DraftScriptResponseFormatError(msg) + return _ParsedDraft(title=title, turns=turns) + + +def _parse_turn(raw_turn: object) -> DraftTurn: + """Parse one generated turn.""" + turn = require_mapping( + raw_turn, + "turn", + error_cls=DraftScriptResponseFormatError, + ) + text = require_non_empty_str_value( + turn.get("text"), + "text", + error_cls=DraftScriptResponseFormatError, + ).strip() + speaker = _optional_non_empty_string(turn.get("speaker"), "speaker") + return DraftTurn(text=text, speaker=speaker) + + +def _optional_non_empty_string(value: object, field_name: str) -> str | None: + """Return an optional stripped string or raise a format error.""" + if value is None: + return None + if not isinstance(value, str): + msg = f"{field_name} must be a string or null." + raise DraftScriptResponseFormatError(msg) + stripped = value.strip() + return stripped or None + + +def _emit_tei(parsed: _ParsedDraft, id_factory: DraftIdFactory) -> str: + """Emit validated TEI XML from a parsed draft.""" + payload: JsonMapping = { + "header": {"file_desc": {"title": parsed.title}}, + "text": { + "body": { + "blocks": [_turn_to_block(turn, id_factory) for turn in parsed.turns] + } + }, + } + try: + document = tei.from_dict(payload) + document.validate() + return tei.emit_xml(document) + except (TypeError, ValueError) as exc: + raise DraftScriptTeiError(str(exc)) from exc + + +def _turn_to_block(turn: DraftTurn, id_factory: DraftIdFactory) -> JsonMapping: + """Convert one generated turn to a `tei_rapporteur` body block.""" + content = [{"type": "text", "value": turn.text}] + if turn.speaker is None: + return { + "type": "paragraph", + "xml_id": id_factory("p"), + "content": content, + } + return { + "type": "utterance", + "speaker": turn.speaker, + "xml_id": id_factory("u"), + "content": content, + } diff --git a/episodic/generation/launcher.py b/episodic/generation/launcher.py new file mode 100644 index 00000000..9a93a0dd --- /dev/null +++ b/episodic/generation/launcher.py @@ -0,0 +1,564 @@ +"""Schedule and execute bounded no-QA generation runs in the API process. + +The public :class:`GenerationRunLauncher` port and +:class:`InProcessGenerationRunLauncher` service connect the generation-run API +resource to canonical run persistence. A launcher claims a +:class:`~episodic.canonical.domain.GenerationRun`, loads its episode, source +documents, and resolved presenter bindings through fresh +:class:`~episodic.canonical.unit_of_work_protocols.CanonicalUnitOfWork` +instances, then delegates draft creation to :class:`DraftScriptGenerator`. +It records ordered lifecycle events, persists the generated TEI through +``persist_draft_script``, records optional provider costs, and applies the +immutable ``GenerationRunStatusUpdate`` command for terminal state changes. + +Admission is bounded before an asyncio task is allocated, and strong task +references support draining or cancellation. The launcher exposes metrics and +tracing ports, records cancellation as a terminal failure, and treats leases +as inspection and manual-recovery hooks rather than restart recovery. The +runtime shuts the launcher down before closing the provider and disposing the +database engine; request-scoped units of work must never be passed to a +background task. +""" + +import asyncio +import dataclasses as dc +import datetime as dt +import typing as typ + +from episodic.canonical.domain import GenerationRun, GenerationRunStatus +from episodic.canonical.generation_persistence import ( + DraftScriptPersistenceRequest, + persist_draft_script, +) +from episodic.canonical.generation_quality import QaStatus +from episodic.canonical.generation_run_errors import RunAlreadyTerminal, RunNotFound +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate +from episodic.canonical.reference_documents import resolve_bindings +from episodic.cost.ports import BillingPeriodKey +from episodic.cost.recorder import CostProviderOperation +from episodic.generation.launcher_support import ( + ClaimedRun, + Clock, + CostRecorderFactory, + DraftIdFactoryFactory, + Failure, + GenerationSourceLimitError, + GenerationSourceLimits, + PersistedTei, + ProviderCallRecordRequest, + SequentialDraftIds, + classify_failure, + draft_generated_payload, + draft_request, + project_presenter_profiles, + provider_call_record, + require_episode, + source_from_document, +) +from episodic.logging import get_logger, log_error, log_info +from episodic.observability import ( + MonotonicClockPort, + NoopTracer, + NoopValueMetrics, + PerfCounterClock, + TracerPort, + ValueMetricsPort, +) + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from episodic.canonical.domain import SourceDocument + from episodic.canonical.object_store import ObjectStorePort + from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + from episodic.generation.draft_script import ( + DraftScriptGenerator, + DraftScriptResult, + DraftScriptSource, + ) + +type TaskSet = set[asyncio.Task[None]] + +_DEFAULT_MAX_CONCURRENCY = 4 +_DEFAULT_MAX_PENDING_RUNS = 16 +_DEFAULT_LEASE_SECONDS = 900 +_METRIC_TERMINAL_STATES = "generation_run_terminal_total" +_METRIC_DRAFT_ERRORS = "generation_run_draft_errors_total" +_METRIC_QA_BYPASS = "generation_run_qa_bypass_total" +_METRIC_DRAFT_LATENCY = "generation_run_draft_latency_ms" +_METRIC_ADMISSION_REJECTED = "generation_run_admission_rejected_total" +_METRIC_PENDING_DEPTH = "generation_run_pending_depth" + +logger = get_logger(__name__) + + +class GenerationRunLauncher(typ.Protocol): + """Port for scheduling asynchronous generation-run execution.""" + + async def launch(self, run_id: uuid.UUID) -> None: + """Schedule generation for one run.""" + + +class GenerationRunAdmissionError(RuntimeError): + """Raised when a launcher cannot retain another pending run.""" + + +@dc.dataclass(frozen=True, slots=True) +class _ExecutionOutcome: + """Describe the terminal result of one launcher task.""" + + outcome: str + failure_category: str | None = None + + +@dc.dataclass(slots=True) +class InProcessGenerationRunLauncher(GenerationRunLauncher): + """Schedule and execute no-QA draft generation in-process.""" + + uow_factory: cabc.Callable[[], CanonicalUnitOfWork] + draft_generator: DraftScriptGenerator + object_store: ObjectStorePort | None = None + cost_recorder_factory: CostRecorderFactory | None = None + clock: Clock = lambda: dt.datetime.now(dt.UTC) + draft_id_factory_factory: DraftIdFactoryFactory = SequentialDraftIds + provider_name: str = "openai" + provider_operation: str = "chat_completions" + max_concurrency: int = _DEFAULT_MAX_CONCURRENCY + max_pending_runs: int = _DEFAULT_MAX_PENDING_RUNS + lease_seconds: int = _DEFAULT_LEASE_SECONDS + source_limits: GenerationSourceLimits = dc.field( + default_factory=GenerationSourceLimits + ) + metrics: ValueMetricsPort = dc.field(default_factory=NoopValueMetrics) + tracer: TracerPort = dc.field(default_factory=NoopTracer) + monotonic_clock: MonotonicClockPort = dc.field(default_factory=PerfCounterClock) + _tasks: TaskSet = dc.field(default_factory=set, init=False) + _task_run_ids: dict[asyncio.Task[None], uuid.UUID] = dc.field( + default_factory=dict, + init=False, + ) + _semaphore: asyncio.Semaphore = dc.field(init=False) + _admitted_run_count: int = dc.field(default=0, init=False) + _cancelled_run_ids: set[uuid.UUID] = dc.field(default_factory=set, init=False) + _is_shutting_down: bool = dc.field(default=False, init=False) + + def __post_init__(self) -> None: + """Validate and initialise launcher state.""" + if self.max_concurrency < 1: + msg = "max_concurrency must be at least 1." + raise ValueError(msg) + if self.max_pending_runs < 0: + msg = "max_pending_runs must be non-negative." + raise ValueError(msg) + self._semaphore = asyncio.Semaphore(self.max_concurrency) + + async def launch(self, run_id: uuid.UUID) -> None: + """Schedule a background task for one generation run.""" + self._admit() + try: + task = asyncio.create_task( + self._run_task(run_id), + name=f"generation-run-{run_id}", + ) + except Exception: + self._admitted_run_count -= 1 + self._record_pending_depth() + raise + self._tasks.add(task) + self._task_run_ids[task] = run_id + task.add_done_callback(self._discard_task) + log_info(logger, "generation_run_launcher.scheduled run_id=%s", run_id) + + async def drain(self) -> None: + """Wait for all currently scheduled tasks to finish.""" + while self._tasks: + await asyncio.gather(*tuple(self._tasks), return_exceptions=True) + + @property + def scheduled_run_count(self) -> int: + """Return the number of background runs retained by the launcher.""" + return len(self._tasks) + + async def shutdown(self) -> None: + """Cancel and drain all scheduled generation tasks.""" + self._is_shutting_down = True + cancelled_tasks = tuple( + (task, self._task_run_ids[task]) for task in self._tasks if not task.done() + ) + for task, _ in cancelled_tasks: + task.cancel() + await self.drain() + for task, run_id in cancelled_tasks: + if task.cancelled() and run_id not in self._cancelled_run_ids: + await asyncio.shield(self._record_cancellation(run_id)) + self._cancelled_run_ids.add(run_id) + + def _discard_task(self, task: asyncio.Task[None]) -> None: + """Remove finished tasks from the strong-reference registry.""" + self._tasks.discard(task) + self._task_run_ids.pop(task, None) + self._admitted_run_count -= 1 + self._record_pending_depth() + + def _admit(self) -> None: + """Reserve bounded capacity before creating a background task.""" + if self._is_shutting_down: + self.metrics.increment_counter( + _METRIC_ADMISSION_REJECTED, + labels={"reason": "shutdown"}, + ) + msg = "Generation run admission is closed during shutdown." + raise GenerationRunAdmissionError(msg) + capacity = self.max_concurrency + self.max_pending_runs + if self._admitted_run_count >= capacity: + self.metrics.increment_counter( + _METRIC_ADMISSION_REJECTED, + labels={"reason": "capacity"}, + ) + msg = "Generation run admission capacity is exhausted." + raise GenerationRunAdmissionError(msg) + self._admitted_run_count += 1 + self._record_pending_depth() + + async def _run_task(self, run_id: uuid.UUID) -> None: + """Execute one scheduled generation run.""" + with self.tracer.start_span( + "generation_run.execute", + attributes={"operation": "generation_run.execute", "run_id": str(run_id)}, + ) as span: + try: + async with self._semaphore: + outcome = await self._execute_run(run_id) + except asyncio.CancelledError: + span.set_attribute("outcome", "cancelled") + span.set_attribute("failure_category", "launcher.shutdown") + await asyncio.shield(self._record_cancellation(run_id)) + self._cancelled_run_ids.add(run_id) + raise + else: + span.set_attribute("outcome", outcome.outcome) + if outcome.failure_category is not None: + span.set_attribute("failure_category", outcome.failure_category) + + async def _execute_run(self, run_id: uuid.UUID) -> _ExecutionOutcome: + """Execute one generation run while its concurrency permit is held.""" + try: + claimed = await self._claim(run_id) + if claimed is None: + return _ExecutionOutcome(outcome="not_claimed") + result = await self._generate(claimed) + await self._record_draft_generated(claimed.run.id, result) + await self._persist_success(claimed, result) + return _ExecutionOutcome(outcome="completed") + except Exception as exc: # noqa: BLE001 # Task boundary must persist unexpected failures. + failure = classify_failure(exc) + await self._record_failure(run_id, failure) + return _ExecutionOutcome( + outcome="failed", + failure_category=failure.category, + ) + + async def _record_cancellation(self, run_id: uuid.UUID) -> None: + """Record cancellation consistently before or during execution.""" + await self._record_failure( + run_id, + Failure( + message="Generation task cancelled during shutdown.", + category="launcher.shutdown", + ), + ) + + async def _claim(self, run_id: uuid.UUID) -> ClaimedRun | None: + """Claim a pending run, then load its input outside the claim transaction.""" + run = await self._claim_and_start(run_id) + if run is None: + return None + async with self.uow_factory() as uow: + episode = await require_episode(uow, run.episode_id) + documents = await uow.source_documents.list_for_job(run.source_bundle_id) + if len(documents) > self.source_limits.max_source_count: + raise GenerationSourceLimitError.source_count() + presenter_profiles = project_presenter_profiles( + await resolve_bindings( + uow, + series_profile_id=episode.series_profile_id, + episode_id=episode.id, + ) + ) + sources = await self._load_sources(documents) + self.metrics.increment_counter( + _METRIC_QA_BYPASS, + labels={"quality_mode": run.quality_mode.value}, + ) + return ClaimedRun( + run=run, + episode=episode, + sources=sources, + presenter_profiles=presenter_profiles, + ) + + async def _claim_and_start(self, run_id: uuid.UUID) -> GenerationRun | None: + """Linearize a claim and make its started event durable before hydration.""" + async with self.uow_factory() as uow: + started_at = self.clock() + run = await uow.generation_runs.claim_run_for_execution( + run_id, + current_node="draft", + started_at=started_at, + lease_expires_at=started_at + dt.timedelta(seconds=self.lease_seconds), + ) + if run is None: + await uow.rollback() + return None + await uow.generation_runs.append_event( + run.id, + kind="run.started", + payload={"current_node": "draft"}, + occurred_at=started_at, + ) + await uow.commit() + return run + + async def _load_sources( + self, + documents: list[SourceDocument], + ) -> tuple[DraftScriptSource, ...]: + """Load bounded source text and reject an aggregate-size overflow.""" + if len(documents) > self.source_limits.max_source_count: + raise GenerationSourceLimitError.source_count() + aggregate_bytes = 0 + sources: list[DraftScriptSource] = [] + for document in documents: + source = await source_from_document( + document, + self.object_store, + self.source_limits, + remaining_aggregate_bytes=( + self.source_limits.max_aggregate_source_bytes - aggregate_bytes + ), + ) + aggregate_bytes += len(source.content.encode()) + sources.append(source) + return tuple(sources) + + async def _generate(self, claimed: ClaimedRun) -> DraftScriptResult: + """Generate one draft and record latency metrics.""" + start = self.monotonic_clock.monotonic_seconds() + try: + return await self.draft_generator.generate( + draft_request( + claimed=claimed, + clock=self.clock, + id_factory_factory=self.draft_id_factory_factory, + ) + ) + finally: + elapsed_ms = (self.monotonic_clock.monotonic_seconds() - start) * 1000 + self.metrics.observe_latency_ms( + _METRIC_DRAFT_LATENCY, + elapsed_ms, + labels={"quality_mode": claimed.run.quality_mode.value}, + ) + + async def _record_draft_generated( + self, + run_id: uuid.UUID, + result: DraftScriptResult, + ) -> None: + """Record that draft generation returned a provider response.""" + async with self.uow_factory() as uow: + await uow.generation_runs.append_event( + run_id, + kind="draft.generated", + payload=draft_generated_payload(result), + occurred_at=self.clock(), + ) + await uow.commit() + + async def _persist_success( + self, + claimed: ClaimedRun, + result: DraftScriptResult, + ) -> None: + """Persist generated TEI, cost records, and terminal success.""" + async with self.uow_factory() as uow: + updated_episode = await persist_draft_script( + uow, + DraftScriptPersistenceRequest( + episode_id=claimed.run.episode_id, + generation_run_id=claimed.run.id, + result=result, + expected_revision=claimed.episode.tei_revision, + clock=self.clock, + ), + ) + await self._record_success_events_and_costs( + uow, + claimed, + result, + PersistedTei( + revision=updated_episode.tei_revision, + content_hash=updated_episode.tei_content_hash, + ), + ) + await uow.commit() + self._record_terminal_metric(GenerationRunStatus.SUCCEEDED, "none") + log_info( + logger, + "generation_run_launcher.succeeded run_id=%s", + claimed.run.id, + ) + + async def _record_success_events_and_costs( + self, + uow: CanonicalUnitOfWork, + claimed: ClaimedRun, + result: DraftScriptResult, + persisted_tei: PersistedTei, + ) -> None: + """Record success-side events, costs, and terminal status.""" + await uow.generation_runs.append_event( + claimed.run.id, + kind="tei.persisted", + payload={ + "tei_revision": persisted_tei.revision, + "content_hash": persisted_tei.content_hash, + "qa_status": QaStatus.SKIPPED.value, + }, + occurred_at=self.clock(), + ) + await self._record_costs(uow, claimed.run.id, result) + await uow.generation_runs.append_event( + claimed.run.id, + kind="run.succeeded", + payload={"current_node": "complete"}, + occurred_at=self.clock(), + ) + await uow.generation_runs.update_run_status( + claimed.run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.SUCCEEDED, + current_node="complete", + ended_at=self.clock(), + ), + ) + + async def _record_costs( + self, + uow: CanonicalUnitOfWork, + run_id: uuid.UUID, + result: DraftScriptResult, + ) -> None: + """Record provider-call and roll-up cost entries when configured.""" + if self.cost_recorder_factory is None: + return + recorder = self.cost_recorder_factory(uow) + if recorder is None: + return + billing_period_key = BillingPeriodKey(self.clock().strftime("%Y-%m")) + await recorder.pin_run_pricing( + str(run_id), + ( + CostProviderOperation( + provider_name=self.provider_name, + model=result.model, + operation=self.provider_operation, + ), + ), + billing_period_key, + ) + await recorder.record_provider_call( + provider_call_record( + ProviderCallRecordRequest( + run_id=run_id, + provider_name=self.provider_name, + provider_operation=self.provider_operation, + billing_period_key=billing_period_key, + result=result, + recorded_at=self.clock(), + ) + ) + ) + await recorder.finalize_run(str(run_id), "draft") + + async def _record_failure(self, run_id: uuid.UUID, failure: Failure) -> None: + """Record a terminal failed run state.""" + async with self.uow_factory() as uow: + try: + await self._append_failure_events(uow, run_id, failure) + await uow.generation_runs.update_run_status( + run_id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.FAILED, + current_node="failed", + ended_at=self.clock(), + error_message=failure.message, + error_category=failure.category, + ), + ) + await uow.commit() + except RunAlreadyTerminal, RunNotFound: + await uow.rollback() + return + self._record_terminal_metric(GenerationRunStatus.FAILED, failure.category) + self.metrics.increment_counter( + _METRIC_DRAFT_ERRORS, + labels={"error_category": failure.category}, + ) + log_error( + logger, + "generation_run_launcher.failed run_id=%s category=%s", + run_id, + failure.category, + ) + + async def _append_failure_events( + self, + uow: CanonicalUnitOfWork, + run_id: uuid.UUID, + failure: Failure, + ) -> None: + """Append failure-related events before terminal status mutation.""" + if failure.should_emit_invalid_tei: + await uow.generation_runs.append_event( + run_id, + kind="tei.invalid", + payload={"error_category": failure.category}, + occurred_at=self.clock(), + ) + await uow.generation_runs.append_event( + run_id, + kind="run.failed", + payload={ + "error_message": failure.message, + "error_category": failure.category, + }, + occurred_at=self.clock(), + ) + + def _record_terminal_metric( + self, + status: GenerationRunStatus, + error_category: str, + ) -> None: + """Record the terminal-state counter.""" + self.metrics.increment_counter( + _METRIC_TERMINAL_STATES, + labels={"status": status.value, "error_category": error_category}, + ) + + def _record_pending_depth(self) -> None: + """Report bounded work waiting beyond the execution capacity.""" + pending_depth = max(0, self._admitted_run_count - self.max_concurrency) + self.metrics.observe_value( + _METRIC_PENDING_DEPTH, + float(pending_depth), + labels={}, + ) + + +__all__ = [ + "GenerationRunLauncher", + "InProcessGenerationRunLauncher", +] diff --git a/episodic/generation/launcher_support.py b/episodic/generation/launcher_support.py new file mode 100644 index 00000000..a363c6b8 --- /dev/null +++ b/episodic/generation/launcher_support.py @@ -0,0 +1,477 @@ +"""Translate canonical generation data at the launcher service boundary. + +The public support types—``CostRecorderFactory``, ``SequentialDraftIds``, +``ClaimedRun``, ``Failure``, ``PersistedTei``, and +``ProviderCallRecordRequest``—keep launcher orchestration independent of +storage and provider representations. The helper services load source text +from canonical :class:`~episodic.canonical.domain.SourceDocument` records or +the object-store port, project resolved host and guest reference-document +bindings, and build the immutable request consumed by ``DraftScriptGenerator``. + +The event-payload and provider-record helpers map ``DraftScriptResult`` usage, +provider metadata, and content hashes into generation events and the +``CostRecorderPort`` contract. ``classify_failure`` preserves stable terminal +categories for the API and metrics. These helpers accept the canonical unit +of work and outbound ports supplied by the launcher; they do not open, +commit, or dispose persistence sessions themselves. +""" + +import collections.abc as cabc +import dataclasses as dc +import datetime as dt +import json +import typing as typ + +from episodic.canonical.domain import ReferenceDocumentKind +from episodic.canonical.episode_errors import ( + EpisodeNotFoundError, + EpisodeRevisionConflictError, +) +from episodic.canonical.generation_persistence import InvalidDraftTeiError +from episodic.cost.ports import ( + BillingPeriodKey, + IdempotencyKey, + PricingModel, + UsageSource, +) +from episodic.cost.recorder import ( + CostRecorderPort, + ProviderCallRecord, +) +from episodic.generation.draft_script import ( + DraftPresenterProfile, + DraftScriptGenerationError, + DraftScriptProviderResponseError, + DraftScriptRequest, + DraftScriptResponseFormatError, + DraftScriptResult, + DraftScriptSource, + DraftScriptTeiError, + DraftScriptTokenBudgetError, + DraftScriptTransientProviderError, +) + +if typ.TYPE_CHECKING: + import uuid + + from episodic.canonical.domain import ( + CanonicalEpisode, + GenerationRun, + JsonMapping, + SourceDocument, + ) + from episodic.canonical.object_store import ObjectStorePort + from episodic.canonical.reference_documents.resolution import ResolvedBinding + from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + +type Clock = cabc.Callable[[], dt.datetime] +type DraftIdFactoryFactory = cabc.Callable[[], cabc.Callable[[str], str]] + +_DEFAULT_MAX_SOURCE_COUNT = 32 +_DEFAULT_MAX_SOURCE_BYTES = 2 * 1024 * 1024 +_DEFAULT_MAX_AGGREGATE_SOURCE_BYTES = 8 * 1024 * 1024 +_DEFAULT_MAX_NORMALIZED_SOURCE_BYTES = 2 * 1024 * 1024 + + +@dc.dataclass(frozen=True, slots=True) +class GenerationSourceLimits: + """Validated bounds applied while building one draft's source input. + + Attributes + ---------- + max_source_count : int + Maximum number of source documents accepted for one draft. + max_source_bytes : int + Maximum bytes retained from one source document. + max_aggregate_source_bytes : int + Maximum bytes retained across all source documents. + max_normalized_source_bytes : int + Maximum UTF-8 bytes after source-text normalisation. + + Raises + ------ + ValueError + If any configured bound is less than one. + """ + + max_source_count: int = _DEFAULT_MAX_SOURCE_COUNT + max_source_bytes: int = _DEFAULT_MAX_SOURCE_BYTES + max_aggregate_source_bytes: int = _DEFAULT_MAX_AGGREGATE_SOURCE_BYTES + max_normalized_source_bytes: int = _DEFAULT_MAX_NORMALIZED_SOURCE_BYTES + + def __post_init__(self) -> None: + """Reject non-positive limits before a launcher begins work.""" + for name, value in dc.asdict(self).items(): + if value < 1: + msg = f"{name} must be at least 1." + raise ValueError(msg) + + +class GenerationSourceLimitError(DraftScriptGenerationError): + """Raised when bounded generation source input exceeds a configured limit. + + This translated generation error keeps limit rejections stable and free of + source content, identifiers, and byte counts. + """ + + @classmethod + def source_count(cls) -> GenerationSourceLimitError: + """Build the stable source-count rejection.""" + message = "Generation source count exceeds limit." + return cls(message) + + @classmethod + def source_bytes(cls) -> GenerationSourceLimitError: + """Build the stable per-source byte rejection.""" + message = "Generation source exceeds byte limit." + return cls(message) + + @classmethod + def aggregate_bytes(cls) -> GenerationSourceLimitError: + """Build the stable aggregate-byte rejection.""" + message = "Generation source aggregate exceeds byte limit." + return cls(message) + + @classmethod + def normalized_bytes(cls) -> GenerationSourceLimitError: + """Build the stable normalized-text rejection.""" + message = "Generation source exceeds normalized limit." + return cls(message) + + +class CostRecorderFactory(typ.Protocol): + """Factory that binds a cost recorder to a unit of work.""" + + def __call__(self, uow: CanonicalUnitOfWork) -> CostRecorderPort | None: + """Return a recorder for the active unit of work, or disable recording.""" + + +class SequentialDraftIds: + """Deterministic per-run TEI id factory.""" + + def __init__(self) -> None: + self._counts: dict[str, int] = {} + + def __call__(self, prefix: str) -> str: + """Return the next identifier for a prefix.""" + next_value = self._counts.get(prefix, 0) + 1 + self._counts[prefix] = next_value + return f"{prefix}-{next_value}" + + +@dc.dataclass(frozen=True, slots=True) +class ClaimedRun: + """Generation input loaded after claiming one run.""" + + run: GenerationRun + episode: CanonicalEpisode + sources: tuple[DraftScriptSource, ...] + presenter_profiles: tuple[DraftPresenterProfile, ...] + + +@dc.dataclass(frozen=True, slots=True) +class Failure: + """Stable failure details recorded on a terminal run.""" + + message: str + category: str + should_emit_invalid_tei: bool = False + + +@dc.dataclass(frozen=True, slots=True) +class PersistedTei: + """TEI persistence details needed by success event recording.""" + + revision: int + content_hash: str | None + + +@dc.dataclass(frozen=True, slots=True) +class ProviderCallRecordRequest: + """Inputs required to build one provider-call cost record.""" + + run_id: uuid.UUID + provider_name: str + provider_operation: str + billing_period_key: BillingPeriodKey + result: DraftScriptResult + recorded_at: dt.datetime + + +async def require_episode( + uow: CanonicalUnitOfWork, + episode_id: uuid.UUID, +) -> CanonicalEpisode: + """Return an episode or raise the episode-not-found error.""" + episode = await uow.episodes.get(episode_id) + if episode is None: + raise EpisodeNotFoundError(episode_id) + return episode + + +async def source_from_document( + document: SourceDocument, + object_store: ObjectStorePort | None, + limits: GenerationSourceLimits | None = None, + *, + remaining_aggregate_bytes: int | None = None, +) -> DraftScriptSource: + """Build generator source input from canonical source provenance.""" + limits = GenerationSourceLimits() if limits is None else limits + metadata_content = document.metadata.get("content") + if isinstance(metadata_content, str) and metadata_content.strip(): + content = metadata_content.strip() + elif document.source_uri.startswith("upload:"): + content = await _read_uploaded_source( + document.source_uri, + object_store, + limits, + remaining_aggregate_bytes=remaining_aggregate_bytes, + ) + else: + content = document.source_uri + _require_source_byte_limit(content, limits.max_source_bytes) + _require_aggregate_byte_limit(content, remaining_aggregate_bytes) + _require_source_size(content, limits.max_normalized_source_bytes) + return DraftScriptSource( + source_id=str(document.id), + source_type=document.source_type, + source_uri=document.source_uri, + content=content, + weight=document.weight, + ) + + +async def _read_uploaded_source( + source_uri: str, + object_store: ObjectStorePort | None, + limits: GenerationSourceLimits, + *, + remaining_aggregate_bytes: int | None, +) -> str: + """Read and normalize UTF-8 source text from an upload provenance URI.""" + if object_store is None: + msg = "An object store is required to load uploaded source content." + raise DraftScriptGenerationError(msg) + key = source_uri.removeprefix("upload:") + payload = await _read_limited_upload_bytes( + object_store, + key, + limits, + remaining_aggregate_bytes=remaining_aggregate_bytes, + ) + return _decode_and_normalize_uploaded_source( + payload, + key, + maximum_normalized_bytes=limits.max_normalized_source_bytes, + ) + + +async def _read_limited_upload_bytes( + object_store: ObjectStorePort, + key: str, + limits: GenerationSourceLimits, + *, + remaining_aggregate_bytes: int | None, +) -> bytearray: + """Read one uploaded object without retaining bytes beyond configured limits.""" + payload = bytearray() + async with object_store.open(key) as chunks: + async for chunk in chunks: + size = len(payload) + len(chunk) + if size > limits.max_source_bytes: + raise GenerationSourceLimitError.source_bytes() + if ( + remaining_aggregate_bytes is not None + and size > remaining_aggregate_bytes + ): + raise GenerationSourceLimitError.aggregate_bytes() + payload.extend(chunk) + return payload + + +def _decode_and_normalize_uploaded_source( + payload: bytearray, + key: str, + *, + maximum_normalized_bytes: int, +) -> str: + """Convert bounded uploaded bytes to normalized source text.""" + try: + content = payload.decode("utf-8-sig") + except UnicodeDecodeError as exc: + msg = f"Uploaded source {key!r} is not valid UTF-8 text." + raise DraftScriptGenerationError(msg) from exc + normalized = "\n".join(content.splitlines()).strip() + if not normalized: + msg = f"Uploaded source {key!r} contains no text." + raise DraftScriptGenerationError(msg) + _require_source_size(normalized, maximum_normalized_bytes) + return normalized + + +def _require_source_size(content: str, maximum: int) -> None: + """Reject normalized source text whose UTF-8 representation is too large.""" + if len(content.encode()) > maximum: + raise GenerationSourceLimitError.normalized_bytes() + + +def _require_source_byte_limit(content: str, maximum: int) -> None: + """Reject source content above the configured per-source byte limit.""" + if len(content.encode()) > maximum: + raise GenerationSourceLimitError.source_bytes() + + +def _require_aggregate_byte_limit( + content: str, + remaining_aggregate_bytes: int | None, +) -> None: + """Reject source content that exceeds the remaining aggregate budget.""" + if ( + remaining_aggregate_bytes is not None + and len(content.encode()) > remaining_aggregate_bytes + ): + raise GenerationSourceLimitError.aggregate_bytes() + + +def draft_request( + *, + claimed: ClaimedRun, + clock: Clock, + id_factory_factory: DraftIdFactoryFactory, +) -> DraftScriptRequest: + """Build a draft-generation request from claimed run data.""" + return DraftScriptRequest( + episode_id=claimed.run.episode_id, + series_profile_id=claimed.episode.series_profile_id, + title=claimed.episode.title, + sources=claimed.sources, + presenter_profiles=claimed.presenter_profiles, + clock=clock, + id_factory=id_factory_factory(), + ) + + +def project_presenter_profiles( + resolved_bindings: list[ResolvedBinding], +) -> tuple[DraftPresenterProfile, ...]: + """Project resolved host and guest revisions into draft input records.""" + presenter_kinds = { + ReferenceDocumentKind.HOST_PROFILE, + ReferenceDocumentKind.GUEST_PROFILE, + } + profiles: list[DraftPresenterProfile] = [] + for resolved in resolved_bindings: + if resolved.document.kind not in presenter_kinds: + continue + content = resolved.revision.content + metadata = resolved.document.metadata + display_name = _first_string(content, "display_name", "name", "title") + display_name = display_name or _first_string( + metadata, "display_name", "name", "title" + ) + source_content = _first_string( + content, + "source_content", + "profile", + "bio", + "biography", + "summary", + "content", + "text", + ) + profiles.append( + DraftPresenterProfile( + display_name=display_name or str(resolved.document.id), + role=resolved.document.kind.value.removesuffix("_profile"), + source_content=source_content or json.dumps(content, sort_keys=True), + ) + ) + return tuple(profiles) + + +def _first_string(values: cabc.Mapping[str, object], *keys: str) -> str | None: + """Return the first non-empty string from the requested mapping keys.""" + for key in keys: + value = values.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def draft_generated_payload(result: DraftScriptResult) -> JsonMapping: + """Build the draft-generated event payload.""" + return { + "model": result.model, + "provider_response_id": result.provider_response_id, + "finish_reason": result.finish_reason, + "content_hash": result.content_hash, + "usage": { + "input_tokens": result.usage.input_tokens, + "output_tokens": result.usage.output_tokens, + "total_tokens": result.usage.total_tokens, + }, + } + + +def provider_call_record(request: ProviderCallRecordRequest) -> ProviderCallRecord: + """Build a provider-call record from a draft result.""" + usage = request.result.provider_call_usage + usage_metrics = ( + dict(usage.usage_metrics) + if usage is not None + else { + "input_tokens": request.result.usage.input_tokens, + "output_tokens": request.result.usage.output_tokens, + } + ) + usage_source = usage.usage_source if usage is not None else UsageSource.PROVIDER + usage_complete = usage.usage_complete if usage is not None else True + return ProviderCallRecord( + idempotency_key=IdempotencyKey( + f"run:{request.run_id}:node:draft:call:" + f"{request.result.provider_response_id}:attempt:0" + ), + parent_cost_entry_id=None, + provider_type="llm", + provider_name=request.provider_name, + model=request.result.model, + workflow_node="draft", + operation=request.provider_operation, + usage=usage_metrics, + usage_source=usage_source, + usage_complete=usage_complete, + pricing_model=PricingModel.PAYG, + retry_attempt=0, + billing_period_key=request.billing_period_key, + workflow_run_id=str(request.run_id), + recorded_at=request.recorded_at.isoformat(), + ) + + +_FAILURE_CATEGORIES: tuple[ + tuple[type[Exception] | tuple[type[Exception], ...], str, bool], + ..., +] = ( + (EpisodeRevisionConflictError, "episode.persistence_conflict", False), + (EpisodeNotFoundError, "episode.not_found", False), + (GenerationSourceLimitError, "generation.source_limit", False), + ((InvalidDraftTeiError, DraftScriptTeiError), "tei.invalid", True), + (DraftScriptTransientProviderError, "provider.transient", False), + (DraftScriptProviderResponseError, "provider.response", False), + (DraftScriptTokenBudgetError, "provider.token_budget", False), + (DraftScriptResponseFormatError, "draft.response_format", False), + (DraftScriptGenerationError, "draft.generation", False), +) + + +def classify_failure(exc: Exception) -> Failure: + """Map launcher failures to stable public error categories.""" + for error_type, category, should_emit_invalid_tei in _FAILURE_CATEGORIES: + if isinstance(exc, error_type): + return Failure( + str(exc), + category, + should_emit_invalid_tei=should_emit_invalid_tei, + ) + return Failure(str(exc), "unexpected") diff --git a/episodic/observability.py b/episodic/observability.py index 496f2017..6075b397 100644 --- a/episodic/observability.py +++ b/episodic/observability.py @@ -1,4 +1,4 @@ -"""Shared observability ports for bounded metrics and monotonic timing. +"""Shared observability ports for tracing, bounded metrics, and monotonic timing. This module defines the canonical observability ports that adapters and services across the codebase wire against: @@ -11,6 +11,11 @@ elapsed operation time. Feature modules (for example :mod:`episodic.qa.chrono`) reuse this port directly rather than declaring parallel hierarchies. +- :class:`TracerPort` provides synchronous span contexts. The + :class:`StructuredLogTracer` adapter logs span names and only the safe + allow-list of ``operation``, ``outcome``, ``failure_category``, + ``representation``, and ``pagination``; it drops all other attributes + because callers may attach sensitive operation metadata. :class:`episodic.metrics_ports.BoundedMetricsPort` is a deliberately narrower structural subtype with ``dict[str, str]`` labels, retained because feature- @@ -21,6 +26,7 @@ """ import dataclasses as dc +import logging import time import typing as typ @@ -49,6 +55,58 @@ def observe_latency_ms( """Observe a latency measurement in milliseconds.""" +class ValueMetricsPort(MetricsPort, typ.Protocol): + """Metrics sink that additionally records bounded scalar values.""" + + def observe_value( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Observe a bounded scalar value.""" + + +class SpanHandle(typ.Protocol): + """Context handle returned when a trace span starts. + + Span handles are synchronous context managers so callers can bound an + operation with ``with tracer.start_span(...):``. + """ + + def __enter__(self) -> typ.Self: + """Enter the span context.""" + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: object, + ) -> bool: + """Complete the span without suppressing an operation exception.""" + + def set_attribute(self, name: str, value: str) -> None: + """Record one bounded operation attribute.""" + + +class TracerPort(typ.Protocol): + """Trace sink that creates synchronous operation spans. + + Attributes are string mappings so callers can provide ordinary dictionaries + or read-only mappings. Implementations must treat attributes as sensitive + operation metadata unless their own storage policy says otherwise. + """ + + def start_span( + self, + name: str, + *, + attributes: cabc.Mapping[str, str], + ) -> SpanHandle: + """Start a span named ``name`` with operation attributes.""" + + class MonotonicClockPort(typ.Protocol): """Clock port for measuring elapsed operation time.""" @@ -78,6 +136,262 @@ def observe_latency_ms( """Ignore latency observations.""" +@dc.dataclass(frozen=True, slots=True) +class NoopValueMetrics(NoopMetrics): + """Default scalar-metrics sink used when no backend is wired.""" + + def observe_value( # noqa: PLR6301 # No-op metrics intentionally retain no state. + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Ignore scalar observations.""" + del name, value, labels + + +@dc.dataclass(frozen=True, slots=True) +class StructuredLogMetrics: + """Production metrics adapter that emits bounded structured observations.""" + + logger: "_StructuredLogSink" = dc.field( # noqa: UP037 # Defined below its adapter. + default_factory=lambda: logging.getLogger(__name__), + ) + + def increment_counter( + self, + name: str, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Emit one bounded counter observation.""" + self.logger.info("metric_counter", extra={"metric_name": name, **labels}) + + def observe_latency_ms( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Emit one bounded latency observation.""" + self._emit_value("metric_latency", name, value, labels=labels) + + def observe_value( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Emit one bounded scalar observation.""" + self._emit_value("metric_value", name, value, labels=labels) + + def _emit_value( + self, + event_name: str, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Emit a structured scalar metric event.""" + self.logger.info( + event_name, + extra={"metric_name": name, "value": str(value), **labels}, + ) + + +@dc.dataclass(frozen=True, slots=True) +class _NoopSpan: + """Inert span context used by :class:`NoopTracer`.""" + + def __enter__(self) -> typ.Self: + """Enter the inert span context.""" + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: object, + ) -> bool: + """Exit without suppressing an operation exception.""" + del exc_type, exc_value, traceback + return False + + def set_attribute( # noqa: PLR6301 # No-op spans intentionally retain no state. + self, + name: str, + value: str, + ) -> None: + """Ignore one operation attribute.""" + del name, value + + +_NOOP_SPAN = _NoopSpan() + + +@dc.dataclass(frozen=True, slots=True) +class NoopTracer: + """Default tracer that adds no tracing overhead or side effects.""" + + def start_span( # noqa: PLR6301 # No-op tracer intentionally retains no state. + self, + name: str, + *, + attributes: cabc.Mapping[str, str], + ) -> SpanHandle: + """Return an inert span context.""" + del name, attributes + return _NOOP_SPAN + + +class _StructuredLogSink(typ.Protocol): + """Logger surface required by :class:`StructuredLogTracer`.""" + + def info( + self, + message: str, + /, + *, + extra: cabc.Mapping[str, str], + ) -> None: + """Emit an INFO-level structured event.""" + + +_SAFE_SPAN_ATTRIBUTES = frozenset({ + "operation", + "outcome", + "failure_category", + "representation", + "pagination", +}) + + +@dc.dataclass(slots=True) +class _StructuredLogSpan: + """Complete a structured-log span with allow-listed attributes only.""" + + logger: _StructuredLogSink + name: str + attributes: dict[str, str] + + def __enter__(self) -> typ.Self: + """Enter the structured-log span context.""" + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: object, + ) -> bool: + """Log completion without suppressing an operation exception.""" + del exc_value, traceback + event = "trace_span_completed" if exc_type is None else "trace_span_failed" + self.logger.info( + event, + extra={"span_name": self.name, **self.attributes}, + ) + return False + + def set_attribute(self, name: str, value: str) -> None: + """Retain an allow-listed bounded operation attribute.""" + if name in _SAFE_SPAN_ATTRIBUTES: + self.attributes[name] = value + + +@dc.dataclass(frozen=True, slots=True) +class StructuredLogTracer: + """Trace adapter that logs safe span lifecycle metadata. + + The adapter logs the event name, span name, and allow-listed bounded + operation attributes at start and completion. It excludes run identifiers, + paths, and other sensitive metadata from log records. + """ + + logger: _StructuredLogSink = dc.field( + default_factory=lambda: logging.getLogger(__name__), + ) + + def start_span( + self, + name: str, + *, + attributes: cabc.Mapping[str, str], + ) -> SpanHandle: + """Log a safe span start and return its completion context.""" + safe_attributes = { + key: value + for key, value in attributes.items() + if key in _SAFE_SPAN_ATTRIBUTES + } + self.logger.info( + "trace_span_started", + extra={"span_name": name, **safe_attributes}, + ) + return _StructuredLogSpan( + logger=self.logger, + name=name, + attributes=safe_attributes, + ) + + +@dc.dataclass(slots=True) +class RecordedSpan: + """Captured span data exposed by :class:`RecordingTracer` for tests.""" + + name: str + attributes: dict[str, str] + is_completed: bool = False + + +@dc.dataclass(slots=True) +class _RecordingSpan: + """Mark a recorded span complete when its context exits.""" + + record: RecordedSpan + + def __enter__(self) -> typ.Self: + """Enter the recording span context.""" + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: object, + ) -> bool: + """Record completion without suppressing an operation exception.""" + del exc_type, exc_value, traceback + self.record.is_completed = True + return False + + def set_attribute(self, name: str, value: str) -> None: + """Capture one attribute for deterministic assertions.""" + self.record.attributes[name] = value + + +@dc.dataclass(slots=True) +class RecordingTracer: + """Test adapter that records spans and their completion state.""" + + spans: list[RecordedSpan] = dc.field(default_factory=list) + + def start_span( + self, + name: str, + *, + attributes: cabc.Mapping[str, str], + ) -> SpanHandle: + """Record a span and return its completion context.""" + record = RecordedSpan(name=name, attributes=dict(attributes)) + self.spans.append(record) + return _RecordingSpan(record) + + @dc.dataclass(frozen=True, slots=True) class PerfCounterClock: """Monotonic clock backed by Python's perf counter.""" diff --git a/episodic/orchestration/_planning_orchestrator.py b/episodic/orchestration/_planning_orchestrator.py index 8ecc24c2..2f0e8994 100644 --- a/episodic/orchestration/_planning_orchestrator.py +++ b/episodic/orchestration/_planning_orchestrator.py @@ -21,6 +21,7 @@ if typ.TYPE_CHECKING: import collections.abc as cabc + from episodic.cost import CostRecorderPort from episodic.llm import ProviderCallUsage from ._dto import ( @@ -31,7 +32,7 @@ PlannedAction, PlannerResult, ) - from ._protocols import CostRecorderPort, PlannerPort, ToolExecutorPort + from ._protocols import PlannerPort, ToolExecutorPort _DEFAULT_PROVIDER_NAME = "openai" diff --git a/episodic/orchestration/_protocols.py b/episodic/orchestration/_protocols.py index 43fe0cc4..d4453c4f 100644 --- a/episodic/orchestration/_protocols.py +++ b/episodic/orchestration/_protocols.py @@ -12,8 +12,6 @@ import typing as typ if typ.TYPE_CHECKING: - from episodic.cost.ports import BillingPeriodKey, CostLedgerEntryId - from episodic.cost.recorder import CostProviderOperation, ProviderCallRecord from episodic.generation import GuestBioSource, GuestBiosResult, ShowNotesResult from episodic.orchestration._dto import ( ActionExecutionResult, @@ -46,31 +44,6 @@ async def plan( """Return a typed execution plan for the supplied generation request.""" -class CostRecorderPort(typ.Protocol): - """Application-level port for cost-ledger side effects.""" - - async def pin_run_pricing( - self, - workflow_run_id: str, - providers: tuple[CostProviderOperation, ...], - billing_period_key: BillingPeriodKey, - ) -> None: - """Pin pricing snapshots for a run before provider calls are recorded.""" - - async def record_provider_call( - self, - record: ProviderCallRecord, - ) -> CostLedgerEntryId: - """Record one priced provider call.""" - - async def finalize_run( - self, - workflow_run_id: str, - workflow_node: str | None, - ) -> CostLedgerEntryId: - """Write the final task roll-up for a completed run.""" - - class CheckpointPort(typ.Protocol): """Persistence port for suspended generation workflow checkpoints.""" diff --git a/episodic/orchestration/langgraph.py b/episodic/orchestration/langgraph.py index 888d292e..cfd67aa9 100644 --- a/episodic/orchestration/langgraph.py +++ b/episodic/orchestration/langgraph.py @@ -63,7 +63,7 @@ from langgraph.graph.state import CompiledStateGraph - from episodic.cost import BillingPeriodKey + from episodic.cost import BillingPeriodKey, CostRecorderPort from episodic.orchestration import _dto as dto from episodic.orchestration import _protocols as protocols else: @@ -95,7 +95,7 @@ class GenerationGraphExtensions: finish_callback: cabc.Callable[[dto.GenerationOrchestrationResult], None] | None = ( None ) - cost_recorder: protocols.CostRecorderPort | None = None + cost_recorder: CostRecorderPort | None = None async def _plan_node( @@ -280,7 +280,7 @@ def _invoke_finish_callback( async def _record_planner_cost_if_available( - cost_recorder: protocols.CostRecorderPort, + cost_recorder: CostRecorderPort, *, workflow_run_id: str, planner_result: dto.PlannerResult, @@ -305,7 +305,7 @@ async def _record_planner_cost_if_available( async def _record_action_costs_from_results( - cost_recorder: protocols.CostRecorderPort, + cost_recorder: CostRecorderPort, *, workflow_run_id: str, action_results: tuple[dto.ActionExecutionResult, ...], @@ -333,7 +333,7 @@ async def _record_action_costs_from_results( async def _record_costs_from_finished_state( state: GenerationGraphState, *, - cost_recorder: protocols.CostRecorderPort | None, + cost_recorder: CostRecorderPort | None, ) -> None: """Record graph provider-call costs from the finished direct path.""" if cost_recorder is None: diff --git a/pyproject.toml b/pyproject.toml index 66885ab4..503504de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -531,8 +531,15 @@ full_name = [ "episodic.observability.NoopMetrics.observe_latency_ms.name", "episodic.observability.NoopMetrics.observe_latency_ms.value", "episodic.observability.NoopMetrics.observe_latency_ms.labels", + "episodic.observability.NoopValueMetrics.observe_value.name", + "episodic.observability.NoopValueMetrics.observe_value.value", + "episodic.observability.NoopValueMetrics.observe_value.labels", + "episodic.observability.NoopTracer.start_span.name", + "episodic.observability.NoopTracer.start_span.attributes", + "episodic.observability._NoopSpan.set_attribute.name", + "episodic.observability._NoopSpan.set_attribute.value", ] -reason = "No-op metrics adapters intentionally retain the production metrics protocol signatures." +reason = "No-op metrics and tracing adapters intentionally retain their runtime-selected protocol signatures." [[tool.skylos.dead_code.entrypoints]] type = "function" @@ -544,6 +551,9 @@ full_name = [ "alembic.versions.20260610_000009_add_source_intake_tables._drop_uploads_table", "alembic.versions.20260610_000009_add_source_intake_tables._drop_ingestion_job_sources_table", "alembic.versions.20260610_000009_add_source_intake_tables._drop_idempotency_records_table", + "alembic.versions.20260624_000010_add_generation_run_tables._drop_enums", + "alembic.versions.20260624_000010_add_generation_run_tables._drop_generation_runs_table", + "alembic.versions.20260624_000010_add_generation_run_tables._drop_generation_events_table", ] reason = "Alembic invokes these downgrade helpers when an operator rolls migration revisions back." @@ -562,8 +572,28 @@ type = "method" full_name = [ "episodic.api.app._ShutdownHooksMiddleware.process_shutdown", "episodic.api.authorization.AuthorizationMiddleware.process_request", + "episodic.api.app._GenerationRouteMetricsMiddleware.process_request", + "episodic.api.app._GenerationRouteMetricsMiddleware.process_response", ] -reason = "Falcon invokes these middleware lifecycle callbacks through its ASGI request and shutdown protocols." +reason = "Falcon invokes these middleware lifecycle callbacks through its ASGI request, response, and shutdown protocols." + +[[tool.skylos.dead_code.entrypoints]] +type = "parameter" +full_name = [ + "episodic.api.app._GenerationRouteMetricsMiddleware.process_request.resp", + "episodic.api.app._GenerationRouteMetricsMiddleware.process_response.resource", + "episodic.api.app._GenerationRouteMetricsMiddleware.process_response.req_succeeded", +] +reason = "Falcon supplies these required middleware callback parameters although bounded metrics do not inspect them." + +[[tool.skylos.dead_code.entrypoints]] +type = "function" +full_name = [ + "episodic.api.app._generation_route_operation", + "episodic.api.app._response_outcome", + "episodic.canonical.adapters.generation_runs._event_page_minimum_seq", +] +reason = "Falcon middleware and the in-memory event-list adapter call these private helpers through registered protocol callbacks." [[tool.skylos.dead_code.entrypoints]] type = "function" @@ -576,6 +606,8 @@ full_name = [ "episodic.canonical.ingestion_sources._validate_weight", "episodic.canonical.uploads._require_non_negative", "episodic.canonical.uploads._require_non_negative_if_present", + "episodic.canonical.domain._require_positive_integer", + "episodic.canonical.domain._validate_draft_without_qa_metadata", ] reason = "Canonical value objects call these validators from dataclass post-initialization hooks." @@ -596,9 +628,12 @@ full_name = [ "episodic.concurrent_interpreters._PerfCounterCpuTaskExecutorClock.monotonic_seconds", "episodic.observability.NoopMetrics.increment_counter", "episodic.observability.NoopMetrics.observe_latency_ms", + "episodic.observability.NoopValueMetrics.observe_value", + "episodic.observability.NoopTracer.start_span", + "episodic.observability.RecordingTracer.start_span", "episodic.observability.PerfCounterClock.monotonic_seconds", ] -reason = "Clock and no-op metrics implementations satisfy runtime-selected protocol interfaces." +reason = "Clock, no-op, and recording adapters satisfy runtime-selected protocol interfaces." [[tool.skylos.dead_code.entrypoints]] type = "function" @@ -660,8 +695,19 @@ full_name = [ "episodic.api.runtime._normalize_database_urls", "episodic.api.runtime._apply_query_connect_overrides", "episodic.api.runtime._psycopg_connection_kwargs", + "episodic.api.runtime._build_llm_port", + "episodic.api.runtime._build_generation_launcher", + "episodic.api.runtime._build_generation_launcher._cost_recorder", + "episodic.api.runtime.create_app_from_env.shutdown_generation", ] -reason = "The Granian application factory and its database readiness composition call this runtime helper chain." +reason = "The Granian application factory calls this runtime composition and shutdown helper chain." + +[[tool.skylos.dead_code.entrypoints]] +type = "method" +full_name = [ + "episodic.api.authorization.StaticBearerTokenAuthorization.decide", +] +reason = "Authorization middleware dispatches to this production AuthorizationPort implementation." [[tool.skylos.dead_code.entrypoints]] type = "method" @@ -675,12 +721,33 @@ reason = "Health observers invoke these methods through the readiness callback a [[tool.skylos.dead_code.entrypoints]] type = "class" full_name = [ - "episodic.canonical.adapters.generation_runs._CheckpointTransitionSpec", + "episodic.canonical.adapters.generation_checkpoints._CheckpointTransitionSpec", "episodic.canonical.adapters.generation_runs.InMemoryGenerationRunStore", "episodic.cost.recorder.CostRecorder", ] reason = "These documented public adapters are instantiated by application composition and local-development consumers." +[[tool.skylos.dead_code.entrypoints]] +type = "class" +full_name = [ + "episodic.api.runtime._GenerationLauncherRuntime", + "episodic.observability._StructuredLogSpan", + "episodic.observability._RecordingSpan", + "episodic.observability.RecordingTracer", +] +reason = "Runtime composition and observability adapters instantiate these supporting types through protocol-facing entry points." + +[[tool.skylos.dead_code.entrypoints]] +type = "method" +full_name = [ + "episodic.observability.StructuredLogMetrics.increment_counter", + "episodic.observability.StructuredLogMetrics.observe_latency_ms", + "episodic.observability.StructuredLogMetrics.observe_value", + "episodic.observability.StructuredLogMetrics._emit_value", + "episodic.observability.StructuredLogTracer.start_span", +] +reason = "Metrics and tracing adapters implement runtime-selected observability protocols and their bounded emission path." + [[tool.skylos.dead_code.entrypoints]] type = "variable" full_name = [ @@ -738,8 +805,11 @@ prefixes = [ "episodic.canonical.ingestion", "episodic.canonical.ingestion_ports", "episodic.canonical.entity_protocols", + "episodic.canonical.episode_errors", + "episodic.canonical.generation_quality", "episodic.canonical.history_protocols", "episodic.canonical.health", + "episodic.canonical.hashing", "episodic.canonical.idempotency", "episodic.canonical.ingestion_sources", "episodic.canonical.object_store", @@ -762,6 +832,7 @@ prefixes = [ "episodic.canonical.services", "episodic.canonical.ingestion_service", "episodic.canonical.idempotency_service", + "episodic.canonical.generation_persistence", "episodic.canonical.source_intake_service", "episodic.canonical.profile_templates", "episodic.canonical.reference_documents", diff --git a/tests/__snapshots__/test_draft_script_generation.ambr b/tests/__snapshots__/test_draft_script_generation.ambr new file mode 100644 index 00000000..e1ebe5ad --- /dev/null +++ b/tests/__snapshots__/test_draft_script_generation.ambr @@ -0,0 +1,4 @@ +# serializer version: 1 +# name: test_draft_script_generator_emits_valid_stable_tei + 'Bridgewater FuturesWelcome to Bridgewater Futures.Thanks for inviting me.

The conversation turns to implementation risks.

' +# --- diff --git a/tests/__snapshots__/test_generation_run_domain.ambr b/tests/__snapshots__/test_generation_run_domain.ambr index 28cbd7c4..1da93299 100644 --- a/tests/__snapshots__/test_generation_run_domain.ambr +++ b/tests/__snapshots__/test_generation_run_domain.ambr @@ -13,7 +13,7 @@ # name: test_generation_run_and_checkpoint_repr_snapshot dict({ 'checkpoint': "Checkpoint(id=UUID('018fdcf0-0000-7000-8000-000000000004'), generation_run_id=UUID('018fdcf0-0000-7000-8000-000000000001'), node='human_review', prompt='Approve the draft?', options=('approve', 'request_changes'), status=, created_at=datetime.datetime(2026, 6, 4, 8, 0, tzinfo=datetime.timezone.utc), responded_at=None, responded_by=None, response_action=None, response_payload={})", - 'generation_run': "GenerationRun(id=UUID('018fdcf0-0000-7000-8000-000000000001'), episode_id=UUID('018fdcf0-0000-7000-8000-000000000002'), source_bundle_id=UUID('018fdcf0-0000-7000-8000-000000000003'), actor='editor@example.com', status=, current_node=None, budget_snapshot={'limit': 10}, configuration={'model': 'gpt-4.1'}, created_at=datetime.datetime(2026, 6, 4, 8, 0, tzinfo=datetime.timezone.utc), updated_at=datetime.datetime(2026, 6, 4, 8, 0, tzinfo=datetime.timezone.utc), started_at=None, ended_at=None, error_message=None)", + 'generation_run': "GenerationRun(id=UUID('018fdcf0-0000-7000-8000-000000000001'), episode_id=UUID('018fdcf0-0000-7000-8000-000000000002'), source_bundle_id=UUID('018fdcf0-0000-7000-8000-000000000003'), actor='editor@example.com', status=, current_node=None, budget_snapshot={'limit': 10}, configuration={'model': 'gpt-4.1'}, created_at=datetime.datetime(2026, 6, 4, 8, 0, tzinfo=datetime.timezone.utc), updated_at=datetime.datetime(2026, 6, 4, 8, 0, tzinfo=datetime.timezone.utc), started_at=None, ended_at=None, error_message=None, error_category=None, quality_mode=, qa_status=, skip_qa_rationale='No-QA vertical-slice draft.')", }) # --- # name: test_generation_run_error_messages_snapshot diff --git a/tests/canonical_storage/_generation_run_support.py b/tests/canonical_storage/_generation_run_support.py new file mode 100644 index 00000000..4a5d7028 --- /dev/null +++ b/tests/canonical_storage/_generation_run_support.py @@ -0,0 +1,177 @@ +"""Shared generation-run SQL storage test support.""" + +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import sqlalchemy as sa + +from episodic.canonical.domain import ( + ApprovalState, + CanonicalEpisode, + EpisodeStatus, + GenerationRun, + GenerationRunStatus, + IngestionJob, + IngestionStatus, + SeriesProfile, + TeiHeader, +) +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.storage import SqlAlchemyUnitOfWork + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.storage.models_base import Base + + +NOW = dt.datetime(2026, 6, 24, 8, 30, tzinfo=dt.UTC) + + +@dc.dataclass(frozen=True, slots=True) +class GenerationRunFixture: + """Optional durable generation-run values for SQL storage tests.""" + + run_id: uuid.UUID | None = None + episode_id: uuid.UUID | None = None + source_bundle_id: uuid.UUID | None = None + status: GenerationRunStatus = GenerationRunStatus.PENDING + created_at: dt.datetime = NOW + + +def make_generation_run( + fixture: GenerationRunFixture | None = None, +) -> GenerationRun: + """Build a no-QA generation run for storage tests.""" + fixture = fixture or GenerationRunFixture() + return GenerationRun( + id=fixture.run_id or uuid.uuid7(), + episode_id=fixture.episode_id or uuid.uuid7(), + source_bundle_id=fixture.source_bundle_id or uuid.uuid7(), + actor="editor@example.com", + status=fixture.status, + current_node=None, + budget_snapshot={"limit_usd": "5.00"}, + configuration={"quality_mode": QualityMode.DRAFT_WITHOUT_QA.value}, + created_at=fixture.created_at, + updated_at=fixture.created_at, + started_at=None, + ended_at=None, + error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", + ) + + +async def persist_generation_run_prerequisites( + session_factory: async_sessionmaker[AsyncSession], + *runs: GenerationRun, +) -> None: + """Persist the episode and source-bundle rows referenced by test runs.""" + series_id = uuid.uuid7() + series = SeriesProfile( + id=series_id, + slug=f"generation-run-{series_id}", + title="Generation run test series", + description=None, + configuration={}, + guardrails={}, + created_at=NOW, + updated_at=NOW, + ) + headers: dict[uuid.UUID, TeiHeader] = {} + episodes: dict[uuid.UUID, CanonicalEpisode] = {} + jobs: dict[uuid.UUID, IngestionJob] = {} + + for run in runs: + if run.episode_id not in episodes: + header = TeiHeader( + id=uuid.uuid7(), + title="Generation run test episode", + payload={}, + raw_xml="", + created_at=NOW, + updated_at=NOW, + ) + headers[run.episode_id] = header + episodes[run.episode_id] = CanonicalEpisode( + id=run.episode_id, + series_profile_id=series.id, + tei_header_id=header.id, + title=header.title, + tei_xml=header.raw_xml, + status=EpisodeStatus.DRAFT, + approval_state=ApprovalState.DRAFT, + created_at=NOW, + updated_at=NOW, + ) + if run.source_bundle_id not in jobs: + jobs[run.source_bundle_id] = IngestionJob( + id=run.source_bundle_id, + series_profile_id=series.id, + target_episode_id=run.episode_id, + status=IngestionStatus.COMPLETED, + requested_at=NOW, + started_at=NOW, + completed_at=NOW, + error_message=None, + created_at=NOW, + updated_at=NOW, + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.series_profiles.add(series) + for header in headers.values(): + await uow.tei_headers.add(header) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + for episode in episodes.values(): + await uow.episodes.add(episode) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + for job in jobs.values(): + await uow.ingestion_jobs.add(job) + await uow.commit() + + +async def count_records( + session_factory: async_sessionmaker[AsyncSession], + record_type: type[Base], +) -> int: + """Count persisted records of one SQLAlchemy model type.""" + async with session_factory() as session: + result = await session.execute( + sa.select(sa.func.count()).select_from(record_type) + ) + return result.scalar_one() + + +@dc.dataclass(frozen=True, slots=True) +class ExecutionClaim: + """Claim values passed to the durable generation-run adapter.""" + + current_node: str | None + started_at: dt.datetime + lease_expires_at: dt.datetime | None + + +async def claim_run_in_independent_uow( + session_factory: async_sessionmaker[AsyncSession], + run_id: uuid.UUID, + claim: ExecutionClaim, +) -> GenerationRun | None: + """Claim a run within a separately committed unit of work.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + claimed = await uow.generation_runs.claim_run_for_execution( + run_id, + current_node=claim.current_node, + started_at=claim.started_at, + lease_expires_at=claim.lease_expires_at, + ) + await uow.commit() + return claimed diff --git a/tests/canonical_storage/test_episode_tei_updates.py b/tests/canonical_storage/test_episode_tei_updates.py new file mode 100644 index 00000000..969496b4 --- /dev/null +++ b/tests/canonical_storage/test_episode_tei_updates.py @@ -0,0 +1,369 @@ +"""Tests for optimistic episode TEI updates.""" + +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +import pytest +import sqlalchemy as sa + +from episodic.canonical.domain import ( + EpisodeTeiUpdate, + GenerationRun, + GenerationRunStatus, +) +from episodic.canonical.episode_errors import ( + EpisodeNotFoundError, + EpisodeRevisionConflictError, +) +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.storage.entity_mappers import _episode_to_record +from episodic.canonical.storage.models import EpisodeRecord + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.domain import ( + CanonicalEpisode, + IngestionJob, + SeriesProfile, + SourceDocument, + TeiHeader, + ) + + +def _tei_hash(tei_xml: str) -> str: + """Return the persisted content hash for an episode TEI payload.""" + return f"sha256:{hashlib.sha256(tei_xml.encode()).hexdigest()}" + + +def _generation_run( + episode: CanonicalEpisode, + ingestion_job: IngestionJob, +) -> GenerationRun: + """Return a generation run linked to an episode fixture.""" + return GenerationRun( + id=uuid.uuid7(), + episode_id=episode.id, + source_bundle_id=ingestion_job.id, + actor="storage-test", + status=GenerationRunStatus.PENDING, + current_node=None, + budget_snapshot={}, + configuration={}, + created_at=episode.created_at, + updated_at=episode.updated_at, + started_at=None, + ended_at=None, + error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="Storage test bypasses QA.", + ) + + +def test_episode_revision_rejects_boolean( + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Boolean revisions must not pass as integers.""" + _, _, episode, _, _ = episode_fixture + + with pytest.raises(ValueError, match="positive integer"): + dc.replace(episode, tei_revision=True) + + +def test_episode_content_hash_requires_string( + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Set content hashes must be strings before whitespace validation.""" + _, _, episode, _, _ = episode_fixture + + with pytest.raises(TypeError, match="must be a string"): + dc.replace(episode, tei_content_hash=typ.cast("typ.Any", 42)) + + +def test_episode_mapper_derives_tei_content_hash( + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Storage records derive their TEI hash from the mapped XML.""" + _, _, episode, _, _ = episode_fixture + stale_episode = dc.replace(episode, tei_content_hash="sha256:stale") + + record = _episode_to_record(stale_episode) + + assert record.tei_content_hash == _tei_hash(episode.tei_xml), ( + f"record hash: {record.tei_content_hash!r}" + ) + + +@pytest.mark.parametrize( + ("overrides", "error_type", "message"), + [ + ({"tei_xml": 42}, TypeError, "tei_xml must be a string"), + ({"qa_status": None}, ValueError, "qa_status must be set"), + ( + {"last_generation_run_id": None}, + ValueError, + "last_generation_run_id must be set", + ), + ({"expected_revision": True}, ValueError, "positive integer"), + ], +) +def test_episode_tei_update_rejects_invalid_domain_values( + overrides: dict[str, object], + error_type: type[Exception], + message: str, +) -> None: + """TEI updates require typed content, provenance, and exact revisions.""" + values: dict[str, object] = { + "tei_xml": "", + "qa_status": QaStatus.SKIPPED, + "last_generation_run_id": uuid.uuid7(), + "expected_revision": 1, + "updated_at": dt.datetime(2026, 6, 24, tzinfo=dt.UTC), + } + values.update(overrides) + + with pytest.raises(error_type, match=message): + EpisodeTeiUpdate(**typ.cast("typ.Any", values)) + + +async def _persist_episode_parents( + factory: async_sessionmaker[AsyncSession], + series: SeriesProfile, + header: TeiHeader, +) -> None: + """Persist the parent rows required by an episode fixture.""" + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.series_profiles.add(series) + await uow.tei_headers.add(header) + await uow.commit() + + +async def _persist_episode_and_ingestion_job( + uow: SqlAlchemyUnitOfWork, + episode: CanonicalEpisode, + ingestion_job: IngestionJob, +) -> None: + """Persist the generation-run foreign-key parents in dependency order.""" + await uow.episodes.add(episode) + await uow.flush() + await uow.ingestion_jobs.add(ingestion_job) + await uow.flush() + + +@pytest.mark.asyncio +async def test_episode_update_tei_records_revision_and_generation_metadata( + session_factory: async_sessionmaker[AsyncSession], + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Updating episode TEI should persist the no-QA generation metadata.""" + series, header, episode, ingestion_job, _ = episode_fixture + run = _generation_run(episode, ingestion_job) + updated_xml = "

Generated script.

" + updated_at = dt.datetime(2026, 6, 24, 12, 0, tzinfo=dt.UTC) + await _persist_episode_parents(session_factory, series, header) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await _persist_episode_and_ingestion_job(uow, episode, ingestion_job) + await uow.generation_runs.create_run(run) + updated = await uow.episodes.update( + episode.id, + update=EpisodeTeiUpdate( + tei_xml=updated_xml, + qa_status=QaStatus.SKIPPED, + last_generation_run_id=run.id, + expected_revision=1, + updated_at=updated_at, + ), + ) + await uow.commit() + + assert updated.tei_xml == updated_xml, ( + f"expected updated TEI {updated_xml!r}, got {updated.tei_xml!r}" + ) + assert updated.tei_revision == 2, ( + f"expected TEI revision 2, got {updated.tei_revision}" + ) + assert updated.tei_content_hash == _tei_hash(updated_xml), ( + f"expected TEI hash {_tei_hash(updated_xml)!r}, " + f"got {updated.tei_content_hash!r}" + ) + assert updated.qa_status is QaStatus.SKIPPED, ( + f"expected skipped QA status, got {updated.qa_status!r}" + ) + assert updated.last_generation_run_id == run.id, ( + f"expected generation run {run.id}, got {updated.last_generation_run_id}" + ) + assert updated.updated_at == updated_at, ( + f"expected update time {updated_at!r}, got {updated.updated_at!r}" + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + fetched = await uow.episodes.get(episode.id) + + assert fetched is not None, f"expected episode {episode.id}, got {fetched!r}" + assert fetched.tei_xml == updated_xml, ( + f"expected persisted TEI {updated_xml!r}, got {fetched.tei_xml!r}" + ) + assert fetched.tei_revision == 2, ( + f"expected persisted revision 2, got {fetched.tei_revision}" + ) + assert fetched.tei_content_hash == _tei_hash(updated_xml), ( + f"expected persisted hash {_tei_hash(updated_xml)!r}, " + f"got {fetched.tei_content_hash!r}" + ) + assert fetched.qa_status is QaStatus.SKIPPED, ( + f"expected persisted skipped QA status, got {fetched.qa_status!r}" + ) + assert fetched.last_generation_run_id == run.id, ( + f"expected persisted run {run.id}, got {fetched.last_generation_run_id}" + ) + + +@pytest.mark.asyncio +async def test_episode_update_tei_keeps_compressed_storage_in_sync( + session_factory: async_sessionmaker[AsyncSession], + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Large updated TEI payloads should refresh compressed storage columns.""" + series, header, episode, ingestion_job, _ = episode_fixture + run = _generation_run(episode, ingestion_job) + updated_xml = "" + ("generated episode " * 1200) + "" + await _persist_episode_parents(session_factory, series, header) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await _persist_episode_and_ingestion_job(uow, episode, ingestion_job) + await uow.generation_runs.create_run(run) + await uow.episodes.update( + episode.id, + update=EpisodeTeiUpdate( + tei_xml=updated_xml, + qa_status=QaStatus.SKIPPED, + last_generation_run_id=run.id, + expected_revision=1, + updated_at=dt.datetime(2026, 6, 24, tzinfo=dt.UTC), + ), + ) + await uow.commit() + + async with session_factory() as session: + result = await session.execute( + sa.select(EpisodeRecord).where(EpisodeRecord.id == episode.id) + ) + record = result.scalar_one() + + assert record.tei_xml == "__zstd__", ( + f"expected compressed-storage marker, got {record.tei_xml!r}" + ) + assert record.tei_xml_zstd is not None, ( + f"expected compressed TEI bytes, got {record.tei_xml_zstd!r}" + ) + assert record.tei_revision == 2, ( + f"expected compressed TEI revision 2, got {record.tei_revision}" + ) + assert record.tei_content_hash == _tei_hash(updated_xml), ( + f"expected compressed TEI hash {_tei_hash(updated_xml)!r}, " + f"got {record.tei_content_hash!r}" + ) + assert record.qa_status is QaStatus.SKIPPED, ( + f"expected compressed record QA status skipped, got {record.qa_status!r}" + ) + assert record.last_generation_run_id == run.id, ( + f"expected compressed record run {run.id}, got {record.last_generation_run_id}" + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + fetched = await uow.episodes.get(episode.id) + + assert fetched is not None, f"expected episode {episode.id}, got {fetched!r}" + assert fetched.tei_xml == updated_xml, ( + "expected decompressed TEI length " + f"{len(updated_xml)}, got {len(fetched.tei_xml)}" + ) + + +@pytest.mark.asyncio +async def test_episode_update_tei_rejects_stale_revision( + session_factory: async_sessionmaker[AsyncSession], + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Updating with a stale expected revision should raise a conflict.""" + series, header, episode, ingestion_job, _ = episode_fixture + run = _generation_run(episode, ingestion_job) + await _persist_episode_parents(session_factory, series, header) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await _persist_episode_and_ingestion_job(uow, episode, ingestion_job) + await uow.generation_runs.create_run(run) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(EpisodeRevisionConflictError): + await uow.episodes.update( + episode.id, + update=EpisodeTeiUpdate( + tei_xml="stale", + qa_status=QaStatus.SKIPPED, + last_generation_run_id=run.id, + expected_revision=2, + updated_at=dt.datetime(2026, 6, 24, tzinfo=dt.UTC), + ), + ) + + +@pytest.mark.asyncio +async def test_episode_update_tei_rejects_unknown_episode( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Updating an absent episode retains the canonical not-found error.""" + with pytest.raises(EpisodeNotFoundError) as raised: + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.episodes.update( + uuid.uuid7(), + update=EpisodeTeiUpdate( + tei_xml="missing", + qa_status=QaStatus.SKIPPED, + last_generation_run_id=uuid.uuid7(), + expected_revision=1, + updated_at=dt.datetime(2026, 6, 24, tzinfo=dt.UTC), + ), + ) + + assert isinstance(raised.value, EpisodeNotFoundError), raised.value diff --git a/tests/canonical_storage/test_generation_run_claims.py b/tests/canonical_storage/test_generation_run_claims.py new file mode 100644 index 00000000..971e7bac --- /dev/null +++ b/tests/canonical_storage/test_generation_run_claims.py @@ -0,0 +1,350 @@ +"""Generation-run execution-claim SQLAlchemy adapter contract tests.""" + +import asyncio +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import pytest +import sqlalchemy as sa + +from episodic.canonical.domain import GenerationRun, GenerationRunStatus +from episodic.canonical.generation_run_errors import RunAlreadyTerminal, RunNotFound +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.storage import generation_runs as generation_runs_module +from episodic.canonical.storage.generation_run_models import GenerationRunRecord +from episodic.canonical.storage.generation_runs import SqlAlchemyGenerationRunStore +from tests.canonical_storage._generation_run_support import ( + NOW, + ExecutionClaim, + claim_run_in_independent_uow, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@dc.dataclass(frozen=True, slots=True) +class _ClaimOutcomeLogExpectations: + """Expected identifiers and lease timestamp for claim-outcome logs.""" + + pending_run_id: uuid.UUID + missing_run_id: uuid.UUID + terminal_run_id: uuid.UUID + lease_expires_at: dt.datetime + + +async def _manually_fail_expired_run( + session_factory: async_sessionmaker[AsyncSession], + run_id: uuid.UUID, + *, + now: dt.datetime, +) -> bool: + """Apply the documented manual-recovery transaction for one run.""" + async with session_factory() as session: + record = await session.scalar( + sa + .select(GenerationRunRecord) + .where( + GenerationRunRecord.id == run_id, + GenerationRunRecord.status == GenerationRunStatus.RUNNING, + GenerationRunRecord.lease_expires_at.is_not(None), + GenerationRunRecord.lease_expires_at <= now, + ) + .with_for_update() + ) + if record is None: + await session.rollback() + return False + + store = SqlAlchemyGenerationRunStore(session) + await store.append_event( + run_id, + kind="run.failed", + payload={ + "error_category": "launcher.lease_expired", + "error_message": "Generation lease expired; failed manually.", + }, + occurred_at=now, + ) + await store.update_run_status( + run_id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.FAILED, + current_node="failed", + ended_at=now, + error_message="Generation lease expired; failed manually.", + error_category="launcher.lease_expired", + ), + ) + await session.commit() + return True + + +async def _assert_manual_recovery_state( + session_factory: async_sessionmaker[AsyncSession], + *, + expired_run_id: uuid.UUID, + non_expired_run_id: uuid.UUID, + terminal_run_id: uuid.UUID, +) -> None: + """Assert that only the expired run was recovered.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + expired = await uow.generation_runs.get_run(expired_run_id) + non_expired = await uow.generation_runs.get_run(non_expired_run_id) + terminal = await uow.generation_runs.get_run(terminal_run_id) + expired_events = await uow.generation_runs.list_events(expired_run_id) + non_expired_events = await uow.generation_runs.list_events(non_expired_run_id) + terminal_events = await uow.generation_runs.list_events(terminal_run_id) + + assert expired is not None, "expired run was not persisted" + assert expired.status is GenerationRunStatus.FAILED, ( + f"expired run recovery state: {expired!r}" + ) + assert expired.error_category == "launcher.lease_expired", ( + f"expired run failure category: {expired!r}" + ) + assert [event.kind for event in expired_events] == ["run.failed"], ( + f"expired run recovery events: {expired_events!r}" + ) + assert non_expired is not None, "non-expired run was not persisted" + assert non_expired.status is GenerationRunStatus.RUNNING, ( + f"non-expired run recovery state: {non_expired!r}" + ) + assert terminal is not None, "terminal run was not persisted" + assert terminal.status is GenerationRunStatus.SUCCEEDED, ( + f"terminal run recovery state: {terminal!r}" + ) + assert non_expired_events == (), ( + f"non-expired recovery must not append events: {non_expired_events!r}" + ) + assert terminal_events == (), ( + f"terminal recovery must not append events: {terminal_events!r}" + ) + + +def _assert_claim_outcome_logs( + events: list[tuple[str, str, dict[str, object]]], + expected: _ClaimOutcomeLogExpectations, +) -> None: + """Assert the ordered structured-log contract for every claim outcome.""" + observed_event_names = [message for _, message, _ in events] + assert observed_event_names == [ + "sql_generation_run_store.claim_run", + "sql_generation_run_store.claim_run_lost", + "sql_generation_run_store.claim_run_missing", + "sql_generation_run_store.claim_run_terminal", + ], events + events_by_name = {message: (level, fields) for level, message, fields in events} + assert len(events_by_name) == len(events) == 4, events + claimed_log = events_by_name["sql_generation_run_store.claim_run"] + assert claimed_log[0] == "info", events + assert claimed_log[1]["run_id"] == str(expected.pending_run_id), events + assert claimed_log[1]["current_node"] == "draft", events + assert ( + claimed_log[1]["lease_expires_at"] == expected.lease_expires_at.isoformat() + ), events + assert events_by_name["sql_generation_run_store.claim_run_lost"] == ( + "info", + {"run_id": str(expected.pending_run_id), "status": "running"}, + ), events + assert events_by_name["sql_generation_run_store.claim_run_missing"] == ( + "warning", + {"run_id": str(expected.missing_run_id)}, + ), events + assert events_by_name["sql_generation_run_store.claim_run_terminal"] == ( + "warning", + {"run_id": str(expected.terminal_run_id), "status": "succeeded"}, + ), events + + +@pytest.mark.asyncio +async def test_generation_run_store_claims_pending_run_once_concurrently( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Two coordinated sessions should produce one winner and one lost claim.""" + run = make_generation_run() + lease_expires_at = NOW + dt.timedelta(minutes=5) + claim_barrier = asyncio.Barrier(2) + await persist_generation_run_prerequisites(session_factory, run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(run) + await uow.commit() + + async def claim_from_independent_session( + current_node: str, + ) -> GenerationRun | None: + await claim_barrier.wait() + return await claim_run_in_independent_uow( + session_factory, + run.id, + ExecutionClaim( + current_node=current_node, + started_at=NOW, + lease_expires_at=lease_expires_at, + ), + ) + + claims = await asyncio.gather( + claim_from_independent_session("draft-a"), + claim_from_independent_session("draft-b"), + ) + + running_claims = [claim for claim in claims if claim is not None] + assert len(running_claims) == 1, f"expected one running claim, got {claims!r}" + assert sum(claim is None for claim in claims) == 1, ( + f"expected one lost claim, got {claims!r}" + ) + assert running_claims[0].status is GenerationRunStatus.RUNNING, ( + f"expected running claim, got {running_claims[0]!r}" + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + persisted = await uow.generation_runs.get_run(run.id) + + assert persisted is not None, f"expected persisted run {run.id}, got {persisted!r}" + assert persisted.status is GenerationRunStatus.RUNNING, ( + f"expected persisted running state, got {persisted.status!r}" + ) + + +@pytest.mark.asyncio +async def test_manual_recovery_only_fails_expired_running_leases( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Manual recovery rolls back non-qualifying rows without appending events.""" + expired_run = make_generation_run() + non_expired_run = make_generation_run() + terminal_run = make_generation_run() + await persist_generation_run_prerequisites( + session_factory, + expired_run, + non_expired_run, + terminal_run, + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + for run in (expired_run, non_expired_run, terminal_run): + await uow.generation_runs.create_run(run) + await uow.commit() + + for run, lease_expires_at in ( + (expired_run, NOW - dt.timedelta(seconds=1)), + (non_expired_run, NOW + dt.timedelta(seconds=1)), + ): + claimed = await claim_run_in_independent_uow( + session_factory, + run.id, + ExecutionClaim( + current_node="draft", + started_at=NOW, + lease_expires_at=lease_expires_at, + ), + ) + assert claimed is not None, f"expected running claim for {run.id}" + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.update_run_status( + terminal_run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.SUCCEEDED, + current_node=None, + ended_at=NOW, + ), + ) + await uow.commit() + + recovered = [ + await _manually_fail_expired_run(session_factory, expired_run.id, now=NOW), + await _manually_fail_expired_run( + session_factory, + non_expired_run.id, + now=NOW, + ), + await _manually_fail_expired_run(session_factory, terminal_run.id, now=NOW), + ] + assert recovered == [True, False, False], f"manual recovery results: {recovered!r}" + + await _assert_manual_recovery_state( + session_factory, + expired_run_id=expired_run.id, + non_expired_run_id=non_expired_run.id, + terminal_run_id=terminal_run.id, + ) + + +@pytest.mark.asyncio +async def test_generation_run_store_logs_claim_outcomes( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Claim outcomes should emit bounded structured operational fields.""" + pending_run = make_generation_run() + terminal_run = make_generation_run() + lease_expires_at = NOW + dt.timedelta(minutes=5) + execution_claim = ExecutionClaim( + current_node="draft", + started_at=NOW, + lease_expires_at=lease_expires_at, + ) + await persist_generation_run_prerequisites( + session_factory, pending_run, terminal_run + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(pending_run) + await uow.generation_runs.create_run(terminal_run) + await uow.generation_runs.update_run_status( + terminal_run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.SUCCEEDED, + current_node=None, + ended_at=NOW, + ), + ) + await uow.commit() + events: list[tuple[str, str, dict[str, object]]] = [] + + def capture_log_event(level: str, message: str, **fields: object) -> None: + events.append((level, message, fields)) + + monkeypatch.setattr(generation_runs_module, "_log_event", capture_log_event) + claimed = await claim_run_in_independent_uow( + session_factory, + pending_run.id, + execution_claim, + ) + lost = await claim_run_in_independent_uow( + session_factory, + pending_run.id, + execution_claim, + ) + + missing_run_id = uuid.uuid7() + with pytest.raises(RunNotFound): + await claim_run_in_independent_uow( + session_factory, + missing_run_id, + execution_claim, + ) + + with pytest.raises(RunAlreadyTerminal): + await claim_run_in_independent_uow( + session_factory, + terminal_run.id, + execution_claim, + ) + + assert claimed is not None, f"expected pending run {pending_run.id} to be claimed" + assert lost is None, f"expected second claim to lose, got {lost!r}" + expected = _ClaimOutcomeLogExpectations( + pending_run_id=pending_run.id, + missing_run_id=missing_run_id, + terminal_run_id=terminal_run.id, + lease_expires_at=lease_expires_at, + ) + _assert_claim_outcome_logs(events, expected) diff --git a/tests/canonical_storage/test_generation_run_storage_runtime.py b/tests/canonical_storage/test_generation_run_storage_runtime.py new file mode 100644 index 00000000..c58528d3 --- /dev/null +++ b/tests/canonical_storage/test_generation_run_storage_runtime.py @@ -0,0 +1,60 @@ +"""Tests for deterministic generation-run storage runtime providers.""" + +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import GenerationRunStatus +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.storage.generation_run_storage_runtime import ( + GenerationRunStorageRuntime, +) +from tests.canonical_storage._generation_run_support import ( + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_generation_run_store_uses_injected_runtime( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Adapter-owned timestamps and event IDs should use UoW runtime seams.""" + run = make_generation_run() + event_id = uuid.UUID("00000000-0000-0000-0000-000000000099") + now = dt.datetime(2026, 7, 23, tzinfo=dt.UTC) + await persist_generation_run_prerequisites(session_factory, run) + runtime = GenerationRunStorageRuntime( + clock=lambda: now, + uuid_factory=lambda: event_id, + ) + + async with SqlAlchemyUnitOfWork( + session_factory, + generation_run_runtime=runtime, + ) as uow: + await uow.generation_runs.create_run(run) + updated = await uow.generation_runs.update_run_status( + run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.RUNNING, + current_node="generate", + ended_at=None, + ), + ) + event = await uow.generation_runs.append_event( + run.id, + kind="run.started", + payload={}, + ) + await uow.commit() + + assert updated.updated_at == now, f"updated timestamp: {updated.updated_at!r}" + assert event.id == event_id, f"event identifier: {event.id!r}" + assert event.occurred_at == now, f"event timestamp: {event.occurred_at!r}" diff --git a/tests/canonical_storage/test_generation_run_terminal_claims.py b/tests/canonical_storage/test_generation_run_terminal_claims.py new file mode 100644 index 00000000..f16782ce --- /dev/null +++ b/tests/canonical_storage/test_generation_run_terminal_claims.py @@ -0,0 +1,49 @@ +"""Durable terminal-claim contract tests for generation runs.""" + +import dataclasses as dc +import datetime as dt +import typing as typ + +import pytest + +from episodic.canonical.domain import GenerationRunStatus +from episodic.canonical.generation_run_errors import RunAlreadyTerminal +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from tests.canonical_storage._generation_run_support import ( + NOW, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status", + [ + GenerationRunStatus.SUCCEEDED, + GenerationRunStatus.FAILED, + GenerationRunStatus.CANCELLED, + ], +) +async def test_sql_generation_run_claim_rejects_terminal_status( + session_factory: async_sessionmaker[AsyncSession], + status: GenerationRunStatus, +) -> None: + """The SQL adapter raises rather than claiming any terminal run.""" + run = dc.replace(make_generation_run(), status=status) + await persist_generation_run_prerequisites(session_factory, run) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(run) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(RunAlreadyTerminal, match="generation run is already"): + await uow.generation_runs.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=NOW + dt.timedelta(minutes=5), + ) diff --git a/tests/canonical_storage/test_generation_runs.py b/tests/canonical_storage/test_generation_runs.py new file mode 100644 index 00000000..d28f6815 --- /dev/null +++ b/tests/canonical_storage/test_generation_runs.py @@ -0,0 +1,368 @@ +"""Generation-run SQLAlchemy adapter contract tests. + +These tests exercise durable generation-run and event-log persistence through +`SqlAlchemyUnitOfWork.generation_runs` and the concrete +`SqlAlchemyGenerationRunStore`. They cover run round-trips, idempotency, +event sequence allocation, terminal immutability, and transaction rollback. +""" + +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import GenerationRunStatus +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.generation_run_errors import RunAlreadyTerminal +from episodic.canonical.generation_run_ports import ( + GenerationEventLog, + GenerationRunRepository, + GenerationRunStatusUpdate, + event_seq, +) +from episodic.canonical.storage import ( + GenerationEventRecord, + GenerationRunRecord, + SqlAlchemyGenerationRunStore, + SqlAlchemyUnitOfWork, +) +from tests.canonical_storage._generation_run_support import ( + NOW, + GenerationRunFixture, + count_records, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_generation_run_store_satisfies_run_and_event_ports( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """The SQLAlchemy adapter should implement run repository and event log.""" + async with session_factory() as session: + store = SqlAlchemyGenerationRunStore(session) + + assert isinstance(store, GenerationRunRepository), f"store type: {type(store)}" + assert isinstance(store, GenerationEventLog), f"store type: {type(store)}" + + +@pytest.mark.asyncio +async def test_generation_run_store_persists_across_unit_of_work( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Runs and events should survive fresh unit-of-work instances.""" + run = make_generation_run() + await persist_generation_run_prerequisites(session_factory, run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + stored = await uow.generation_runs.create_run( + run, + idempotency_key="persist-run", + ) + first_event = await uow.generation_runs.append_event( + stored.id, + kind="generation_run.created", + payload={"actor": stored.actor}, + occurred_at=NOW, + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + fetched = await uow.generation_runs.get_run(run.id) + events = await uow.generation_runs.list_events(run.id) + + assert fetched == stored, f"expected stored run {stored!r}, got {fetched!r}" + assert fetched is not None, f"expected run {run.id} to persist, got {fetched!r}" + assert fetched.quality_mode is QualityMode.DRAFT_WITHOUT_QA, ( + f"expected draft-without-QA quality mode, got {fetched.quality_mode!r}" + ) + assert fetched.qa_status is QaStatus.SKIPPED, f"QA status: {fetched.qa_status}" + assert fetched.skip_qa_rationale == "No-QA vertical-slice draft.", ( + f"unexpected skip-QA rationale: {fetched.skip_qa_rationale!r}" + ) + assert events == (first_event,), f"stored events: {events!r}" + assert events[0].seq == 1, f"expected first event sequence 1, got {events[0].seq}" + + +@pytest.mark.asyncio +async def test_generation_run_store_reuses_idempotency_key( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """A retried idempotency key should return the first stored run.""" + first = make_generation_run() + duplicate = make_generation_run() + await persist_generation_run_prerequisites(session_factory, first, duplicate) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + stored_first = await uow.generation_runs.create_run( + first, + idempotency_key="same-key", + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + stored_duplicate = await uow.generation_runs.create_run( + duplicate, + idempotency_key="same-key", + ) + await uow.commit() + + assert stored_duplicate == stored_first, f"duplicate run: {stored_duplicate.id}" + record_count = await count_records(session_factory, GenerationRunRecord) + assert record_count == 1, f"expected one generation run, got {record_count}" + + +@pytest.mark.asyncio +async def test_generation_events_allocate_gap_free_sequences_per_run( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Each run should own a gap-free event sequence starting at 1.""" + first_run = make_generation_run() + second_run = make_generation_run() + await persist_generation_run_prerequisites(session_factory, first_run, second_run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(first_run) + await uow.generation_runs.create_run(second_run) + first_event = await uow.generation_runs.append_event( + first_run.id, + kind="step.started", + payload={"step": 1}, + ) + second_event = await uow.generation_runs.append_event( + first_run.id, + kind="step.finished", + payload={"step": 1}, + ) + other_run_event = await uow.generation_runs.append_event( + second_run.id, + kind="step.started", + payload={"step": 1}, + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + first_events = await uow.generation_runs.list_events(first_run.id) + second_events = await uow.generation_runs.list_events(second_run.id) + after_first = await uow.generation_runs.list_events( + first_run.id, + after_seq=event_seq(1), + ) + + assert [event.seq for event in first_events] == [event_seq(1), event_seq(2)], ( + f"expected first-run sequences [1, 2], got {first_events!r}" + ) + assert [event.seq for event in second_events] == [event_seq(1)], ( + f"expected second-run sequence [1], got {second_events!r}" + ) + assert after_first == (second_event,), ( + f"expected event after sequence 1 to be {second_event!r}, got {after_first!r}" + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(ValueError, match="after_seq and offset cannot be combined"): + await uow.generation_runs.list_events( + first_run.id, + after_seq=event_seq(1), + offset=1, + ) + assert first_events == (first_event, second_event), ( + f"unexpected first-run events: {first_events!r}" + ) + assert second_events == (other_run_event,), ( + f"unexpected second-run events: {second_events!r}" + ) + event_count = await count_records(session_factory, GenerationEventRecord) + assert event_count == 3, f"expected three generation events, got {event_count}" + + +@pytest.mark.asyncio +async def test_generation_run_store_updates_status_and_rejects_terminal_mutation( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Status updates should persist and terminal runs should be immutable.""" + run = make_generation_run() + await persist_generation_run_prerequisites(session_factory, run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(run) + running = await uow.generation_runs.update_run_status( + run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.RUNNING, + current_node="draft", + ended_at=None, + ), + ) + succeeded = await uow.generation_runs.update_run_status( + run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.SUCCEEDED, + current_node=None, + ended_at=NOW, + ), + ) + await uow.commit() + + assert running.status is GenerationRunStatus.RUNNING, ( + f"expected running status, got {running.status!r}" + ) + assert running.current_node == "draft", ( + f"expected current node 'draft', got {running.current_node!r}" + ) + assert succeeded.status is GenerationRunStatus.SUCCEEDED, ( + f"expected succeeded status, got {succeeded.status!r}" + ) + assert succeeded.ended_at == NOW, ( + f"expected end time {NOW!r}, got {succeeded.ended_at!r}" + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(RunAlreadyTerminal, match="generation run is already"): + await uow.generation_runs.update_run_status( + run.id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.RUNNING, + current_node="retry", + ended_at=None, + ), + ) + with pytest.raises(RunAlreadyTerminal, match="generation run is already"): + await uow.generation_runs.append_event( + run.id, + kind="retry.started", + payload={}, + ) + + +@pytest.mark.asyncio +async def test_generation_run_store_claims_pending_run_once( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """The execution claim should be a guarded pending-to-running transition.""" + run = make_generation_run() + lease_expires_at = NOW + dt.timedelta(minutes=5) + await persist_generation_run_prerequisites(session_factory, run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(run) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as first_uow: + claimed = await first_uow.generation_runs.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=lease_expires_at, + ) + await first_uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as second_uow: + lost = await second_uow.generation_runs.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=lease_expires_at, + ) + + assert claimed is not None, f"expected run {run.id} to be claimed, got {claimed!r}" + assert claimed.status is GenerationRunStatus.RUNNING, ( + f"expected claimed run to be running, got {claimed.status!r}" + ) + assert claimed.current_node == "draft", ( + f"expected claimed node 'draft', got {claimed.current_node!r}" + ) + assert claimed.started_at == NOW, ( + f"expected claim start {NOW!r}, got {claimed.started_at!r}" + ) + assert lost is None, f"expected second claim to lose, got {lost!r}" + + +@pytest.mark.asyncio +async def test_generation_run_store_lists_runs_by_episode_status_and_page( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Run listing should be ordered, paged, and filterable by status.""" + episode_id = uuid.uuid7() + first = make_generation_run( + GenerationRunFixture(episode_id=episode_id, created_at=NOW) + ) + running = dc.replace( + make_generation_run( + GenerationRunFixture( + episode_id=episode_id, + created_at=NOW + dt.timedelta(seconds=1), + ) + ), + status=GenerationRunStatus.RUNNING, + ) + other_episode = make_generation_run( + GenerationRunFixture(created_at=NOW + dt.timedelta(seconds=2)) + ) + await persist_generation_run_prerequisites( + session_factory, + first, + running, + other_episode, + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(first) + await uow.generation_runs.create_run(running) + await uow.generation_runs.create_run(other_episode) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + page = await uow.generation_runs.list_runs(episode_id, limit=1, offset=1) + running_only = await uow.generation_runs.list_runs( + episode_id, + status=GenerationRunStatus.RUNNING, + ) + + assert page == (running,), f"expected second paged run {running.id}, got {page!r}" + assert running_only == (running,), ( + f"expected only running run {running.id}, got {running_only!r}" + ) + + +@pytest.mark.asyncio +async def test_generation_run_store_rejects_negative_limit( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """SQL-backed run listing should reject negative limits.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(ValueError, match="limit"): + await uow.generation_runs.list_runs(uuid.uuid7(), limit=-1) + + +@pytest.mark.asyncio +async def test_generation_run_store_rejects_negative_offset( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """SQL-backed run listing should reject negative offsets.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(ValueError, match="offset"): + await uow.generation_runs.list_runs(uuid.uuid7(), offset=-1) + + +@pytest.mark.asyncio +async def test_generation_run_store_rolls_back_uncommitted_run( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Uncommitted runs should not survive unit-of-work rollback.""" + run = make_generation_run() + await persist_generation_run_prerequisites(session_factory, run) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.generation_runs.create_run(run) + await uow.rollback() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + fetched = await uow.generation_runs.get_run(run.id) + + assert fetched is None, f"expected rolled-back run to be absent, got {fetched!r}" diff --git a/tests/canonical_storage/test_sql_episode_tei_property_contract.py b/tests/canonical_storage/test_sql_episode_tei_property_contract.py new file mode 100644 index 00000000..136402e8 --- /dev/null +++ b/tests/canonical_storage/test_sql_episode_tei_property_contract.py @@ -0,0 +1,231 @@ +"""Generated SQLAlchemy invariants for optimistic TEI revisions.""" + +import asyncio +import datetime as dt +import typing as typ + +import hypothesis.strategies as st +import pytest +from hypothesis import HealthCheck, given, settings + +from episodic.canonical.domain import CanonicalEpisode, EpisodeTeiUpdate, GenerationRun +from episodic.canonical.episode_errors import EpisodeRevisionConflictError +from episodic.canonical.generation_quality import QaStatus +from episodic.canonical.hashing import sha256_text +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from tests.canonical_storage._generation_run_support import ( + GenerationRunFixture, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + import uuid + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + type SessionFactory = async_sessionmaker[AsyncSession] +else: + type SessionFactory = object + +_TEI_BODIES = st.text(alphabet="abc", min_size=1, max_size=8) +_UPDATED_AT = dt.datetime(2026, 7, 24, tzinfo=dt.UTC) + + +def _tei_xml(body: str) -> str: + """Build a small valid TEI payload for one generated body.""" + return f"

{body}

" + + +async def _persist_run( + factory: SessionFactory, +) -> GenerationRun: + """Persist one run and its episode/source-bundle prerequisites.""" + run = make_generation_run() + await persist_generation_run_prerequisites(factory, run) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.generation_runs.create_run(run) + await uow.commit() + return run + + +async def _assert_single_winning_update( + factory: SessionFactory, + episode_id: uuid.UUID, + outcomes: tuple[CanonicalEpisode | EpisodeRevisionConflictError, ...], +) -> None: + """Assert a concurrent optimistic update produced one durable winner.""" + winners: list[CanonicalEpisode] = [] + conflicts: list[EpisodeRevisionConflictError] = [] + for outcome in outcomes: + match outcome: + case CanonicalEpisode() as winner: + winners.append(winner) + case EpisodeRevisionConflictError() as conflict: + conflicts.append(conflict) + assert len(winners) == 1, f"winning updates: {outcomes!r}" + assert len(conflicts) == 1, f"revision conflicts: {outcomes!r}" + + winner = winners[0] + async with SqlAlchemyUnitOfWork(factory) as uow: + stored = await uow.episodes.get(episode_id) + assert stored is not None, f"episode {episode_id} was not persisted" + assert stored.tei_revision == 2, f"final revision: {stored.tei_revision}" + assert stored.tei_xml == winner.tei_xml, ( + f"final TEI {stored.tei_xml!r}, winning TEI {winner.tei_xml!r}" + ) + assert stored.tei_content_hash == winner.tei_content_hash, ( + "final hash " + f"{stored.tei_content_hash!r}, winning hash {winner.tei_content_hash!r}" + ) + assert stored.qa_status is winner.qa_status, ( + f"final QA status {stored.qa_status!r}, winning QA status {winner.qa_status!r}" + ) + assert stored.last_generation_run_id == winner.last_generation_run_id, ( + "final provenance " + f"{stored.last_generation_run_id}, winning {winner.last_generation_run_id}" + ) + + +def _tei_update( + tei_xml: str, + run_id: uuid.UUID, + expected_revision: int, +) -> EpisodeTeiUpdate: + """Build a deterministic optimistic TEI update command.""" + return EpisodeTeiUpdate( + tei_xml=tei_xml, + qa_status=QaStatus.SKIPPED, + last_generation_run_id=run_id, + expected_revision=expected_revision, + updated_at=_UPDATED_AT, + ) + + +@given(body=_TEI_BODIES) +@settings( + max_examples=5, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_tei_updates_matching_revision_increment_once( + session_factory: SessionFactory, + body: str, +) -> None: + """Generated matching revisions persist one TEI revision increment.""" + run = await _persist_run(session_factory) + tei_xml = _tei_xml(body) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + updated = await uow.episodes.update( + run.episode_id, + update=_tei_update(tei_xml, run.id, 1), + ) + await uow.commit() + + assert updated.tei_revision == 2, f"revision: {updated.tei_revision}" + assert updated.tei_content_hash == sha256_text(tei_xml), ( + f"content hash: {updated.tei_content_hash!r}" + ) + assert updated.qa_status is QaStatus.SKIPPED, f"QA status: {updated.qa_status}" + assert updated.last_generation_run_id == run.id, ( + f"provenance: {updated.last_generation_run_id}" + ) + + +@given(body=_TEI_BODIES, revision_offset=st.integers(min_value=1, max_value=3)) +@settings( + max_examples=5, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_tei_updates_stale_revision_preserves_state( + session_factory: SessionFactory, + body: str, + revision_offset: int, +) -> None: + """Generated stale revisions leave durable TEI state unchanged.""" + run = await _persist_run(session_factory) + expected_revision = 1 + revision_offset + tei_xml = _tei_xml(body) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(EpisodeRevisionConflictError) as raised: + await uow.episodes.update( + run.episode_id, + update=_tei_update(tei_xml, run.id, expected_revision), + ) + await uow.rollback() + + assert raised.value.expected_revision == expected_revision, ( + f"conflict revision: {raised.value.expected_revision}" + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + stored = await uow.episodes.get(run.episode_id) + assert stored is not None, f"episode {run.episode_id} was not persisted" + assert stored.tei_revision == 1, f"stale update revision: {stored.tei_revision}" + assert stored.tei_xml == "", f"stale update TEI: {stored.tei_xml!r}" + assert stored.tei_content_hash == sha256_text(""), ( + f"stale update hash: {stored.tei_content_hash!r}" + ) + assert stored.qa_status is None, f"stale update QA status: {stored.qa_status}" + assert stored.last_generation_run_id is None, ( + f"stale update provenance: {stored.last_generation_run_id}" + ) + + +@given(first_body=_TEI_BODIES, second_body=_TEI_BODIES) +@settings( + max_examples=4, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_tei_update_race_has_one_winner( + session_factory: SessionFactory, + first_body: str, + second_body: str, +) -> None: + """Two sessions using one revision precondition admit exactly one update.""" + factory = session_factory + first_run = make_generation_run() + second_run = make_generation_run( + GenerationRunFixture(episode_id=first_run.episode_id) + ) + await persist_generation_run_prerequisites(factory, first_run, second_run) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.generation_runs.create_run(first_run) + await uow.generation_runs.create_run(second_run) + await uow.commit() + + barrier = asyncio.Barrier(2) + + async def update( + run: GenerationRun, + body: str, + ) -> CanonicalEpisode | EpisodeRevisionConflictError: + """Attempt one revision-guarded update from an independent session.""" + async with SqlAlchemyUnitOfWork(factory) as uow: + await barrier.wait() + try: + updated = await uow.episodes.update( + run.episode_id, + update=_tei_update(_tei_xml(body), run.id, 1), + ) + await uow.commit() + except EpisodeRevisionConflictError as exc: + await uow.rollback() + return exc + return updated + + first, second = await asyncio.gather( + update(first_run, first_body), + update(second_run, second_body), + ) + await _assert_single_winning_update( + factory, + first_run.episode_id, + (first, second), + ) diff --git a/tests/canonical_storage/test_sql_generation_run_property_contract.py b/tests/canonical_storage/test_sql_generation_run_property_contract.py new file mode 100644 index 00000000..1c885e0d --- /dev/null +++ b/tests/canonical_storage/test_sql_generation_run_property_contract.py @@ -0,0 +1,306 @@ +"""Generated SQLAlchemy generation-run persistence invariants.""" + +import asyncio +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import hypothesis.strategies as st +import pytest +import sqlalchemy as sa +from hypothesis import HealthCheck, given, settings + +from episodic.canonical.domain import GenerationRun, GenerationRunStatus +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate, event_seq +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.storage.generation_run_models import GenerationRunRecord +from episodic.canonical.storage.generation_runs import SqlAlchemyGenerationRunStore +from tests.canonical_storage._generation_run_support import ( + NOW, + ExecutionClaim, + GenerationRunFixture, + claim_run_in_independent_uow, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + type SessionFactory = async_sessionmaker[AsyncSession] +else: + type SessionFactory = object + +_EVENT_KINDS = st.lists( + st.text(alphabet="abc", min_size=1, max_size=8), + min_size=1, + max_size=6, +) + + +@dc.dataclass(frozen=True, slots=True) +class _PageRequest: + """One generated request for run and event pagination.""" + + run_count: int + limit: int + offset: int + after_seq: int + + +async def _recover_expired_lease( + session_factory: SessionFactory, + run_id: uuid.UUID, +) -> bool: + """Apply the documented recovery transition only to an expired running run.""" + async with session_factory() as session: + record = await session.scalar( + sa + .select(GenerationRunRecord) + .where( + GenerationRunRecord.id == run_id, + GenerationRunRecord.status == GenerationRunStatus.RUNNING, + GenerationRunRecord.lease_expires_at.is_not(None), + GenerationRunRecord.lease_expires_at <= NOW, + ) + .with_for_update() + ) + if record is None: + await session.rollback() + return False + store = SqlAlchemyGenerationRunStore(session) + await store.append_event( + run_id, + kind="run.failed", + payload={"error_category": "launcher.lease_expired"}, + occurred_at=NOW, + ) + await store.update_run_status( + run_id, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.FAILED, + current_node="failed", + ended_at=NOW, + error_message="Generation lease expired; failed manually.", + error_category="launcher.lease_expired", + ), + ) + await session.commit() + return True + + +@given(first_kinds=_EVENT_KINDS, second_kinds=_EVENT_KINDS) +@settings( + max_examples=6, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_event_batches_are_gap_free_and_isolated( + session_factory: SessionFactory, + first_kinds: list[str], + second_kinds: list[str], +) -> None: + """Persisted event batches keep contiguous sequence spaces per run.""" + factory = session_factory + first_run = make_generation_run() + second_run = make_generation_run() + await persist_generation_run_prerequisites(factory, first_run, second_run) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.generation_runs.create_run(first_run) + await uow.generation_runs.create_run(second_run) + for kind in first_kinds: + await uow.generation_runs.append_event(first_run.id, kind=kind, payload={}) + for kind in second_kinds: + await uow.generation_runs.append_event(second_run.id, kind=kind, payload={}) + await uow.commit() + + async with SqlAlchemyUnitOfWork(factory) as uow: + first_events = await uow.generation_runs.list_events(first_run.id) + second_events = await uow.generation_runs.list_events(second_run.id) + + assert [event.seq for event in first_events] == list( + range(1, len(first_kinds) + 1) + ), first_events + assert [event.seq for event in second_events] == list( + range(1, len(second_kinds) + 1) + ), second_events + assert [event.kind for event in first_events] == first_kinds, first_events + assert [event.kind for event in second_events] == second_kinds, second_events + + +@given( + request=st.builds( + _PageRequest, + run_count=st.integers(min_value=1, max_value=5), + limit=st.integers(min_value=0, max_value=6), + offset=st.integers(min_value=0, max_value=6), + after_seq=st.integers(min_value=0, max_value=6), + ) +) +@settings( + max_examples=6, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_pages_match_creation_and_cursor_reference_slices( + session_factory: SessionFactory, + request: _PageRequest, +) -> None: + """Run and event pages match their independently calculated durable slices.""" + factory = session_factory + episode_id = uuid.uuid7() + runs = [ + make_generation_run( + GenerationRunFixture( + episode_id=episode_id, + created_at=NOW + dt.timedelta(seconds=index), + ) + ) + for index in range(request.run_count) + ] + await persist_generation_run_prerequisites(factory, *runs) + async with SqlAlchemyUnitOfWork(factory) as uow: + for run in runs: + await uow.generation_runs.create_run(run) + for index in range(request.run_count): + await uow.generation_runs.append_event( + runs[0].id, + kind=f"event-{index}", + payload={}, + ) + await uow.commit() + + cursor = min(request.after_seq, request.run_count) + event_offset = 0 if cursor else request.offset + async with SqlAlchemyUnitOfWork(factory) as uow: + run_page = await uow.generation_runs.list_runs( + episode_id, + limit=request.limit, + offset=request.offset, + ) + event_page = await uow.generation_runs.list_events( + runs[0].id, + after_seq=event_seq(cursor) if cursor else None, + limit=request.limit, + offset=event_offset, + ) + + assert run_page == tuple(runs[request.offset : request.offset + request.limit]), ( + "run page did not match " + f"offset={request.offset}, limit={request.limit}: {run_page!r}" + ) + expected_event_indexes = range( + cursor + event_offset, + min(request.run_count, cursor + event_offset + request.limit), + ) + assert [event.kind for event in event_page] == [ + f"event-{index}" for index in expected_event_indexes + ], event_page + + +@given(key=st.text(alphabet="abc123", min_size=1, max_size=12)) +@settings( + max_examples=6, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_idempotency_keys_are_scoped_to_principals( + session_factory: SessionFactory, + key: str, +) -> None: + """The same key replays per principal while remaining independent across actors.""" + factory = session_factory + first_run = make_generation_run() + replay_run = make_generation_run() + other_principal_run = make_generation_run() + await persist_generation_run_prerequisites( + factory, + first_run, + replay_run, + other_principal_run, + ) + async with SqlAlchemyUnitOfWork(factory) as uow: + first = await uow.generation_runs.create_run( + first_run, + idempotency_key=key, + idempotency_principal_id="principal-a", + ) + replay = await uow.generation_runs.create_run( + replay_run, + idempotency_key=key, + idempotency_principal_id="principal-a", + ) + other = await uow.generation_runs.create_run( + other_principal_run, + idempotency_key=key, + idempotency_principal_id="principal-b", + ) + await uow.commit() + + assert replay.id == first.id, (replay.id, first.id) + assert other.id == other_principal_run.id, (other.id, other_principal_run.id) + assert other.id != first.id, (other.id, first.id) + + +@given( + first_node=st.text(alphabet="ab", min_size=1, max_size=4), + second_node=st.text(alphabet="cd", min_size=1, max_size=4), + lease_offset_seconds=st.integers(min_value=-2, max_value=2), +) +@settings( + max_examples=5, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@pytest.mark.asyncio +async def test_sql_claim_races_and_lease_recovery_preserve_one_terminal_path( + session_factory: SessionFactory, + first_node: str, + second_node: str, + lease_offset_seconds: int, +) -> None: + """A pending run has one claimant and recovery acts only on expired leases.""" + factory = session_factory + run = make_generation_run() + await persist_generation_run_prerequisites(factory, run) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.generation_runs.create_run(run) + await uow.commit() + + claim_barrier = asyncio.Barrier(2) + + async def claim(node: str) -> GenerationRun | None: + """Attempt one conditional claim after both sessions are ready.""" + await claim_barrier.wait() + return await claim_run_in_independent_uow( + factory, + run.id, + ExecutionClaim( + current_node=node, + started_at=NOW, + lease_expires_at=NOW + dt.timedelta(seconds=lease_offset_seconds), + ), + ) + + first, second = await asyncio.gather(claim(first_node), claim(second_node)) + assert sum(result is not None for result in (first, second)) == 1, (first, second) + recovered = await _recover_expired_lease(factory, run.id) + + assert recovered is (lease_offset_seconds <= 0), ( + recovered, + lease_offset_seconds, + ) + async with SqlAlchemyUnitOfWork(factory) as uow: + persisted = await uow.generation_runs.get_run(run.id) + events = await uow.generation_runs.list_events(run.id) + assert persisted is not None, f"expected persisted run {run.id}" + if recovered: + assert persisted.status is GenerationRunStatus.FAILED, persisted.status + assert [event.kind for event in events] == ["run.failed"], events + else: + assert persisted.status is GenerationRunStatus.RUNNING, persisted.status + assert events == (), events diff --git a/tests/features/no_qa_generation_slice.feature b/tests/features/no_qa_generation_slice.feature new file mode 100644 index 00000000..c85a701f --- /dev/null +++ b/tests/features/no_qa_generation_slice.feature @@ -0,0 +1,60 @@ +Feature: No-QA source-to-script generation slice + + As an integration client + I want to generate a draft script without QA and download the TEI + So that I can validate the source-to-script workflow over REST + + Background: + Given a Vidai Mock inference server is running + And a series profile exists + And a host presenter profile and a guest presenter profile are bound + + Scenario: Draft generation without QA produces a downloadable TEI-P5 script + Given an ingestion job with an attached source document + When I create a draft-without-qa generation run for the ingested episode + Then the run creation responds 202 Accepted with a Location header + And the response carries a Retry-After header + And the run is created with qa_status "skipped" and my rationale recorded + When I poll the generation run until it reaches a terminal state + Then the run status is "succeeded" + And the event log contains a "tei.persisted" event + When I fetch the episode TEI as application/tei+xml + Then the response is a TEI-P5 attachment with qa_status "skipped" + And the TEI validates against the Episodic TEI-P5 profile + + Scenario: Reusing an idempotency key with the same body replays the run + Given an ingestion job with an attached source document + When I create a draft-without-qa run twice with the same idempotency key and body + Then both responses describe the same run id + And the replayed response carries the same Location and Retry-After + + Scenario: Reusing an idempotency key with a different body conflicts + Given an ingestion job with an attached source document + When I create a draft-without-qa run, then reuse the key with a different rationale + Then the second response is 409 Conflict + + Scenario: A missing rationale is rejected + Given an ingestion job with an attached source document + When I create a draft-without-qa run without a skip_qa_rationale + Then the response is 400 Bad Request + + Scenario: An unsupported quality mode is unprocessable + Given an ingestion job with an attached source document + When I create a generation run with quality_mode "qa_gated" + Then the response is 422 Unprocessable Entity + + Scenario: Generation failure is reported on the run + Given an ingestion job with an attached source document + And the inference server is configured to fail + When I create a draft-without-qa generation run for the ingested episode + And I poll the generation run until it reaches a terminal state + Then the run status is "failed" + And the run records an error message and an error category + + Scenario: A malformed completion is reported as a failed run + Given an ingestion job with an attached source document + And the inference server is configured to return a non-TEI completion + When I create a draft-without-qa generation run for the ingested episode + And I poll the generation run until it reaches a terminal state + Then the run status is "failed" + And the event log contains a "tei.invalid" event diff --git a/tests/fixtures/generation_run_api.py b/tests/fixtures/generation_run_api.py new file mode 100644 index 00000000..33e6b642 --- /dev/null +++ b/tests/fixtures/generation_run_api.py @@ -0,0 +1,134 @@ +"""Public fixtures for authenticated generation-run REST integration tests. + +The helpers create principal-owned ready ingestion jobs, provide valid no-QA +request bodies, and expose deterministic launcher and authorization fakes. +""" + +import dataclasses as dc +import typing as typ +from unittest import mock + +from episodic.api.authorization import ( + AuthorizationContext, + AuthorizationDecision, + AuthorizationResult, +) + +if typ.TYPE_CHECKING: + import uuid + + import httpx + + +@dc.dataclass(slots=True) +class RecordingLauncher: + """Record generation runs scheduled by the HTTP adapter. + + Attributes + ---------- + run_ids : list[uuid.UUID] + Run identifiers recorded in launch order. + """ + + run_ids: list[uuid.UUID] = dc.field(default_factory=list) + launch: mock.AsyncMock = dc.field(init=False) + + def __post_init__(self) -> None: + """Bind the asynchronous launch surface to the run recorder.""" + self.launch = mock.AsyncMock(side_effect=self.run_ids.append) + + +class HeaderPrincipalAuthorization: + """Authenticate test principals from fixed bearer-token values. + + The adapter permits ``Bearer principal-a`` and ``Bearer principal-b`` and + rejects every other credential. + """ + + @staticmethod + async def decide( + context: AuthorizationContext, + ) -> AuthorizationResult: + """Map test bearer tokens to their principals.""" + principals = { + "Bearer principal-a": "principal-a", + "Bearer principal-b": "principal-b", + } + principal_id = principals.get(context.authorization_header or "") + if principal_id is None: + return AuthorizationResult(AuthorizationDecision.UNAUTHORIZED) + return AuthorizationResult(AuthorizationDecision.PERMIT, principal_id) + + +async def create_ready_ingestion_job( + client: httpx.AsyncClient, + headers: dict[str, str] | None = None, + key_prefix: str = "generation", +) -> str: + """Create a ready source bundle owned by the supplied test principal. + + Parameters + ---------- + client : httpx.AsyncClient + ASGI client used to create the profile, ingestion job, and source. + headers : dict[str, str] | None, optional + Authorization headers determining the persisted job owner. + key_prefix : str, default="generation" + Prefix used to isolate the fixture's idempotency keys. + + Returns + ------- + str + Persisted ingestion-job identifier ready for draft generation. + """ + request_headers = {} if headers is None else headers + profile = await client.post( + "/v1/series-profiles", + headers={**request_headers, "Idempotency-Key": f"{key_prefix}-profile-key"}, + json={ + "slug": "generation-api-profile", + "title": "Generation API profile", + "description": "Generation endpoint fixture.", + "configuration": {}, + "actor": "editor@example.com", + }, + ) + assert profile.status_code == 201, profile.text + job = await client.post( + "/v1/ingestion-jobs", + headers={**request_headers, "Idempotency-Key": f"{key_prefix}-job-key"}, + json={"series_profile_id": profile.json()["id"]}, + ) + assert job.status_code == 201, job.text + source = await client.post( + f"/v1/ingestion-jobs/{job.json()['id']}/sources", + headers={**request_headers, "Idempotency-Key": f"{key_prefix}-source-key"}, + json={ + "type": "source_uri", + "source_uri": "https://example.test/source.txt", + "source_type": "research_note", + "weight": 1.0, + "metadata": {"content": "A concise source for the episode."}, + }, + ) + assert source.status_code == 201, source.text + return typ.cast("str", job.json()["id"]) + + +def generation_payload() -> dict[str, object]: + """Return a valid no-QA generation request body. + + Returns + ------- + dict[str, object] + Payload accepted by the generation-run endpoint; its client actor is + deliberately ignored in favour of the authenticated principal. + """ + return { + "quality_mode": "draft_without_qa", + "skip_qa_rationale": "Initial editorial draft.", + "actor": "editor@example.com", + "template_id": "future-template", + "prompt_overrides": {"tone": "clear"}, + "budget_hints": {"max_tokens": 1200}, + } diff --git a/tests/generation_run_launcher_support.py b/tests/generation_run_launcher_support.py new file mode 100644 index 00000000..e5c71619 --- /dev/null +++ b/tests/generation_run_launcher_support.py @@ -0,0 +1,337 @@ +"""Support fixtures for generation-run launcher tests.""" + +import asyncio +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +from episodic.canonical.domain import ( + GenerationRun, + GenerationRunStatus, + IngestionJob, + IngestionStatus, + IntakeState, + ReferenceBinding, + ReferenceBindingTargetKind, + ReferenceDocument, + ReferenceDocumentKind, + ReferenceDocumentLifecycleState, + ReferenceDocumentRevision, + SeriesProfile, +) +from episodic.canonical.generation_persistence import ( + EpisodeMaterialisationRequest, + materialise_episode_from_ingestion, +) +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.ingestion_sources import AttachmentKind, IngestionJobSource +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.cost.ports import BillingPeriodKey, CostLedgerEntryId, UsageSource +from episodic.generation.draft_script import ( + DraftScriptGenerator, + DraftScriptRequest, + DraftScriptResult, +) +from episodic.generation.launcher import InProcessGenerationRunLauncher +from episodic.llm import LLMUsage, ProviderCallUsage + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.cost.recorder import CostProviderOperation, ProviderCallRecord + +NOW = dt.datetime(2026, 6, 24, 12, 0, tzinfo=dt.UTC) + + +@dc.dataclass(frozen=True, slots=True) +class LauncherOptions: + """Execution and admission capacities for a test launcher.""" + + max_concurrency: int = 4 + max_pending_runs: int = 16 + + +@dc.dataclass(slots=True) +class RecordingDraftGenerator: + """Draft generator fake that returns one configured result.""" + + result: DraftScriptResult + requests: list[DraftScriptRequest] = dc.field(default_factory=list) + + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Capture the request and return the configured result.""" + self.requests.append(request) + return self.result + + +@dc.dataclass(slots=True) +class FailingDraftGenerator: + """Draft generator fake that raises the configured exception.""" + + error: Exception + + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Raise the configured generation error.""" + _ = request + raise self.error + + +class BlockingDraftGenerator: + """Draft generator fake that blocks until the task is cancelled.""" + + def __init__(self) -> None: + self.started = asyncio.Event() + + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Block until cancelled by the launcher shutdown hook.""" + _ = request + self.started.set() + await asyncio.Event().wait() + raise AssertionError + + +class ReleasableDraftGenerator: + """Draft generator fake that completes when the test releases it.""" + + def __init__(self, result: DraftScriptResult) -> None: + self.result = result + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def generate(self, request: DraftScriptRequest) -> DraftScriptResult: + """Wait for the test and then return the configured draft.""" + _ = request + self.started.set() + await self.release.wait() + return self.result + + +@dc.dataclass(slots=True) +class RecordingCostRecorder: + """Cost recorder fake that captures provider calls and roll-ups.""" + + provider_calls: list[ProviderCallRecord] = dc.field(default_factory=list) + finalized_runs: list[tuple[str, str | None]] = dc.field(default_factory=list) + + @staticmethod + async def pin_run_pricing( + workflow_run_id: str, + providers: tuple[CostProviderOperation, ...], + billing_period_key: BillingPeriodKey, + ) -> None: + """Accept pricing pins for the fake recorder.""" + _ = (workflow_run_id, providers, billing_period_key) + + async def record_provider_call( + self, + record: ProviderCallRecord, + ) -> CostLedgerEntryId: + """Capture one provider-call record.""" + self.provider_calls.append(record) + return CostLedgerEntryId(f"entry:{len(self.provider_calls)}") + + async def finalize_run( + self, + workflow_run_id: str, + workflow_node: str | None, + ) -> CostLedgerEntryId: + """Capture final run roll-up requests.""" + self.finalized_runs.append((workflow_run_id, workflow_node)) + return CostLedgerEntryId("entry:rollup") + + +def _clock() -> dt.datetime: + """Return the frozen launcher timestamp.""" + return NOW + + +def _series_profile() -> SeriesProfile: + """Return a series profile fixture.""" + return SeriesProfile( + id=uuid.UUID("00000000-0000-0000-0000-000000000201"), + slug="bridgewater", + title="Bridgewater", + description=None, + configuration={}, + guardrails={}, + created_at=NOW, + updated_at=NOW, + ) + + +def _ingestion_job(series_profile_id: uuid.UUID) -> IngestionJob: + """Return a ready intake job fixture.""" + return IngestionJob( + id=uuid.UUID("00000000-0000-0000-0000-000000000301"), + series_profile_id=series_profile_id, + target_episode_id=None, + status=IngestionStatus.PENDING, + requested_at=NOW, + started_at=None, + completed_at=None, + error_message=None, + created_at=NOW, + updated_at=NOW, + intake_state=IntakeState.READY_FOR_GENERATION, + ) + + +def _source(job_id: uuid.UUID) -> IngestionJobSource: + """Return one source attachment carrying text for generation.""" + return IngestionJobSource( + id=uuid.UUID("00000000-0000-0000-0000-000000000401"), + ingestion_job_id=job_id, + attachment_kind=AttachmentKind.SOURCE_URI, + upload_id=None, + source_uri="https://example.test/source.md", + source_type="research_brief", + weight=1.0, + metadata={"content": "Bridgewater launch source text."}, + created_at=NOW, + ) + + +def _presenter_sets( + series_profile_id: uuid.UUID, +) -> tuple[tuple[ReferenceDocument, ReferenceDocumentRevision, ReferenceBinding], ...]: + """Return host and guest profiles bound to the series.""" + sets = [] + for offset, kind, name, summary in ( + (1, ReferenceDocumentKind.HOST_PROFILE, "Host One", "Host profile."), + (2, ReferenceDocumentKind.GUEST_PROFILE, "Guest One", "Guest profile."), + ): + document = ReferenceDocument( + id=uuid.UUID(f"00000000-0000-0000-0000-{600 + offset:012d}"), + owner_series_profile_id=series_profile_id, + kind=kind, + lifecycle_state=ReferenceDocumentLifecycleState.ACTIVE, + metadata={"name": name}, + created_at=NOW, + updated_at=NOW, + ) + revision = ReferenceDocumentRevision( + id=uuid.UUID(f"00000000-0000-0000-0000-{700 + offset:012d}"), + reference_document_id=document.id, + content={"summary": summary}, + content_hash=f"presenter-{offset}", + author="editor@example.test", + change_note="Create presenter profile.", + created_at=NOW, + ) + binding = ReferenceBinding( + id=uuid.UUID(f"00000000-0000-0000-0000-{800 + offset:012d}"), + reference_document_revision_id=revision.id, + target_kind=ReferenceBindingTargetKind.SERIES_PROFILE, + series_profile_id=series_profile_id, + episode_template_id=None, + ingestion_job_id=None, + effective_from_episode_id=None, + created_at=NOW, + ) + sets.append((document, revision, binding)) + return tuple(sets) + + +def draft_result(tei_xml: str) -> DraftScriptResult: + """Return a generated draft result with provider usage.""" + return DraftScriptResult( + tei_xml=tei_xml, + content_hash=f"sha256:{hashlib.sha256(tei_xml.encode()).hexdigest()}", + usage=LLMUsage(input_tokens=10, output_tokens=20, total_tokens=30), + model="gpt-4o-mini", + provider_response_id="resp-draft-1", + finish_reason="stop", + provider_call_usage=ProviderCallUsage( + usage_metrics={"input_tokens": 10, "output_tokens": 20}, + usage_source=UsageSource.PROVIDER, + usage_complete=True, + provider_response_id="resp-draft-1", + finish_reason="stop", + started_at=NOW.isoformat(), + latency_ms=125, + ), + ) + + +def valid_tei() -> str: + """Return valid generated TEI for launcher tests.""" + return ( + '' + "Bridgewater Futures" + 'Welcome.' + ) + + +def _run(episode_id: uuid.UUID, source_bundle_id: uuid.UUID) -> GenerationRun: + """Return a pending no-QA generation run.""" + return GenerationRun( + id=uuid.UUID("00000000-0000-0000-0000-000000000501"), + episode_id=episode_id, + source_bundle_id=source_bundle_id, + actor="editor@example.test", + status=GenerationRunStatus.PENDING, + current_node=None, + budget_snapshot={}, + configuration={}, + created_at=NOW, + updated_at=NOW, + started_at=None, + ended_at=None, + error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="Vertical slice draft.", + ) + + +async def prepare_pending_run( + factory: async_sessionmaker[AsyncSession], +) -> tuple[uuid.UUID, uuid.UUID]: + """Persist a ready episode and pending generation run.""" + series = _series_profile() + job = _ingestion_job(series.id) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.series_profiles.add(series) + await uow.flush() + await uow.ingestion_jobs.add(job) + await uow.ingestion_job_sources.add(_source(job.id)) + for document, revision, binding in _presenter_sets(series.id): + await uow.reference_documents.add(document) + await uow.reference_document_revisions.add(revision) + await uow.flush() + await uow.reference_bindings.add(binding) + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + ), + ) + run = _run(episode.id, job.id) + await uow.generation_runs.create_run(run) + await uow.commit() + return run.id, episode.id + + +def launcher( + factory: async_sessionmaker[AsyncSession], + generator: DraftScriptGenerator, + cost_recorder: RecordingCostRecorder | None = None, + *, + options: LauncherOptions | None = None, +) -> InProcessGenerationRunLauncher: + """Build a launcher for tests.""" + effective_options = LauncherOptions() if options is None else options + return InProcessGenerationRunLauncher( + uow_factory=lambda: SqlAlchemyUnitOfWork(factory), + draft_generator=generator, + cost_recorder_factory=( + None if cost_recorder is None else lambda uow: cost_recorder + ), + clock=_clock, + max_concurrency=effective_options.max_concurrency, + max_pending_runs=effective_options.max_pending_runs, + ) diff --git a/tests/steps/generation_orchestration_vidaimock.py b/tests/steps/generation_orchestration_vidaimock.py index 2ba178fc..0893f171 100644 --- a/tests/steps/generation_orchestration_vidaimock.py +++ b/tests/steps/generation_orchestration_vidaimock.py @@ -13,8 +13,6 @@ if typ.TYPE_CHECKING: from pathlib import Path - from tests.steps.test_generation_orchestration_steps import OrchestrationBDDContext - def find_free_port() -> int: """Bind to an ephemeral port and return its number before releasing it.""" @@ -146,7 +144,7 @@ def _await_port_ready( def start_vidaimock_process( - orchestration_context: OrchestrationBDDContext, + orchestration_context: VidaiMockProcessContext, config_dir: Path, port: int, ) -> None: @@ -174,3 +172,10 @@ def start_vidaimock_process( ) _await_port_ready(orchestration_context.process, "127.0.0.1", port) + + +class VidaiMockProcessContext(typ.Protocol): + """Minimal mutable state required by the Vidai Mock process helper.""" + + process: subprocess.Popen[str] | None + base_url: str diff --git a/tests/steps/no_qa_generation_slice_assertions.py b/tests/steps/no_qa_generation_slice_assertions.py new file mode 100644 index 00000000..e20dbd67 --- /dev/null +++ b/tests/steps/no_qa_generation_slice_assertions.py @@ -0,0 +1,54 @@ +"""Error-envelope assertions for no-QA generation BDD scenarios.""" + +import uuid + +from tests.steps.no_qa_generation_slice_support import ( + ErrorEnvelopeExpectation, + NoQaGenerationSliceContext, + assert_error_envelope, +) + + +def second_response_is_conflict(context: NoQaGenerationSliceContext) -> None: + """Verify changed input is rejected under the reused key.""" + response = context.responses[1] + details = response.json()["details"] + assert isinstance(details, dict), f"idempotency error details: {details!r}" + record_id = details.get("record_id") + assert isinstance(record_id, str), f"idempotency record id: {record_id!r}" + uuid.UUID(record_id) + assert_error_envelope( + response, + ErrorEnvelopeExpectation( + status=409, + code="idempotency_conflict", + message="Idempotency key body mismatch.", + details={"record_id": record_id}, + ), + ) + + +def response_is_bad_request(context: NoQaGenerationSliceContext) -> None: + """Verify malformed quality metadata is a bad request.""" + assert_error_envelope( + context.responses[0], + ErrorEnvelopeExpectation( + status=400, + code="validation_error", + message="Missing required field: skip_qa_rationale", + details={"field": "skip_qa_rationale", "constraint": "required"}, + ), + ) + + +def response_is_unprocessable(context: NoQaGenerationSliceContext) -> None: + """Verify a recognized unsupported mode is unprocessable.""" + assert_error_envelope( + context.responses[0], + ErrorEnvelopeExpectation( + status=422, + code="quality_mode_unsupported", + message="Unsupported quality_mode: qa_gated.", + details={"quality_mode": "qa_gated"}, + ), + ) diff --git a/tests/steps/no_qa_generation_slice_support.py b/tests/steps/no_qa_generation_slice_support.py new file mode 100644 index 00000000..549d421a --- /dev/null +++ b/tests/steps/no_qa_generation_slice_support.py @@ -0,0 +1,305 @@ +"""Support for the no-QA source-to-script behavioural slice.""" + +import dataclasses as dc +import json +import subprocess # noqa: S404 - terminates a controlled local test process. +import typing as typ + +import httpx + +from episodic.api import create_app +from episodic.api.authorization import StaticBearerTokenAuthorization +from episodic.generation import InProcessGenerationRunLauncher +from episodic.generation.draft_script import ( + LLMDraftScriptGenerator, + LLMDraftScriptGeneratorConfig, +) +from episodic.llm.openai_adapter import ( + OpenAICompatibleLLMAdapter, + OpenAICompatibleLLMConfig, +) +from tests.fixtures.api import build_api_dependencies +from tests.steps.generation_orchestration_vidaimock import ( + find_free_port, + start_vidaimock_process, +) + +if typ.TYPE_CHECKING: + import asyncio + import collections.abc as cabc + from pathlib import Path + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.api.dependencies import ApiDependencies + +_VALID_DRAFT = json.dumps({ + "title": "A deterministic no-QA draft", + "turns": [ + {"speaker": "host", "text": "Welcome to the generated episode."}, + {"speaker": "guest", "text": "The source supports this discussion."}, + ], +}) +_AUTHORIZATION_TOKEN = "-".join(("no", "qa", "slice", "token")) +_AUTHORIZATION_HEADER = {"Authorization": f"Bearer {_AUTHORIZATION_TOKEN}"} + + +@dc.dataclass(slots=True) +class NoQaGenerationSliceContext: + """Hold infrastructure and observations for one behavioural scenario.""" + + session_factory: async_sessionmaker[AsyncSession] + runner: asyncio.Runner + process: subprocess.Popen[str] | None = None + base_url: str = "" + dependencies: ApiDependencies | None = None + launcher: InProcessGenerationRunLauncher | None = None + llm_adapter: OpenAICompatibleLLMAdapter | None = None + llm_client: httpx.AsyncClient | None = None + profile_id: str | None = None + ingestion_job_id: str | None = None + responses: list[httpx.Response] = dc.field(default_factory=list) + run_response: httpx.Response | None = None + events_response: httpx.Response | None = None + tei_response: httpx.Response | None = None + + async def request( + self, + method: str, + path: str, + *, + headers: cabc.Mapping[str, str] | None = None, + json: object | None = None, + ) -> httpx.Response: + """Issue one request against the in-process Falcon application.""" + dependencies = require(self.dependencies, "API dependencies") + transport = httpx.ASGITransport( + app=typ.cast("typ.Any", create_app(dependencies)) + ) + request_headers = _AUTHORIZATION_HEADER | ( + {} if headers is None else dict(headers) + ) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + return await client.request( + method, + path, + headers=request_headers, + json=json, + ) + + async def close(self) -> None: + """Release asynchronous resources owned by the scenario.""" + if self.launcher is not None: + await self.launcher.shutdown() + if self.llm_adapter is not None: + await self.llm_adapter.aclose() + + def tear_down(self) -> None: + """Release asynchronous resources and stop the Vidai Mock process.""" + try: + self.runner.run(self.close()) + finally: + if self.process is not None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + + +@dc.dataclass(frozen=True, slots=True) +class ErrorEnvelopeExpectation: + """Expected values for one standard API error envelope. + + Attributes + ---------- + status : int + Expected HTTP status code. + code : str + Expected machine-readable error code. + message : str + Expected human-readable error message. + details : dict[str, object] + Expected structured error details. + """ + + status: int + code: str + message: str + details: dict[str, object] + + +def require[RequiredValue]( + value: RequiredValue | None, + label: str, +) -> RequiredValue: + """Return initialized scenario state or fail with a useful assertion.""" + assert value is not None, f"Expected {label} to be initialized." + return value + + +def assert_response_status(response: httpx.Response, expected: int) -> None: + """Assert an HTTP response status with its body as failure context.""" + assert response.status_code == expected, ( + f"expected HTTP {expected}, got {response.status_code}: {response.text}" + ) + + +def assert_error_envelope( + response: httpx.Response, + expected: ErrorEnvelopeExpectation, +) -> None: + """Assert the standard API error envelope exactly. + + Parameters + ---------- + response : httpx.Response + HTTP response containing the error envelope to inspect. + expected : ErrorEnvelopeExpectation + Expected status and error-envelope field values. + + The error envelope must contain exactly ``code``, ``message``, and + ``details``, with each value exactly matching ``expected.code``, + ``expected.message``, and ``expected.details``, respectively. + """ + assert_response_status(response, expected.status) + payload = response.json() + assert set(payload) == {"code", "message", "details"}, ( + f"error envelope keys: {payload!r}" + ) + assert payload["code"] == expected.code, f"error code: {payload['code']!r}" + assert payload["message"] == expected.message, ( + f"error message: {payload['message']!r}" + ) + assert payload["details"] == expected.details, ( + f"error details: {payload['details']!r}" + ) + + +def assert_tei_response( + response: httpx.Response, + run_response: httpx.Response, + qa_status: str, +) -> None: + """Assert TEI attachment metadata and its QA provenance.""" + assert_response_status(response, 200) + content_type = response.headers["content-type"] + assert content_type.startswith("application/tei+xml"), ( + f"content type: {content_type}" + ) + disposition = response.headers["content-disposition"] + assert "attachment" in disposition, f"disposition: {disposition}" + observed_qa_status = run_response.json()["qa_status"] + assert observed_qa_status == qa_status, f"QA status: {observed_qa_status!r}" + + +def assert_replay_headers(first: httpx.Response, second: httpx.Response) -> None: + """Assert an idempotent replay retains polling metadata.""" + first_headers = (first.headers["Location"], first.headers["Retry-After"]) + second_headers = (second.headers["Location"], second.headers["Retry-After"]) + assert first_headers == second_headers, f"replay headers: {second_headers!r}" + + +def configure_vidaimock(context: NoQaGenerationSliceContext, tmp_path: Path) -> None: + """Start Vidai Mock and wire the real generator and launcher to it.""" + provider_dir = tmp_path / "providers" + template_dir = tmp_path / "templates" / "draft" + provider_dir.mkdir(parents=True) + template_dir.mkdir(parents=True) + _write_provider_config(provider_dir) + _write_response_template(template_dir) + start_vidaimock_process(context, tmp_path, port=find_free_port()) + + context.llm_client = httpx.AsyncClient() + context.llm_adapter = OpenAICompatibleLLMAdapter( + config=OpenAICompatibleLLMConfig( + base_url=context.base_url, + api_key="test-key", + max_attempts=1, + ), + client=context.llm_client, + ) + dependencies = build_api_dependencies( + context.session_factory, + authorization=StaticBearerTokenAuthorization( + token=_AUTHORIZATION_TOKEN, + principal_id="no-qa-slice-principal", + ), + ) + context.launcher = InProcessGenerationRunLauncher( + uow_factory=dependencies.uow_factory, + draft_generator=LLMDraftScriptGenerator( + llm=context.llm_adapter, + config=LLMDraftScriptGeneratorConfig(model="valid-draft"), + ), + ) + context.dependencies = dc.replace(dependencies, launcher=context.launcher) + + +def select_malformed_completion(context: NoQaGenerationSliceContext) -> None: + """Select the deterministic malformed provider response.""" + adapter = require(context.llm_adapter, "LLM adapter") + launcher = require(context.launcher, "generation launcher") + launcher.draft_generator = LLMDraftScriptGenerator( + llm=adapter, + config=LLMDraftScriptGeneratorConfig(model="malformed-draft"), + ) + + +def enable_provider_failure(context: NoQaGenerationSliceContext) -> None: + """Force Vidai Mock to drop every provider request.""" + client = require(context.llm_client, "LLM HTTP client") + client.headers["X-Vidai-Chaos-Drop"] = "100" + + +def generation_payload(**overrides: object) -> dict[str, object]: + """Return the canonical no-QA creation request.""" + payload: dict[str, object] = { + "quality_mode": "draft_without_qa", + "skip_qa_rationale": "Prepare an editorial draft before QA.", + "actor": "editor@example.com", + } + payload.update(overrides) + return payload + + +def _write_provider_config(provider_dir: Path) -> None: + (provider_dir / "draft.yaml").write_text( + "\n".join(( + 'name: "draft"', + 'matcher: "/v1/chat/completions"', + "request_mapping:", + ' model: "{{ json.model }}"', + 'response_template: "draft/response.json.j2"', + )) + + "\n", + encoding="utf-8", + ) + + +def _write_response_template(template_dir: Path) -> None: + valid_content = json.dumps(_VALID_DRAFT) + invalid_tei_draft = json.dumps({ + "title": "Invalid TEI draft", + "turns": [{"speaker": "\u0001", "text": "Invalid XML speaker."}], + }) + malformed_content = json.dumps(invalid_tei_draft) + (template_dir / "response.json.j2").write_text( + f"""{{ + "id": "chatcmpl-{{{{ uuid() }}}}", + "created": {{{{ timestamp() }}}}, + "object": "chat.completion", + "model": "{{{{ model }}}}", + "choices": [{{"index": 0, "message": {{"role": "assistant", "content": + {{% if model == "malformed-draft" %}}{malformed_content} + {{% else %}}{valid_content}{{% endif %}} + }}, "finish_reason": "stop"}}], + "usage": {{"prompt_tokens": 20, "completion_tokens": 12, "total_tokens": 32}} +}} +""", + encoding="utf-8", + ) diff --git a/tests/steps/source_intake_api_helpers.py b/tests/steps/source_intake_api_helpers.py index ec9f7a1a..d2d9fefc 100644 --- a/tests/steps/source_intake_api_helpers.py +++ b/tests/steps/source_intake_api_helpers.py @@ -45,6 +45,7 @@ async def create_series_profile(client: httpx.AsyncClient) -> str: """Create a series profile through the public API and return its id.""" response = await client.post( "/v1/series-profiles", + headers={"Idempotency-Key": f"bdd-profile-{uuid.uuid4()}"}, json={ "slug": f"bdd-source-intake-{uuid.uuid4()}", "title": "BDD Source Intake", @@ -90,7 +91,7 @@ async def create_pending_upload( now = dt.datetime.now(dt.UTC) upload = Upload( id=uuid.uuid4(), - owner_principal_id="api-user", + owner_principal_id="principal-a", content_type="text/plain", declared_size=1, actual_size=None, diff --git a/tests/steps/source_intake_support.py b/tests/steps/source_intake_support.py index e6eaadab..275fe2e6 100644 --- a/tests/steps/source_intake_support.py +++ b/tests/steps/source_intake_support.py @@ -10,6 +10,7 @@ from episodic.api import create_app from episodic.canonical.storage import FilesystemObjectStore from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import HeaderPrincipalAuthorization from tests.steps.source_intake_api_helpers import ( create_ingestion_job, create_ingestion_job_response, @@ -271,6 +272,7 @@ def _run_source_intake_call( async def _run_workflow() -> None: dependencies = build_api_dependencies( config.session_factory, + authorization=HeaderPrincipalAuthorization(), object_store=FilesystemObjectStore(config.tmp_path / "bdd-objects"), upload_max_bytes=config.upload_max_bytes, ) @@ -280,6 +282,7 @@ async def _run_workflow() -> None: async with httpx.AsyncClient( transport=transport, base_url="http://testserver", + headers={"Authorization": "Bearer principal-a"}, ) as client: await action(client, context) diff --git a/tests/steps/test_generation_run_lifecycle_steps.py b/tests/steps/test_generation_run_lifecycle_steps.py index 94dfa9fe..a73490c5 100644 --- a/tests/steps/test_generation_run_lifecycle_steps.py +++ b/tests/steps/test_generation_run_lifecycle_steps.py @@ -20,6 +20,7 @@ GenerationRun, GenerationRunStatus, ) +from episodic.canonical.generation_quality import QaStatus, QualityMode from episodic.canonical.generation_run_errors import CheckpointAlreadyTerminal if typ.TYPE_CHECKING: @@ -66,6 +67,9 @@ def _run() -> GenerationRun: started_at=None, ended_at=None, error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", ) diff --git a/tests/steps/test_http_service_scaffold_steps.py b/tests/steps/test_http_service_scaffold_steps.py index a3c3d6e4..5237b520 100644 --- a/tests/steps/test_http_service_scaffold_steps.py +++ b/tests/steps/test_http_service_scaffold_steps.py @@ -209,6 +209,8 @@ def given_granian_service_running( "SOURCE_INTAKE_OBJECT_STORE_ROOT": str( http_service_scaffold_context.object_store_root ), + "API_AUTHORIZATION_BEARER_TOKEN": "http-service-test-token", + "API_AUTHORIZATION_PRINCIPAL_ID": "http-service-test-principal", } from episodic.api import runtime diff --git a/tests/steps/test_no_qa_generation_slice.py b/tests/steps/test_no_qa_generation_slice.py new file mode 100644 index 00000000..fda41c3b --- /dev/null +++ b/tests/steps/test_no_qa_generation_slice.py @@ -0,0 +1,385 @@ +"""Behavioural coverage for the no-QA source-to-script REST slice.""" + +import asyncio # noqa: TC003 - pytest resolves fixture annotations at runtime. +import typing as typ +from pathlib import Path # noqa: TC003 - pytest resolves step annotations at runtime. + +import pytest +from pytest_bdd import given, parsers, scenario, then, when + +from episodic.canonical.tei import parse_tei_header +from tests.steps import no_qa_generation_slice_assertions +from tests.steps.no_qa_generation_slice_support import ( + NoQaGenerationSliceContext, + assert_replay_headers, + assert_response_status, + assert_tei_response, + configure_vidaimock, + enable_provider_failure, + generation_payload, + require, + select_malformed_completion, +) + +_FEATURE = "../features/no_qa_generation_slice.feature" + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + import httpx + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.fixture +def context( + session_factory: async_sessionmaker[AsyncSession], + _function_scoped_runner: asyncio.Runner, +) -> cabc.Iterator[NoQaGenerationSliceContext]: + """Provide shared scenario state and release external resources afterward.""" + ctx = NoQaGenerationSliceContext(session_factory, _function_scoped_runner) + yield ctx + ctx.tear_down() + + +@scenario(_FEATURE, "Draft generation without QA produces a downloadable TEI-P5 script") +def test_no_qa_generation_produces_downloadable_tei() -> None: + """Run the no-QA generation happy path.""" + + +@scenario(_FEATURE, "Reusing an idempotency key with the same body replays the run") +def test_no_qa_generation_idempotency_replays_same_body() -> None: + """Run the idempotent replay scenario.""" + + +@scenario(_FEATURE, "Reusing an idempotency key with a different body conflicts") +def test_no_qa_generation_idempotency_conflicts_on_body_mismatch() -> None: + """Run the idempotency conflict scenario.""" + + +@scenario(_FEATURE, "A missing rationale is rejected") +def test_no_qa_generation_rejects_missing_rationale() -> None: + """Run the missing-rationale scenario.""" + + +@scenario(_FEATURE, "An unsupported quality mode is unprocessable") +def test_no_qa_generation_rejects_unsupported_quality_mode() -> None: + """Run the unsupported-quality-mode scenario.""" + + +@scenario(_FEATURE, "Generation failure is reported on the run") +def test_no_qa_generation_reports_generation_failure() -> None: + """Run the provider failure scenario.""" + + +@scenario(_FEATURE, "A malformed completion is reported as a failed run") +def test_no_qa_generation_reports_malformed_completion() -> None: + """Run the malformed-completion scenario.""" + + +@given("a Vidai Mock inference server is running") +def vidai_mock_server_running( + context: NoQaGenerationSliceContext, tmp_path: Path +) -> None: + """Start Vidai Mock and wire the complete application stack.""" + configure_vidaimock(context, tmp_path) + + +@given("a series profile exists") +def series_profile_exists(context: NoQaGenerationSliceContext) -> None: + """Create the series profile used by the ingestion job.""" + response = context.runner.run( + context.request( + "POST", + "/v1/series-profiles", + json={ + "slug": "no-qa-bdd", + "title": "No-QA BDD", + "description": "BDD fixture.", + "configuration": {}, + "actor": "editor@example.com", + }, + ) + ) + assert response.status_code == 201, response.text + context.profile_id = response.json()["id"] + + +@given("a host presenter profile and a guest presenter profile are bound") +def presenter_profiles_are_bound(context: NoQaGenerationSliceContext) -> None: + """Create and bind host and guest profile revisions to the series.""" + profile_id = require(context.profile_id, "series profile") + for kind, name in (("host_profile", "BDD Host"), ("guest_profile", "BDD Guest")): + document = context.runner.run( + context.request( + "POST", + f"/v1/series-profiles/{profile_id}/reference-documents", + json={ + "kind": kind, + "lifecycle_state": "active", + "metadata": {"name": name}, + }, + ) + ) + assert document.status_code == 201, document.text + document_id = document.json()["id"] + revision = context.runner.run( + context.request( + "POST", + f"/v1/series-profiles/{profile_id}/reference-documents/" + f"{document_id}/revisions", + json={ + "content": {"summary": f"{name} profile content."}, + "content_hash": f"no-qa-{kind}", + "author": "editor@example.com", + "change_note": "Create BDD presenter.", + }, + ) + ) + assert revision.status_code == 201, revision.text + binding = context.runner.run( + context.request( + "POST", + "/v1/reference-bindings", + json={ + "reference_document_revision_id": revision.json()["id"], + "target_kind": "series_profile", + "series_profile_id": profile_id, + }, + ) + ) + assert binding.status_code == 201, binding.text + + +@given("an ingestion job with an attached source document") +def ingestion_job_has_source(context: NoQaGenerationSliceContext) -> None: + """Create a ready ingestion job with deterministic source content.""" + profile_id = require(context.profile_id, "series profile") + job = context.runner.run( + context.request( + "POST", + "/v1/ingestion-jobs", + headers={"Idempotency-Key": "no-qa-job"}, + json={"series_profile_id": profile_id}, + ) + ) + assert job.status_code == 201, job.text + context.ingestion_job_id = job.json()["id"] + source = context.runner.run( + context.request( + "POST", + f"/v1/ingestion-jobs/{context.ingestion_job_id}/sources", + headers={"Idempotency-Key": "no-qa-source"}, + json={ + "type": "source_uri", + "source_uri": "https://example.test/source.txt", + "source_type": "research_note", + "weight": 1.0, + "metadata": {"content": "A deterministic source for the episode."}, + }, + ) + ) + assert source.status_code == 201, source.text + + +@given("the inference server is configured to fail") +def inference_server_fails(context: NoQaGenerationSliceContext) -> None: + """Enable deterministic provider failure injection.""" + enable_provider_failure(context) + + +@given("the inference server is configured to return a non-TEI completion") +def inference_server_returns_non_tei(context: NoQaGenerationSliceContext) -> None: + """Select a completion that cannot form a TEI draft.""" + select_malformed_completion(context) + + +def _create( + context: NoQaGenerationSliceContext, + payload: dict[str, object], + key: str = "no-qa-run", +) -> httpx.Response: + job_id = require(context.ingestion_job_id, "ingestion job") + response = context.runner.run( + context.request( + "POST", + f"/v1/ingestion-jobs/{job_id}/generation-runs", + headers={"Idempotency-Key": key}, + json=payload, + ) + ) + context.responses.append(response) + launcher = require(context.launcher, "generation launcher") + context.runner.run(launcher.drain()) + return response + + +@when("I create a draft-without-qa generation run for the ingested episode") +def create_draft_without_qa_run(context: NoQaGenerationSliceContext) -> None: + """Create one no-QA generation run.""" + _create(context, generation_payload()) + + +@when("I create a draft-without-qa run twice with the same idempotency key and body") +def create_draft_without_qa_run_twice(context: NoQaGenerationSliceContext) -> None: + """Replay an identical generation request.""" + _create(context, generation_payload()) + _create(context, generation_payload()) + + +@when("I create a draft-without-qa run, then reuse the key with a different rationale") +def create_draft_without_qa_run_with_changed_body( + context: NoQaGenerationSliceContext, +) -> None: + """Reuse an idempotency key with changed input.""" + _create(context, generation_payload()) + _create(context, generation_payload(skip_qa_rationale="Changed rationale.")) + + +@when("I create a draft-without-qa run without a skip_qa_rationale") +def create_draft_without_qa_run_without_rationale( + context: NoQaGenerationSliceContext, +) -> None: + """Submit a request without its required rationale.""" + _create( + context, {"quality_mode": "draft_without_qa", "actor": "editor@example.com"} + ) + + +@when(parsers.parse('I create a generation run with quality_mode "{quality_mode}"')) +def create_generation_run_with_quality_mode( + context: NoQaGenerationSliceContext, quality_mode: str +) -> None: + """Submit a request with the requested quality mode.""" + _create(context, generation_payload(quality_mode=quality_mode)) + + +@when("I poll the generation run until it reaches a terminal state") +def poll_generation_run_until_terminal(context: NoQaGenerationSliceContext) -> None: + """Drain the in-process launcher and fetch terminal run and event state.""" + launcher = require(context.launcher, "generation launcher") + context.runner.run(launcher.drain()) + location = context.responses[0].headers["Location"] + context.run_response = context.runner.run(context.request("GET", location)) + run_id = context.run_response.json()["id"] + context.events_response = context.runner.run( + context.request("GET", f"/v1/generation-runs/{run_id}/events") + ) + + +@when("I fetch the episode TEI as application/tei+xml") +def fetch_episode_tei_as_xml(context: NoQaGenerationSliceContext) -> None: + """Fetch the generated episode as a TEI attachment.""" + episode_id = context.responses[0].json()["episode_id"] + context.tei_response = context.runner.run( + context.request( + "GET", + f"/v1/episodes/{episode_id}/tei", + headers={"Accept": "application/tei+xml"}, + ) + ) + + +@then("the run creation responds 202 Accepted with a Location header") +def run_creation_returns_accepted(context: NoQaGenerationSliceContext) -> None: + """Verify the asynchronous operation response metadata.""" + response = context.responses[0] + assert_response_status(response, 202) + location = response.headers.get("Location") + assert location, f"expected a non-empty Location header, got {location!r}" + + +@then("the response carries a Retry-After header") +def response_carries_retry_after(context: NoQaGenerationSliceContext) -> None: + """Verify the server supplies a polling interval.""" + retry_after = context.responses[0].headers["Retry-After"] + assert retry_after == "1", f"expected Retry-After '1', got {retry_after!r}" + + +@then( + parsers.parse( + 'the run is created with qa_status "{qa_status}" and my rationale recorded' + ) +) +def run_records_qa_status(context: NoQaGenerationSliceContext, qa_status: str) -> None: + """Verify QA bypass provenance is represented on creation.""" + body = context.responses[0].json() + assert body["qa_status"] == qa_status, f"QA status: {body['qa_status']!r}" + assert body["skip_qa_rationale"] == generation_payload()["skip_qa_rationale"], ( + f"rationale: {body['skip_qa_rationale']!r}" + ) + + +@then(parsers.parse('the run status is "{status}"')) +def run_status_is(context: NoQaGenerationSliceContext, status: str) -> None: + """Verify the polled run reached the expected terminal state.""" + observed_status = require(context.run_response, "run response").json()["status"] + assert observed_status == status, f"run status: {observed_status!r}" + + +@then(parsers.parse('the event log contains a "{event_kind}" event')) +def event_log_contains(context: NoQaGenerationSliceContext, event_kind: str) -> None: + """Verify the durable event stream contains the named event.""" + items = require(context.events_response, "event response").json()["items"] + observed_kinds = {item["kind"] for item in items} + assert event_kind in observed_kinds, f"event kinds: {observed_kinds!r}" + + +@then(parsers.parse('the response is a TEI-P5 attachment with qa_status "{qa_status}"')) +def response_is_tei_attachment( + context: NoQaGenerationSliceContext, qa_status: str +) -> None: + """Verify raw TEI download metadata and QA provenance.""" + assert_tei_response( + require(context.tei_response, "TEI response"), + require(context.run_response, "run response"), + qa_status, + ) + + +@then("the TEI validates against the Episodic TEI-P5 profile") +def tei_validates(context: NoQaGenerationSliceContext) -> None: + """Validate the downloaded document through the canonical TEI parser.""" + observed_title = parse_tei_header( + require(context.tei_response, "TEI response").text + ).title + assert observed_title == "A deterministic no-QA draft", f"title: {observed_title}" + + +@then("both responses describe the same run id") +def responses_describe_same_run(context: NoQaGenerationSliceContext) -> None: + """Verify an identical replay resolves to the original run.""" + first_run_id = context.responses[0].json()["id"] + second_run_id = context.responses[1].json()["id"] + assert first_run_id == second_run_id, f"replay run id: {second_run_id!r}" + + +@then("the replayed response carries the same Location and Retry-After") +def replay_preserves_polling_headers(context: NoQaGenerationSliceContext) -> None: + """Verify replay retains long-running operation headers.""" + assert_replay_headers(context.responses[0], context.responses[1]) + + +@then("the second response is 409 Conflict") +def second_response_is_conflict(context: NoQaGenerationSliceContext) -> None: + """Verify changed input is rejected under the reused key.""" + no_qa_generation_slice_assertions.second_response_is_conflict(context) + + +@then("the response is 400 Bad Request") +def response_is_bad_request(context: NoQaGenerationSliceContext) -> None: + """Verify malformed quality metadata is a bad request.""" + no_qa_generation_slice_assertions.response_is_bad_request(context) + + +@then("the response is 422 Unprocessable Entity") +def response_is_unprocessable(context: NoQaGenerationSliceContext) -> None: + """Verify a recognized unsupported mode is unprocessable.""" + no_qa_generation_slice_assertions.response_is_unprocessable(context) + + +@then("the run records an error message and an error category") +def run_records_error(context: NoQaGenerationSliceContext) -> None: + """Verify terminal failures expose stable diagnostic fields.""" + body = require(context.run_response, "run response").json() + assert body["error_message"], f"error message: {body['error_message']!r}" + assert body["error_category"], f"error category: {body['error_category']!r}" diff --git a/tests/steps/test_no_qa_generation_slice_support.py b/tests/steps/test_no_qa_generation_slice_support.py new file mode 100644 index 00000000..4e526871 --- /dev/null +++ b/tests/steps/test_no_qa_generation_slice_support.py @@ -0,0 +1,47 @@ +"""Unit tests for no-QA behavioural-slice infrastructure.""" + +import asyncio +import typing as typ + +import pytest + +from tests.steps.no_qa_generation_slice_support import NoQaGenerationSliceContext + + +class _FailingLauncher: + """Fail shutdown to exercise process cleanup after asynchronous failure.""" + + async def shutdown(self) -> None: + """Raise the controlled teardown failure.""" + msg = "launcher shutdown failed" + raise RuntimeError(msg) + + +class _Process: + """Record process cleanup calls.""" + + terminated = False + + def terminate(self) -> None: + """Record graceful termination.""" + self.terminated = True + + def wait(self, *, timeout: float) -> None: + """Complete the graceful wait.""" + + +def test_tear_down_terminates_vidai_mock_after_async_cleanup_failure() -> None: + """The child process is terminated even when launcher shutdown raises.""" + process = _Process() + with asyncio.Runner() as runner: + context = NoQaGenerationSliceContext( + session_factory=typ.cast("typ.Any", object()), + runner=runner, + process=typ.cast("typ.Any", process), + launcher=typ.cast("typ.Any", _FailingLauncher()), + ) + + with pytest.raises(RuntimeError, match="launcher shutdown failed"): + context.tear_down() + + assert process.terminated, "expected Vidai Mock termination after cleanup failure" diff --git a/tests/test_api_authorization.py b/tests/test_api_authorization.py index bafc628e..486b51a2 100644 --- a/tests/test_api_authorization.py +++ b/tests/test_api_authorization.py @@ -10,6 +10,8 @@ AuthorizationContext, AuthorizationDecision, AuthorizationPort, + AuthorizationResult, + StaticBearerTokenAuthorization, ) from tests.fixtures.api import build_api_dependencies @@ -46,6 +48,44 @@ async def decide(self, context: AuthorizationContext) -> AuthorizationDecision: raise RuntimeError(msg) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("header", "expected"), + [ + ("Bearer configured-token", AuthorizationDecision.PERMIT), + ("bearer configured-token", AuthorizationDecision.PERMIT), + ("BEARER configured-token", AuthorizationDecision.PERMIT), + ("Basic configured-token", AuthorizationDecision.UNAUTHORIZED), + ("Bearer", AuthorizationDecision.UNAUTHORIZED), + ("Bearer wrong-token", AuthorizationDecision.UNAUTHORIZED), + (None, AuthorizationDecision.UNAUTHORIZED), + ], +) +async def test_static_bearer_authorization_parses_scheme_and_token( + header: str | None, + expected: AuthorizationDecision, +) -> None: + """Accept case-insensitive Bearer schemes and reject malformed credentials.""" + configured_credential = "".join(("configured", "-token")) + authorization = StaticBearerTokenAuthorization( + token=configured_credential, + principal_id="configured-principal", + ) + + result = await authorization.decide( + AuthorizationContext( + method="GET", path="/v1/example", authorization_header=header + ) + ) + + assert isinstance(result, AuthorizationResult), result + assert result.decision is expected, result + if expected is AuthorizationDecision.PERMIT: + assert result.principal_id == "configured-principal", result + else: + assert result.principal_id is None, result + + def _build_client( session_factory: async_sessionmaker[AsyncSession], authorization: AuthorizationPort, diff --git a/tests/test_cost_ports_protocols.py b/tests/test_cost_ports_protocols.py index a3fc383f..be211b28 100644 --- a/tests/test_cost_ports_protocols.py +++ b/tests/test_cost_ports_protocols.py @@ -21,7 +21,9 @@ TaskRollupLedgerEntry, UsageSource, ) +from episodic.cost.recorder import CostRecorderPort from episodic.llm.ports import LLMResponse, LLMUsage, ProviderCallUsage +from tests.generation_run_launcher_support import RecordingCostRecorder _DEFAULT_BILLING_PERIOD = BillingPeriodKey("2026-06") _DEFAULT_SNAPSHOT_KEY = RunPricingKey( @@ -162,6 +164,24 @@ def test_cost_protocol_fakes_satisfy_public_ports() -> None: assert inspect.iscoroutinefunction(metering.consume), "Expected condition to hold" +def test_recording_cost_recorder_satisfies_public_port() -> None: + """The generation-run recorder fake matches the public cost contract.""" + recorder = RecordingCostRecorder() + + assert isinstance(recorder, CostRecorderPort), ( + "Expected value to have the required type" + ) + assert inspect.iscoroutinefunction(recorder.pin_run_pricing), ( + "Expected condition to hold" + ) + assert inspect.iscoroutinefunction(recorder.record_provider_call), ( + "Expected condition to hold" + ) + assert inspect.iscoroutinefunction(recorder.finalize_run), ( + "Expected condition to hold" + ) + + def test_pricing_engine_returns_priced_call() -> None: """The pricing engine has the deterministic domain pricing surface.""" snapshot = _make_snapshot(PricingSnapshotId("snapshot:test")) diff --git a/tests/test_draft_script_generation.py b/tests/test_draft_script_generation.py new file mode 100644 index 00000000..7089ef28 --- /dev/null +++ b/tests/test_draft_script_generation.py @@ -0,0 +1,286 @@ +"""Tests for single-pass draft script generation.""" + +import dataclasses as dc +import datetime as dt +import hashlib +import json +import typing as typ +import uuid + +import pytest + +from episodic.canonical.tei import parse_tei_header +from episodic.generation.draft_script import ( + DraftPresenterProfile, + DraftScriptProviderResponseError, + DraftScriptRequest, + DraftScriptResponseFormatError, + DraftScriptSource, + DraftScriptTokenBudgetError, + DraftScriptTransientProviderError, + LLMDraftScriptGenerator, + LLMDraftScriptGeneratorConfig, +) +from episodic.llm import ( + LLMProviderResponseError, + LLMRequest, + LLMResponse, + LLMTokenBudgetExceededError, + LLMTransientProviderError, + LLMUsage, +) + +if typ.TYPE_CHECKING: + from syrupy.assertion import SnapshotAssertion + + +class FakeLLMPort: + """Capture draft-generation requests and return a canned response.""" + + def __init__( + self, + response: LLMResponse | None = None, + error: Exception | None = None, + ) -> None: + self.response = response + self.error = error + self.requests: list[LLMRequest] = [] + + async def generate(self, request: LLMRequest) -> LLMResponse: + """Return the canned response or raise the configured error.""" + self.requests.append(request) + if self.error is not None: + raise self.error + if self.response is None: + raise AssertionError + return self.response + + +class SequentialDraftIds: + """Deterministic TEI identifier factory for snapshots.""" + + def __init__(self) -> None: + self.counts: dict[str, int] = {} + + def __call__(self, prefix: str) -> str: + """Return the next identifier for a TEI element prefix.""" + next_value = self.counts.get(prefix, 0) + 1 + self.counts[prefix] = next_value + return f"{prefix}-{next_value}" + + +def _clock() -> dt.datetime: + """Return the frozen draft-generation timestamp.""" + return dt.datetime(2026, 6, 24, 12, 0, tzinfo=dt.UTC) + + +def _valid_response() -> LLMResponse: + """Return a valid draft script JSON response.""" + payload = { + "title": "Bridgewater Futures", + "turns": [ + {"speaker": "Host", "text": "Welcome to Bridgewater Futures."}, + {"speaker": "Guest", "text": "Thanks for inviting me."}, + {"text": "The conversation turns to implementation risks."}, + ], + } + return LLMResponse( + text=json.dumps(payload), + model="vidai-mock", + provider_response_id="resp-draft-1", + finish_reason="stop", + usage=LLMUsage(input_tokens=100, output_tokens=50, total_tokens=150), + ) + + +def _request() -> DraftScriptRequest: + """Return a representative draft-generation request.""" + return DraftScriptRequest( + episode_id=uuid.UUID("00000000-0000-0000-0000-000000000001"), + series_profile_id=uuid.UUID("00000000-0000-0000-0000-000000000002"), + title="Bridgewater Futures", + sources=( + DraftScriptSource( + source_id="source-1", + source_type="research_brief", + source_uri="https://example.test/source", + content="Bridgewater is preparing a new product launch.", + weight=1.0, + ), + ), + presenter_profiles=( + DraftPresenterProfile( + display_name="Host", + role="host", + source_content="Experienced technical presenter.", + ), + DraftPresenterProfile( + display_name="Guest", + role="guest", + source_content="Product lead for the launch.", + ), + ), + clock=_clock, + id_factory=SequentialDraftIds(), + ) + + +def test_draft_source_rejects_non_string_content() -> None: + """Source validation distinguishes type errors from blank text.""" + with pytest.raises(TypeError, match="content must be a string"): + DraftScriptSource( + source_id="source-1", + source_type="research_brief", + source_uri="https://example.test/source", + content=typ.cast("str", 1), + weight=1.0, + ) + + +@pytest.mark.asyncio +async def test_draft_script_generator_emits_valid_stable_tei( + snapshot: SnapshotAssertion, +) -> None: + """LLM draft output should become validated deterministic TEI-P5.""" + fake_llm = FakeLLMPort(_valid_response()) + generator = LLMDraftScriptGenerator( + llm=fake_llm, + config=LLMDraftScriptGeneratorConfig(model="vidai-mock"), + ) + + result = await generator.generate(_request()) + + parsed_title = parse_tei_header(result.tei_xml).title + assert parsed_title == "Bridgewater Futures", ( + f"expected generated TEI title 'Bridgewater Futures', got {parsed_title!r}" + ) + expected_hash = hashlib.sha256(result.tei_xml.encode()).hexdigest() + assert result.content_hash == f"sha256:{expected_hash}", ( + f"expected generated TEI hash sha256:{expected_hash}, " + f"got {result.content_hash!r}" + ) + assert result.usage.total_tokens == 150, ( + f"expected 150 total tokens, got {result.usage.total_tokens}" + ) + assert result.provider_response_id == "resp-draft-1", ( + "expected provider response 'resp-draft-1', " + f"got {result.provider_response_id!r}" + ) + assert fake_llm.requests[0].model == "vidai-mock", ( + f"expected model 'vidai-mock', got {fake_llm.requests[0].model!r}" + ) + assert fake_llm.requests[0].system_prompt is not None, ( + "expected a system prompt, got None" + ) + assert result.tei_xml == snapshot, "generated TEI must match the approved snapshot" + + +@pytest.mark.asyncio +async def test_draft_script_generator_serializes_deterministic_prompt() -> None: + """Generator should send the complete deterministic draft context to the LLM.""" + fake_llm = FakeLLMPort(_valid_response()) + generator = LLMDraftScriptGenerator( + llm=fake_llm, + config=LLMDraftScriptGeneratorConfig(model="vidai-mock"), + ) + + await generator.generate(_request()) + + expected_prompt = { + "episode_id": "00000000-0000-0000-0000-000000000001", + "presenter_profiles": [ + { + "display_name": "Host", + "role": "host", + "source_content": "Experienced technical presenter.", + }, + { + "display_name": "Guest", + "role": "guest", + "source_content": "Product lead for the launch.", + }, + ], + "requested_at": "2026-06-24T12:00:00+00:00", + "series_profile_id": "00000000-0000-0000-0000-000000000002", + "sources": [ + { + "content": "Bridgewater is preparing a new product launch.", + "source_id": "source-1", + "source_type": "research_brief", + "source_uri": "https://example.test/source", + "weight": 1.0, + } + ], + "title": "Bridgewater Futures", + } + prompt = fake_llm.requests[0].prompt + assert json.loads(prompt) == expected_prompt, ( + f"expected complete draft prompt payload, got {prompt!r}" + ) + assert prompt == json.dumps(expected_prompt, indent=2, sort_keys=True), ( + "expected stable, sorted JSON prompt serialization" + ) + + +@pytest.mark.parametrize( + ("llm_error", "expected_error"), + [ + (LLMTokenBudgetExceededError(), DraftScriptTokenBudgetError), + (LLMProviderResponseError(), DraftScriptProviderResponseError), + (LLMTransientProviderError(), DraftScriptTransientProviderError), + ], +) +@pytest.mark.asyncio +async def test_draft_script_generator_maps_llm_errors( + llm_error: Exception, + expected_error: type[Exception], +) -> None: + """Provider failures should cross the generator boundary as draft errors.""" + generator = LLMDraftScriptGenerator( + llm=FakeLLMPort(error=llm_error), + config=LLMDraftScriptGeneratorConfig(model="vidai-mock"), + ) + + with pytest.raises(expected_error): + await generator.generate(_request()) + + +@pytest.mark.asyncio +async def test_draft_script_generator_rejects_malformed_completion() -> None: + """Malformed LLM JSON should not reach TEI persistence.""" + generator = LLMDraftScriptGenerator( + llm=FakeLLMPort( + LLMResponse( + text=json.dumps({ + "title": "Bridgewater Futures", + "turns": [{"speaker": "Host"}], + }), + model="vidai-mock", + provider_response_id="bad", + finish_reason="stop", + usage=LLMUsage(input_tokens=1, output_tokens=1, total_tokens=2), + ) + ), + config=LLMDraftScriptGeneratorConfig(model="vidai-mock"), + ) + + with pytest.raises(DraftScriptResponseFormatError, match="text"): + await generator.generate(_request()) + + +@pytest.mark.asyncio +async def test_draft_script_generator_rejects_oversized_completion() -> None: + """Response-size limits must apply before JSON parsing.""" + generator = LLMDraftScriptGenerator( + llm=FakeLLMPort(dc.replace(_valid_response(), text="x" * 10)), + config=LLMDraftScriptGeneratorConfig( + model="vidai-mock", + max_response_bytes=9, + ), + ) + + with pytest.raises( + DraftScriptResponseFormatError, + match="LLM response exceeds the configured maximum size", + ): + await generator.generate(_request()) diff --git a/tests/test_env_runtime_wiring.py b/tests/test_env_runtime_wiring.py index 0503e0cb..01465127 100644 --- a/tests/test_env_runtime_wiring.py +++ b/tests/test_env_runtime_wiring.py @@ -1,6 +1,8 @@ """Tests for runtime environment wiring of the HTTP app.""" -import hashlib +import asyncio +import os +import pathlib import typing as typ import httpx @@ -12,6 +14,11 @@ from pathlib import Path from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.api.dependencies import ApiDependencies + from episodic.llm import LLMRequest, LLMResponse + from episodic.observability import MetricsPort def test_create_app_from_env_requires_database_url( @@ -83,6 +90,120 @@ def test_normalize_database_urls_uses_query_port_for_probe() -> None: assert probe_kwargs["port"] == 6544, "Expected values to match" +@pytest.mark.asyncio +async def test_build_generation_launcher_wires_cost_recorder( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """Runtime launcher construction should use the SQL-backed cost ledger.""" + from episodic.api.runtime import ( + RuntimeConfig, + _build_generation_launcher, + _GenerationLauncherRuntime, + ) + from episodic.canonical.storage import SqlAlchemyUnitOfWork + from episodic.cost.recorder import CostRecorder + from episodic.generation import ( + GenerationSourceLimits, + InProcessGenerationRunLauncher, + ) + from episodic.observability import StructuredLogMetrics + + launcher = _build_generation_launcher( + lambda: SqlAlchemyUnitOfWork(session_factory), + _UnusedLLMPort(), + _GenerationLauncherRuntime(metrics=StructuredLogMetrics()), + config=RuntimeConfig( + database_url="postgresql+psycopg://unused", + source_intake_object_store_root=tmp_path, + llm_base_url=None, + llm_api_key=None, + draft_model="draft-model", + pricing_snapshot_directory=pathlib.Path("config/pricing-snapshots"), + authorization_bearer_token=os.environ["API_AUTHORIZATION_BEARER_TOKEN"], + authorization_principal_id="runtime-test-principal", + generation_source_limits=GenerationSourceLimits(), + ), + ) + + assert isinstance(launcher, InProcessGenerationRunLauncher), ( + f"expected in-process launcher, got {type(launcher).__name__}" + ) + assert launcher.cost_recorder_factory is not None, ( + "expected a cost recorder factory, got None" + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + recorder = launcher.cost_recorder_factory(uow) + assert isinstance(recorder, CostRecorder), ( + f"expected CostRecorder, got {type(recorder).__name__}" + ) + assert recorder.ledger is uow.cost_ledger, ( + f"expected unit-of-work ledger {uow.cost_ledger!r}, got {recorder.ledger!r}" + ) + + +@pytest.mark.asyncio +async def test_create_app_from_env_wires_configured_llm_launcher( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Provider configuration should create a launcher and shutdown hook.""" + from unittest import mock + + from episodic.api import runtime as runtime_module + from episodic.generation import InProcessGenerationRunLauncher + from episodic.llm.openai_adapter import OpenAICompatibleLLMAdapter + + monkeypatch.setenv("DATABASE_URL", "postgresql://example.test/episodic") + monkeypatch.setenv("SOURCE_INTAKE_OBJECT_STORE_ROOT", str(tmp_path)) + monkeypatch.setenv("OPENAI_BASE_URL", "https://llm.example.test/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + captured_dependencies: ApiDependencies | None = None + + async def shutdown_database() -> None: + await asyncio.sleep(0) + + async def check_database() -> bool: + await asyncio.sleep(0) + return True + + def unit_of_work_factory() -> object: + return object() + + def capture_dependencies(dependencies: ApiDependencies) -> object: + nonlocal captured_dependencies + captured_dependencies = dependencies + return object() + + probe = runtime_module.ReadinessProbe(name="database", check=check_database) + with ( + mock.patch.object( + runtime_module, + "_build_database_probe", + return_value=(probe, unit_of_work_factory, shutdown_database), + ), + mock.patch.object( + runtime_module, + "create_app", + side_effect=capture_dependencies, + ), + ): + runtime_module.create_app_from_env() + + assert captured_dependencies is not None, "expected captured dependencies, got None" + assert isinstance(captured_dependencies.llm_port, OpenAICompatibleLLMAdapter), ( + f"expected OpenAI adapter, got {type(captured_dependencies.llm_port).__name__}" + ) + assert isinstance(captured_dependencies.launcher, InProcessGenerationRunLauncher), ( + "expected in-process launcher, got " + f"{type(captured_dependencies.launcher).__name__}" + ) + assert len(captured_dependencies.shutdown_hooks) == 2, ( + f"expected two shutdown hooks, got {len(captured_dependencies.shutdown_hooks)}" + ) + await captured_dependencies.shutdown_hooks[0]() + + @pytest.mark.asyncio @pytest.mark.parametrize( "strip_driver", @@ -159,8 +280,12 @@ async def test_create_app_from_env_runs_shutdown_hooks_during_lifespan( shutdown_hook_called = False original_build = runtime_module._build_database_probe - def _tracking_build(database_url: str) -> tuple[object, ...]: - probe, uow, original_hook = original_build(database_url) + def _tracking_build( + database_url: str, + *, + metrics: "MetricsPort", # noqa: UP037 - imported only during type checking. + ) -> tuple[object, ...]: + probe, uow, original_hook = original_build(database_url, metrics=metrics) async def _tracked_hook() -> None: nonlocal shutdown_hook_called @@ -216,52 +341,20 @@ def test_runtime_exposes_container_granian_contract() -> None: ) -@pytest.mark.asyncio -async def test_create_app_from_env_wires_object_store_for_uploads( - migrated_database_url: str, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Runtime-created apps accept uploads when object storage is configured.""" - object_store_root = tmp_path / "objects" - monkeypatch.setenv("DATABASE_URL", migrated_database_url) - monkeypatch.setenv("SOURCE_INTAKE_OBJECT_STORE_ROOT", str(object_store_root)) +class _UnusedLLMPort: + """LLM port fake used only to satisfy runtime launcher construction.""" - from episodic.api.runtime import create_app_from_env + async def generate( + self, + request: "LLMRequest", # noqa: UP037 - imported only during type checking + ) -> "LLMResponse": # noqa: UP037 - imported only during type checking + """Fail if runtime wiring accidentally invokes the fake.""" + _ = request + raise AssertionError - app = create_app_from_env() - try: - transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", app)) - async with httpx.AsyncClient( - transport=transport, - base_url="http://testserver", - ) as client: - payload = b"runtime upload\n" - response = await client.post( - "/v1/uploads", - headers={"Idempotency-Key": "runtime-upload"}, - files={ - "file": ("source.txt", payload, "text/plain"), - "content_type": (None, "text/plain"), - "declared_size": (None, str(len(payload))), - "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), - }, - ) - finally: - await scaffold_support.run_asgi_lifespan( - typ.cast("_ASGIApp", app), - ( - scaffold_support.LifespanEvent(type="lifespan.startup"), - scaffold_support.LifespanEvent(type="lifespan.shutdown"), - ), - ) - assert response.status_code == 201, response.text - response_body = response.json() - expected_hash = hashlib.sha256(payload).hexdigest() - stored_path = object_store_root / "uploads" / response_body["id"] - assert response_body["content_hash"] == f"sha256:{expected_hash}", ( - "Expected values to match" - ) - assert stored_path.is_file(), f"expected upload payload at {stored_path}" - assert stored_path.read_bytes() == payload, "Expected values to match" +@pytest.fixture(autouse=True) +def _configure_runtime_authorization(monkeypatch: pytest.MonkeyPatch) -> None: + """Provide the required production authorization settings for runtime tests.""" + monkeypatch.setenv("API_AUTHORIZATION_BEARER_TOKEN", "runtime-test-token") + monkeypatch.setenv("API_AUTHORIZATION_PRINCIPAL_ID", "runtime-test-principal") diff --git a/tests/test_episode_tei_api.py b/tests/test_episode_tei_api.py new file mode 100644 index 00000000..9c85f013 --- /dev/null +++ b/tests/test_episode_tei_api.py @@ -0,0 +1,376 @@ +"""Integration tests for episode TEI retrieval.""" + +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +import falcon +import httpx +import pytest + +from episodic.api import create_app +from episodic.api.resources.episode_tei import negotiate_tei_media_type +from episodic.canonical.domain import EpisodeTeiUpdate +from episodic.canonical.generation_quality import QaStatus +from episodic.observability import RecordingTracer +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import ( + HeaderPrincipalAuthorization, + RecordingLauncher, +) + +_GENERATED_TEI = "

Generated script.

" + +if typ.TYPE_CHECKING: + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.api.dependencies import ApiDependencies + + +@pytest.mark.parametrize( + ("accept", "expected"), + [ + ("application/tei+xml;q=0, application/json;q=0.5", "application/json"), + ( + "application/json;q=0.1, application/tei+xml;q=0.9", + "application/tei+xml", + ), + ("application/tei+xml;q=0.1, */*;q=0.5", "application/json"), + ], +) +def test_negotiate_tei_media_type_honours_accept_quality( + accept: str, + expected: str, +) -> None: + """TEI negotiation selects the highest-quality supported representation.""" + actual = negotiate_tei_media_type(accept) + assert actual == expected, ( + f"Accept {accept!r}: expected {expected!r}, got {actual!r}" + ) + + +def test_negotiate_tei_media_type_rejects_zero_quality_ranges() -> None: + """TEI negotiation rejects headers that exclude every supported type.""" + with pytest.raises(falcon.HTTPNotAcceptable): + negotiate_tei_media_type("application/json;q=0, application/tei+xml;q=0") + + +@pytest.mark.asyncio +async def test_episode_tei_json_and_xml_retrieval( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Return metadata by default and a TEI attachment when requested.""" + launcher = RecordingLauncher() + tracer = RecordingTracer() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + tracer=tracer, + ) + headers = {"Authorization": "Bearer principal-a"} + async with httpx.AsyncClient( + transport=httpx.ASGITransport( + app=typ.cast("_ASGIApp", create_app(dependencies)) + ), + base_url="http://testserver", + ) as client: + episode_id = await _create_generation_run(client, headers=headers) + before_draft = await client.get( + f"/v1/episodes/{episode_id}/tei", headers=headers + ) + tei_xml = _GENERATED_TEI + await _persist_generated_tei(dependencies, episode_id, launcher.run_ids[0]) + json_response = await client.get( + f"/v1/episodes/{episode_id}/tei", headers=headers + ) + xml_response = await client.get( + f"/v1/episodes/{episode_id}/tei", + headers={**headers, "Accept": "application/tei+xml"}, + ) + unacceptable = await client.get( + f"/v1/episodes/{episode_id}/tei", + headers={**headers, "Accept": "text/plain"}, + ) + + assert before_draft.status_code == 404, ( + f"expected missing draft status 404, got {before_draft.status_code}" + ) + assert unacceptable.status_code == 406, ( + f"expected unacceptable media status 406, got {unacceptable.status_code}" + ) + _assert_tei_json_response( + json_response, + episode_id=episode_id, + generation_run_id=launcher.run_ids[0], + tei_xml=tei_xml, + ) + _assert_tei_xml_response( + xml_response, + episode_id=episode_id, + tei_xml=tei_xml, + ) + assert json_response.headers["ETag"] != xml_response.headers["ETag"], ( + "expected representation-specific ETags for JSON metadata and TEI XML" + ) + _assert_tei_read_spans(tracer) + + +@pytest.mark.asyncio +async def test_episode_tei_retrieval_enforces_principal_ownership( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Hide generated TEI from other and unauthenticated principals.""" + launcher = RecordingLauncher() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + principal_a = {"Authorization": "Bearer principal-a"} + principal_b = {"Authorization": "Bearer principal-b"} + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + episode_id = await _create_generation_run(client, headers=principal_a) + await _persist_generated_tei(dependencies, episode_id, launcher.run_ids[0]) + endpoint = f"/v1/episodes/{episode_id}/tei" + allowed = await client.get(endpoint, headers=principal_a) + unauthenticated = await client.get(endpoint) + cross_principal = await client.get(endpoint, headers=principal_b) + + assert allowed.status_code == 200, allowed.text + assert unauthenticated.status_code == 401, unauthenticated.text + assert cross_principal.status_code == 404, cross_principal.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("accept", [None, "application/tei+xml"]) +@pytest.mark.parametrize( + ("validator_kind", "expected_status"), + [ + ("matching", 304), + ("nonmatching", 200), + ("wildcard", 304), + ("absent", 200), + ], +) +async def test_episode_tei_honours_conditional_get_validators( + session_factory: async_sessionmaker[AsyncSession], + accept: str | None, + validator_kind: str, + expected_status: int, +) -> None: + """Return a body only when the selected TEI representation has changed.""" + launcher = RecordingLauncher() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + ) + principal_headers = {"Authorization": "Bearer principal-a"} + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + episode_id = await _create_generation_run(client, headers=principal_headers) + await _persist_generated_tei(dependencies, episode_id, launcher.run_ids[0]) + headers = ( + principal_headers + if accept is None + else principal_headers | {"Accept": accept} + ) + initial_response = await client.get( + f"/v1/episodes/{episode_id}/tei", headers=headers + ) + validator = { + "matching": initial_response.headers["ETag"], + "nonmatching": '"not-a-match"', + "wildcard": "*", + "absent": None, + }[validator_kind] + conditional_response = await client.get( + f"/v1/episodes/{episode_id}/tei", + headers=headers + if validator is None + else headers | {"If-None-Match": validator}, + ) + + assert initial_response.status_code == 200, initial_response.text + assert conditional_response.status_code == expected_status, ( + conditional_response.text + ) + assert conditional_response.headers["ETag"] == initial_response.headers["ETag"], ( + "expected conditional response to retain the selected representation ETag" + ) + if expected_status == 304: + assert conditional_response.content == b"", conditional_response.content + + +async def _persist_generated_tei( + dependencies: ApiDependencies, + episode_id: uuid.UUID, + generation_run_id: uuid.UUID, +) -> None: + """Persist generated TEI needed by endpoint retrieval tests.""" + async with dependencies.uow_factory() as uow: + await uow.episodes.update( + episode_id, + update=EpisodeTeiUpdate( + tei_xml=_GENERATED_TEI, + qa_status=QaStatus.SKIPPED, + last_generation_run_id=generation_run_id, + expected_revision=1, + updated_at=dt.datetime(2026, 7, 22, 12, 0, tzinfo=dt.UTC), + ), + ) + await uow.commit() + + +def _assert_tei_json_response( + response: httpx.Response, + *, + episode_id: uuid.UUID, + generation_run_id: uuid.UUID, + tei_xml: str, +) -> None: + """Assert the TEI metadata response preserves generation provenance.""" + assert response.status_code == 200, response.text + expected_payload = { + "episode_id": str(episode_id), + "tei_header_id": response.json()["tei_header_id"], + "tei_xml": tei_xml, + "content_hash": _tei_hash(tei_xml), + "version": 2, + "last_generation_run_id": str(generation_run_id), + "quality_mode": "draft_without_qa", + "qa_status": "skipped", + "updated_at": "2026-07-22T12:00:00+00:00", + } + assert response.json() == expected_payload, ( + f"expected TEI metadata {expected_payload!r}, got {response.json()!r}" + ) + assert response.headers["ETag"] == _response_etag(response), ( + f"expected JSON ETag for serialized response, got {response.headers['ETag']!r}" + ) + + +def _assert_tei_xml_response( + response: httpx.Response, + *, + episode_id: uuid.UUID, + tei_xml: str, +) -> None: + """Assert the TEI attachment response preserves its download contract.""" + assert response.status_code == 200, ( + f"expected TEI response status 200, got {response.status_code}" + ) + assert response.text == tei_xml, ( + f"expected TEI body {tei_xml!r}, got {response.text!r}" + ) + assert response.headers["Content-Type"].startswith("application/tei+xml"), ( + f"expected TEI content type, got {response.headers['Content-Type']!r}" + ) + assert response.headers["Content-Disposition"] == ( + f'attachment; filename="episode-{episode_id}.xml"' + ), ( + "expected episode attachment Content-Disposition, got " + f"{response.headers['Content-Disposition']!r}" + ) + assert response.headers["ETag"] == _response_etag(response), ( + f"expected ETag for generated TEI, got {response.headers['ETag']!r}" + ) + + +def _assert_tei_read_spans(tracer: RecordingTracer) -> None: + """Assert the TEI retrieval trace sequence.""" + expected_tei_spans = [ + { + "operation": "episode_tei.read", + "outcome": "not_found", + "failure_category": "episode.not_found", + }, + { + "operation": "episode_tei.read", + "representation": "application/json", + "outcome": "success", + }, + { + "operation": "episode_tei.read", + "representation": "application/tei+xml", + "outcome": "success", + }, + { + "operation": "episode_tei.read", + "outcome": "rejected", + "failure_category": "not_acceptable", + }, + ] + assert [ + span.attributes for span in tracer.spans if span.name == "episode_tei.read" + ] == expected_tei_spans, tracer.spans + + +def _response_etag(response: httpx.Response) -> str: + """Return the quoted SHA-256 ETag for the response representation bytes.""" + return f'"{hashlib.sha256(response.content).hexdigest()}"' + + +async def _create_generation_run( + client: httpx.AsyncClient, + *, + headers: dict[str, str] | None = None, +) -> uuid.UUID: + """Create a ready generation run owned by the supplied test principal.""" + request_headers = {} if headers is None else headers + profile = await client.post( + "/v1/series-profiles", + headers={**request_headers, "Idempotency-Key": "tei-profile-key"}, + json={ + "slug": "tei-retrieval-profile", + "title": "TEI retrieval profile", + "description": "TEI endpoint fixture.", + "configuration": {}, + "actor": "editor@example.com", + }, + ) + assert profile.status_code == 201, profile.text + job = await client.post( + "/v1/ingestion-jobs", + headers={**request_headers, "Idempotency-Key": "tei-job-key"}, + json={"series_profile_id": profile.json()["id"]}, + ) + assert job.status_code == 201, job.text + source = await client.post( + f"/v1/ingestion-jobs/{job.json()['id']}/sources", + headers={**request_headers, "Idempotency-Key": "tei-source-key"}, + json={ + "type": "source_uri", + "source_uri": "https://example.test/source.txt", + "source_type": "research_note", + "weight": 1.0, + "metadata": {"content": "Source text."}, + }, + ) + assert source.status_code == 201, source.text + run = await client.post( + f"/v1/ingestion-jobs/{job.json()['id']}/generation-runs", + headers={**request_headers, "Idempotency-Key": "tei-generation-key"}, + json={ + "quality_mode": "draft_without_qa", + "skip_qa_rationale": "TEI retrieval test.", + "actor": "editor@example.com", + }, + ) + assert run.status_code == 202, run.text + return uuid.UUID(run.json()["episode_id"]) + + +def _tei_hash(tei_xml: str) -> str: + return f"sha256:{hashlib.sha256(tei_xml.encode()).hexdigest()}" diff --git a/tests/test_episode_tei_tracing.py b/tests/test_episode_tei_tracing.py new file mode 100644 index 00000000..12fd9755 --- /dev/null +++ b/tests/test_episode_tei_tracing.py @@ -0,0 +1,46 @@ +"""Integration coverage for episode-TEI retrieval tracing.""" + +import dataclasses as dc +import typing as typ + +import httpx +import pytest + +from episodic.api import create_app +from episodic.observability import RecordingTracer +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import HeaderPrincipalAuthorization + +if typ.TYPE_CHECKING: + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_episode_tei_trace_records_invalid_identifier( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Record invalid TEI reads without including the raw route value.""" + tracer = RecordingTracer() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + tracer=tracer, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + response = await client.get( + "/v1/episodes/not-a-uuid/tei", + headers={"Authorization": "Bearer principal-a"}, + ) + + assert response.status_code == 400, response.text + assert tracer.spans[0].name == "episode_tei.read", tracer.spans + assert tracer.spans[0].attributes == { + "operation": "episode_tei.read", + "outcome": "rejected", + "failure_category": "invalid_input", + }, tracer.spans diff --git a/tests/test_generation_persistence.py b/tests/test_generation_persistence.py new file mode 100644 index 00000000..d4d3ae30 --- /dev/null +++ b/tests/test_generation_persistence.py @@ -0,0 +1,389 @@ +"""Tests for draft-generation persistence services.""" + +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import ( + GenerationRun, + GenerationRunStatus, + IngestionJob, + IngestionStatus, + IntakeState, + SeriesProfile, +) +from episodic.canonical.generation_persistence import ( + DraftScriptPersistenceRequest, + EpisodeMaterialisationRequest, + InvalidDraftTeiError, + SourceCountLimitExceededError, + materialise_episode_from_ingestion, + persist_draft_script, +) +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.ingestion_sources import AttachmentKind, IngestionJobSource +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.generation.draft_script import DraftScriptResult +from episodic.llm import LLMUsage + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +NOW = dt.datetime(2026, 6, 24, 12, 0, tzinfo=dt.UTC) + + +class SequentialUuids: + """Deterministic UUID factory for persistence service tests.""" + + def __init__(self) -> None: + self.values = iter([ + uuid.UUID("00000000-0000-0000-0000-000000000101"), + uuid.UUID("00000000-0000-0000-0000-000000000102"), + uuid.UUID("00000000-0000-0000-0000-000000000103"), + uuid.UUID("00000000-0000-0000-0000-000000000104"), + ]) + + def __call__(self) -> uuid.UUID: + """Return the next deterministic UUID.""" + return next(self.values) + + +def _clock() -> dt.datetime: + """Return the frozen persistence timestamp.""" + return NOW + + +async def _persist_materialisation_input( + factory: async_sessionmaker[AsyncSession], + job: IngestionJob, + *, + source: IngestionJobSource | None = None, +) -> None: + """Persist one ingestion job and its optional attached source.""" + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.series_profiles.add(_series_profile()) + await uow.flush() + await uow.ingestion_jobs.add(job) + if source is not None: + await uow.ingestion_job_sources.add(source) + await uow.commit() + + +@pytest.mark.asyncio +async def test_materialisation_rejects_sources_beyond_configured_limit( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Read only the configured source limit plus one before taking the job lock.""" + job = _ingestion_job(_series_profile().id, None) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.series_profiles.add(_series_profile()) + await uow.flush() + await uow.ingestion_jobs.add(job) + for _index in range(33): + await uow.ingestion_job_sources.add( + dc.replace(_source(job.id), id=uuid.uuid7()) + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(SourceCountLimitExceededError) as raised: + await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + max_source_count=32, + ), + ) + + assert raised.value.ingestion_job_id == job.id, raised.value + + +def _series_profile() -> SeriesProfile: + """Return a series profile fixture.""" + return SeriesProfile( + id=uuid.UUID("00000000-0000-0000-0000-000000000201"), + slug="bridgewater", + title="Bridgewater", + description=None, + configuration={}, + guardrails={}, + created_at=NOW, + updated_at=NOW, + ) + + +def _ingestion_job( + series_profile_id: uuid.UUID, + episode_id: uuid.UUID | None, + *, + intake_state: IntakeState = IntakeState.READY_FOR_GENERATION, +) -> IngestionJob: + """Return an intake job ready for generation.""" + return IngestionJob( + id=uuid.UUID("00000000-0000-0000-0000-000000000301"), + series_profile_id=series_profile_id, + target_episode_id=episode_id, + status=IngestionStatus.PENDING, + requested_at=NOW, + started_at=None, + completed_at=None, + error_message=None, + created_at=NOW, + updated_at=NOW, + intake_state=intake_state, + ) + + +def _source(job_id: uuid.UUID) -> IngestionJobSource: + """Return one attached source for a ready ingestion job.""" + return IngestionJobSource( + id=uuid.UUID("00000000-0000-0000-0000-000000000401"), + ingestion_job_id=job_id, + attachment_kind=AttachmentKind.SOURCE_URI, + upload_id=None, + source_uri="https://example.test/source.md", + source_type="research_brief", + weight=1.0, + metadata={"content_hash": "sha256:source"}, + created_at=NOW, + ) + + +def _run(episode_id: uuid.UUID, source_bundle_id: uuid.UUID) -> GenerationRun: + """Return a pending no-QA generation run for persistence tests.""" + return GenerationRun( + id=uuid.UUID("00000000-0000-0000-0000-000000000501"), + episode_id=episode_id, + source_bundle_id=source_bundle_id, + actor="editor@example.test", + status=GenerationRunStatus.RUNNING, + current_node="draft", + budget_snapshot={}, + configuration={}, + created_at=NOW, + updated_at=NOW, + started_at=NOW, + ended_at=None, + error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="Vertical slice test.", + ) + + +def _draft_result(tei_xml: str) -> DraftScriptResult: + """Return a draft script result for persistence tests.""" + return DraftScriptResult( + tei_xml=tei_xml, + content_hash=f"sha256:{hashlib.sha256(tei_xml.encode()).hexdigest()}", + usage=LLMUsage(input_tokens=10, output_tokens=20, total_tokens=30), + model="vidai-mock", + provider_response_id="resp-1", + finish_reason="stop", + ) + + +async def _persist_ready_job( + factory: async_sessionmaker[AsyncSession], +) -> tuple[SeriesProfile, IngestionJob]: + """Persist a ready intake job with one source attachment.""" + series = _series_profile() + job = _ingestion_job(series.id, None) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.series_profiles.add(series) + await uow.flush() + await uow.ingestion_jobs.add(job) + await uow.ingestion_job_sources.add(_source(job.id)) + await uow.commit() + return series, job + + +@pytest.mark.asyncio +async def test_materialise_episode_from_ingestion_creates_placeholder_episode( + session_factory: object, +) -> None: + """A ready ingestion job should materialise an episode and source rows.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + episode_id = uuid.UUID("00000000-0000-0000-0000-000000000101") + _, job = await _persist_ready_job(factory) + + async with SqlAlchemyUnitOfWork(factory) as uow: + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + await uow.commit() + + assert episode.id == episode_id, f"episode id: {episode.id}" + assert episode.title == "Bridgewater Futures", f"title: {episode.title!r}" + assert episode.tei_revision == 1, f"TEI revision: {episode.tei_revision}" + assert "Draft generation pending." in episode.tei_xml, ( + f"pending TEI: {episode.tei_xml!r}" + ) + + async with SqlAlchemyUnitOfWork(factory) as uow: + fetched = await uow.episodes.get(episode_id) + persisted_job = await uow.ingestion_jobs.get(job.id) + documents = await uow.source_documents.list_for_job(job.id) + + assert fetched is not None, f"expected persisted episode {episode_id}, got None" + assert fetched.tei_header_id == episode.tei_header_id, ( + f"expected TEI header {episode.tei_header_id}, got {fetched.tei_header_id}" + ) + assert persisted_job is not None, f"expected persisted job {job.id}" + assert persisted_job.target_episode_id == episode_id, persisted_job + assert persisted_job.updated_at == NOW, persisted_job + document_episode_ids = [document.canonical_episode_id for document in documents] + assert document_episode_ids == [episode_id], ( + f"source episodes: {document_episode_ids}" + ) + assert documents[0].content_hash == "sha256:source", ( + "expected source content hash 'sha256:source', " + f"got {documents[0].content_hash!r}" + ) + + +@pytest.mark.asyncio +async def test_materialise_episode_from_ingestion_reuses_persisted_episode( + session_factory: object, +) -> None: + """Repeated materialization should converge on the job's first episode.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + _, job = await _persist_ready_job(factory) + + async with SqlAlchemyUnitOfWork(factory) as uow: + first = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(factory) as uow: + second = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Ignored retry title", + clock=_clock, + uuid_factory=uuid.uuid4, + ), + ) + persisted_job = await uow.ingestion_jobs.get(job.id) + documents = await uow.source_documents.list_for_job(job.id) + + assert second.id == first.id, f"replayed episode id: {second.id}" + assert second.title == "Bridgewater Futures", ( + f"expected original title 'Bridgewater Futures', got {second.title!r}" + ) + assert persisted_job is not None, f"missing ingestion job {job.id}" + assert persisted_job.target_episode_id == first.id, ( + f"expected target episode {first.id}, got {persisted_job.target_episode_id}" + ) + assert len(documents) == 1, ( + "expected one source document after replay, " + f"got {len(documents)}: {documents!r}" + ) + + +@pytest.mark.asyncio +async def test_persist_draft_script_records_no_qa_revision_metadata( + session_factory: object, +) -> None: + """Persisting a generated draft should update TEI and provenance fields.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + _, job = await _persist_ready_job(factory) + draft_xml = ( + '' + "Bridgewater Futures" + 'Welcome.' + ) + + async with SqlAlchemyUnitOfWork(factory) as uow: + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + run = _run(episode.id, job.id) + await uow.generation_runs.create_run(run) + updated = await persist_draft_script( + uow, + DraftScriptPersistenceRequest( + episode_id=episode.id, + generation_run_id=run.id, + result=_draft_result(draft_xml), + expected_revision=episode.tei_revision, + clock=_clock, + ), + ) + await uow.commit() + + assert updated.tei_xml == draft_xml, ( + f"expected persisted draft XML {draft_xml!r}, got {updated.tei_xml!r}" + ) + assert updated.tei_revision == 2, f"TEI revision: {updated.tei_revision}" + expected_content_hash = _draft_result(draft_xml).content_hash + assert updated.tei_content_hash == expected_content_hash, ( + f"expected content hash {expected_content_hash!r}, " + f"got {updated.tei_content_hash!r}" + ) + assert updated.qa_status is QaStatus.SKIPPED, f"QA status: {updated.qa_status}" + assert updated.last_generation_run_id == run.id, ( + f"expected generation run id {run.id}, got {updated.last_generation_run_id}" + ) + assert updated.updated_at == NOW, f"updated_at: {updated.updated_at!r}" + + +@pytest.mark.asyncio +async def test_persist_draft_script_rejects_invalid_tei( + session_factory: object, +) -> None: + """Invalid generated TEI should become a typed persistence failure.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + _, job = await _persist_ready_job(factory) + + async with SqlAlchemyUnitOfWork(factory) as uow: + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + run = _run(episode.id, job.id) + await uow.generation_runs.create_run(run) + with pytest.raises(InvalidDraftTeiError): + await persist_draft_script( + uow, + DraftScriptPersistenceRequest( + episode_id=episode.id, + generation_run_id=run.id, + result=_draft_result("broken"), + expected_revision=episode.tei_revision, + clock=_clock, + ), + ) diff --git a/tests/test_generation_persistence_failures.py b/tests/test_generation_persistence_failures.py new file mode 100644 index 00000000..e85b09fc --- /dev/null +++ b/tests/test_generation_persistence_failures.py @@ -0,0 +1,351 @@ +"""Typed failure tests for draft-generation persistence services.""" + +import dataclasses as dc +import typing as typ +import uuid +from unittest import mock + +import pytest +from sqlalchemy.exc import IntegrityError + +from episodic.canonical.domain import IntakeState +from episodic.canonical.generation_persistence import ( + DraftContentHashMismatchError, + DraftScriptPersistenceRequest, + EpisodeMaterialisationRequest, + GenerationSourceUploadNotFoundError, + IngestionJobNotReadyError, + MissingAttachedSourcesError, + SourceDocumentProjectionError, + _upload_for_source, + materialise_episode_from_ingestion, + persist_draft_script, +) +from episodic.canonical.ingestion_sources import AttachmentKind +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.uploads import Upload, UploadState +from tests.test_generation_persistence import ( + SequentialUuids, + _clock, + _draft_result, + _ingestion_job, + _persist_materialisation_input, + _persist_ready_job, + _run, + _series_profile, + _source, +) + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.domain import SourceDocument + from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + + +class _MissingUploadRepository: + """Return no upload for the focused source-resolution failure path.""" + + async def get(self, upload_id: uuid.UUID) -> None: + """Report the requested upload as absent.""" + del upload_id + + +class _MissingUploadUnitOfWork: + """Provide the only repository needed by ``_upload_for_source``.""" + + uploads = _MissingUploadRepository() + + +def test_source_document_projection_error_retains_missing_document_ids() -> None: + """Projection failures should retain immutable missing-row diagnostics.""" + first_id = uuid.UUID("00000000-0000-0000-0000-000000000001") + second_id = uuid.UUID("00000000-0000-0000-0000-000000000002") + + error = SourceDocumentProjectionError({second_id, first_id}) + + assert error.missing_document_ids == (first_id, second_id), ( + f"missing IDs: {error.missing_document_ids!r}" + ) + assert str(first_id) in str(error), f"projection error: {error!s}" + assert str(second_id) in str(error), f"projection error: {error!s}" + + +async def _materialise( + factory: async_sessionmaker[AsyncSession], + ingestion_job_id: uuid.UUID, +) -> object: + """Materialize one episode using deterministic test seams.""" + async with SqlAlchemyUnitOfWork(factory) as uow: + return await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=ingestion_job_id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + + +@pytest.mark.asyncio +async def test_materialise_episode_requires_attached_sources( + session_factory: object, +) -> None: + """A source-free job raises the source-specific persistence error.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + job = _ingestion_job(_series_profile().id, None) + + await _persist_materialisation_input(factory, job) + with pytest.raises(MissingAttachedSourcesError, match="sources") as raised: + await _materialise(factory, job.id) + + assert raised.value.ingestion_job_id == job.id, ( + f"source-free job id: {raised.value.ingestion_job_id}" + ) + + +@pytest.mark.asyncio +async def test_materialise_episode_requires_ready_job_before_sources( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """A non-ready source-free job reports the readiness failure first.""" + job = _ingestion_job( + _series_profile().id, + None, + intake_state=IntakeState.AWAITING_SOURCES, + ) + + await _persist_materialisation_input(session_factory, job) + with pytest.raises(IngestionJobNotReadyError, match="not ready") as raised: + await _materialise(session_factory, job.id) + + assert raised.value.ingestion_job_id == job.id, ( + f"source-free non-ready job id: {raised.value.ingestion_job_id}" + ) + + +@pytest.mark.asyncio +async def test_materialise_episode_requires_ready_ingestion_job( + session_factory: object, +) -> None: + """A non-ready job raises the readiness-specific persistence error.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + job = _ingestion_job( + _series_profile().id, + None, + intake_state=IntakeState.AWAITING_SOURCES, + ) + + await _persist_materialisation_input(factory, job, source=_source(job.id)) + with pytest.raises(IngestionJobNotReadyError, match="not ready") as raised: + await _materialise(factory, job.id) + + assert raised.value.ingestion_job_id == job.id, ( + f"non-ready job id: {raised.value.ingestion_job_id}" + ) + + +@pytest.mark.asyncio +async def test_source_upload_resolution_rejects_missing_upload() -> None: + """A missing source upload raises a typed source-resolution error.""" + job = _ingestion_job(_series_profile().id, None) + upload_id = uuid.UUID("00000000-0000-0000-0000-000000000601") + source = dc.replace( + _source(job.id), + attachment_kind=AttachmentKind.UPLOAD, + upload_id=upload_id, + source_uri=None, + ) + + with pytest.raises(GenerationSourceUploadNotFoundError) as raised: + await _upload_for_source( + typ.cast("CanonicalUnitOfWork", _MissingUploadUnitOfWork()), + source, + ) + + assert raised.value.upload_id == upload_id, ( + f"missing upload id: {raised.value.upload_id}" + ) + + +@pytest.mark.asyncio +async def test_materialise_rolls_back_reservation_for_missing_upload( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A missing upload does not leave a reusable placeholder episode behind.""" + job = _ingestion_job(_series_profile().id, None) + upload_id = uuid.UUID("00000000-0000-0000-0000-000000000601") + source = dc.replace( + _source(job.id), + attachment_kind=AttachmentKind.UPLOAD, + upload_id=upload_id, + source_uri=None, + ) + upload = Upload( + id=upload_id, + owner_principal_id=None, + content_type="text/plain", + declared_size=1, + actual_size=1, + declared_sha256=None, + content_hash="sha256:upload", + storage_key="uploads/missing-after-reservation", + state=UploadState.READY, + metadata={}, + created_at=_clock(), + updated_at=_clock(), + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.series_profiles.add(_series_profile()) + await uow.flush() + await uow.ingestion_jobs.add(job) + await uow.flush() + await uow.uploads.add(upload) + await uow.flush() + await uow.ingestion_job_sources.add(source) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + missing_upload = mock.AsyncMock(return_value=None) + monkeypatch.setattr(uow.uploads, "get", missing_upload) + with pytest.raises(GenerationSourceUploadNotFoundError): + await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + persisted_job = await uow.ingestion_jobs.get(job.id) + episode = await uow.episodes.get( + uuid.UUID("00000000-0000-0000-0000-000000000101") + ) + header = await uow.tei_headers.get( + uuid.UUID("00000000-0000-0000-0000-000000000102") + ) + documents = await uow.source_documents.list_for_job(job.id) + + assert persisted_job is not None, f"missing ingestion job {job.id}" + assert persisted_job.target_episode_id is None, persisted_job + assert episode is None, f"unexpected placeholder episode: {episode}" + assert header is None, f"unexpected placeholder header: {header}" + assert documents == [], f"unexpected projections: {documents}" + + +@pytest.mark.asyncio +async def test_persist_draft_script_rejects_mismatched_content_hash( + session_factory: object, +) -> None: + """Generated TEI requires a matching declared content hash.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + _, job = await _persist_ready_job(factory) + tei_xml = ( + '' + "Bridgewater Futures" + 'Welcome.' + ) + + async with SqlAlchemyUnitOfWork(factory) as uow: + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + run = _run(episode.id, job.id) + await uow.generation_runs.create_run(run) + result = dc.replace(_draft_result(tei_xml), content_hash="sha256:wrong") + with pytest.raises(DraftContentHashMismatchError) as raised: + await persist_draft_script( + uow, + DraftScriptPersistenceRequest( + episode_id=episode.id, + generation_run_id=run.id, + result=result, + expected_revision=episode.tei_revision, + clock=_clock, + ), + ) + + assert raised.value.expected_hash != raised.value.actual_hash, ( + f"hashes: {raised.value.expected_hash!r}, {raised.value.actual_hash!r}" + ) + + +@pytest.mark.asyncio +async def test_materialise_reraises_unrelated_projection_integrity_error( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only a source-document duplicate may be recovered as a retry race.""" + _, job = await _persist_ready_job(session_factory) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + error = ValueError("bad FK") + fail_projection = mock.AsyncMock(side_effect=IntegrityError("", {}, error)) + monkeypatch.setattr(uow.source_documents, "add_projection", fail_projection) + with pytest.raises(IntegrityError, match="bad FK"): + await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + + +@pytest.mark.asyncio +async def test_materialise_verifies_duplicate_projection_rows( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A duplicate race succeeds only after all deterministic rows are found.""" + _, job = await _persist_ready_job(session_factory) + request = EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + first = await materialise_episode_from_ingestion(uow, request) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + original_list = uow.source_documents.list_for_job + list_count = 0 + + async def hide_existing_projection( + job_id: uuid.UUID, + ) -> cabc.Sequence[SourceDocument]: + """Simulate a concurrent transaction's stale pre-insert read.""" + nonlocal list_count + list_count += 1 + if list_count == 1: + return () + return await original_list(job_id) + + monkeypatch.setattr( + uow.source_documents, + "list_for_job", + hide_existing_projection, + ) + second = await materialise_episode_from_ingestion(uow, request) + + assert second.id == first.id, f"duplicate projection episode: {second.id}" + assert list_count == 2, ( + f"expected projection and verification reads, got {list_count}" + ) diff --git a/tests/test_generation_persistence_locking.py b/tests/test_generation_persistence_locking.py new file mode 100644 index 00000000..f5c277c5 --- /dev/null +++ b/tests/test_generation_persistence_locking.py @@ -0,0 +1,138 @@ +"""Concurrency-boundary tests for generation materialization.""" + +import asyncio +import typing as typ + +import pytest +import sqlalchemy as sa + +from episodic.canonical.generation_persistence import ( + EpisodeMaterialisationRequest, + materialise_episode_from_ingestion, +) +from episodic.canonical.storage import ( + EpisodeRecord, + IngestionJobRecord, + SourceDocumentRecord, + SqlAlchemyUnitOfWork, + TeiHeaderRecord, +) +from tests.test_generation_persistence import ( + SequentialUuids, + _clock, + _persist_ready_job, +) + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.domain import IngestionJob + from episodic.canonical.ingestion_sources import IngestionJobSource + + +@pytest.mark.asyncio +async def test_materialisation_releases_job_lock_before_source_work( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Source paging finishes before the materializer locks the job row.""" + _, job = await _persist_ready_job(session_factory) + source_page_loaded = False + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + original_list = uow.ingestion_job_sources.list_for_job_paged + original_get_for_update = uow.ingestion_jobs.get_for_update + + async def list_sources_before_lock( + ingestion_job_id: uuid.UUID, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJobSource]: + """Record source work performed outside the ingestion-job lock.""" + nonlocal source_page_loaded + source_page_loaded = True + return await original_list(ingestion_job_id, limit=limit, offset=offset) + + async def lock_after_source_paging( + ingestion_job_id: uuid.UUID, + ) -> IngestionJob | None: + """Verify the short reservation lock follows source paging.""" + assert source_page_loaded, "ingestion-job lock preceded source paging" + return await original_get_for_update(ingestion_job_id) + + monkeypatch.setattr( + uow.ingestion_job_sources, + "list_for_job_paged", + list_sources_before_lock, + ) + monkeypatch.setattr( + uow.ingestion_jobs, + "get_for_update", + lock_after_source_paging, + ) + await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + + +@pytest.mark.asyncio +async def test_materialisation_converges_under_two_session_contention( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Concurrent materialisation keeps one durable episode and source projection.""" + _, job = await _persist_ready_job(session_factory) + barrier = asyncio.Barrier(2) + + async def materialise_in_independent_unit_of_work() -> uuid.UUID: + """Start one materialisation attempt concurrently with its peer.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await barrier.wait() + episode = await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=job.id, + title="Bridgewater Futures", + clock=_clock, + uuid_factory=SequentialUuids(), + ), + ) + return episode.id + + episode_ids = await asyncio.gather( + materialise_in_independent_unit_of_work(), + materialise_in_independent_unit_of_work(), + ) + episode_id_set = set(episode_ids) + assert len(episode_id_set) == 1, f"materialised episode ids: {episode_ids!r}" + expected_episode_id = episode_id_set.pop() + + async with session_factory() as session: + persisted_job = await session.scalar( + sa.select(IngestionJobRecord).where(IngestionJobRecord.id == job.id) + ) + episode_count = await session.scalar(sa.select(sa.func.count(EpisodeRecord.id))) + header_count = await session.scalar( + sa.select(sa.func.count(TeiHeaderRecord.id)) + ) + document_count = await session.scalar( + sa.select(sa.func.count(SourceDocumentRecord.id)) + ) + + assert persisted_job is not None, f"ingestion job not found: {job.id}" + assert persisted_job.target_episode_id == expected_episode_id, ( + f"target episode id: {persisted_job.target_episode_id}; " + f"expected: {expected_episode_id}" + ) + assert episode_count == 1, f"episode count: {episode_count}" + assert header_count == 1, f"TEI header count: {header_count}" + assert document_count == 1, f"source document count: {document_count}" diff --git a/tests/test_generation_persistence_not_found.py b/tests/test_generation_persistence_not_found.py new file mode 100644 index 00000000..160028a5 --- /dev/null +++ b/tests/test_generation_persistence_not_found.py @@ -0,0 +1,37 @@ +"""Not-found contract tests for draft-generation persistence.""" + +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.generation_persistence import ( + EpisodeMaterialisationRequest, + materialise_episode_from_ingestion, +) +from episodic.canonical.source_intake_errors import IngestionJobNotFoundError +from episodic.canonical.storage import SqlAlchemyUnitOfWork + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_materialise_episode_from_ingestion_rejects_unknown_job( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Unknown jobs should retain the source-intake not-found contract.""" + unknown_job_id = uuid.uuid7() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + with pytest.raises(IngestionJobNotFoundError, match=str(unknown_job_id)): + await materialise_episode_from_ingestion( + uow, + EpisodeMaterialisationRequest( + ingestion_job_id=unknown_job_id, + title="Unknown job", + clock=lambda: dt.datetime(2026, 6, 24, 12, 0, tzinfo=dt.UTC), + uuid_factory=uuid.uuid7, + ), + ) diff --git a/tests/test_generation_run_api.py b/tests/test_generation_run_api.py new file mode 100644 index 00000000..53daf246 --- /dev/null +++ b/tests/test_generation_run_api.py @@ -0,0 +1,375 @@ +"""Integration tests for generation-run REST resources.""" + +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import httpx +import pytest + +from episodic.api import create_app +from episodic.observability import RecordingTracer +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import ( + HeaderPrincipalAuthorization, + RecordingLauncher, + create_ready_ingestion_job, + generation_payload, +) + +if typ.TYPE_CHECKING: + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.api.types import UowFactory + + +async def _append_generation_events( + uow_factory: UowFactory, + run_id: uuid.UUID, +) -> None: + """Persist events needed to verify event pagination.""" + async with uow_factory() as uow: + for kind in ("run.started", "draft.generated"): + await uow.generation_runs.append_event( + run_id, + kind=kind, + payload={"kind": kind}, + occurred_at=dt.datetime(2026, 7, 22, tzinfo=dt.UTC), + ) + await uow.commit() + + +def _assert_generation_event_page(response: httpx.Response) -> None: + """Assert the cursor-filtered generation event page contract.""" + assert response.status_code == 200, response.text + assert [event["kind"] for event in response.json()["items"]] == [ + "draft.generated" + ], f"expected one draft.generated event, got {response.json()['items']!r}" + assert response.json()["after_seq"] == 1, ( + f"expected after_seq 1, got {response.json()['after_seq']!r}" + ) + assert response.json()["offset"] == 0, ( + f"expected offset 0, got {response.json()['offset']!r}" + ) + assert response.json()["total"] == 1, ( + f"expected one matching event, got {response.json()['total']!r}" + ) + + +class _ExpectedError(typ.NamedTuple): + """Describe one stable REST error response.""" + + status: int + code: str + message: str + details: dict[str, str] + + +def _assert_error_envelope( + response: httpx.Response, + expected: _ExpectedError, +) -> None: + """Assert one complete API error response contract.""" + assert response.status_code == expected.status, response.text + payload = response.json() + assert set(payload) == {"code", "message", "details"}, payload + assert payload["code"] == expected.code, payload + assert payload["message"] == expected.message, payload + assert payload["details"] == expected.details, payload + + +def _assert_generation_run_replay( + first: httpx.Response, + replay: httpx.Response, + launcher: RecordingLauncher, +) -> None: + """Assert idempotent replay preserves the accepted response contract.""" + assert first.status_code == 202, first.text + assert replay.status_code == 202, replay.text + assert replay.json() == first.json(), ( + f"expected replay payload {first.json()!r}, got {replay.json()!r}" + ) + assert replay.headers["Location"] == first.headers["Location"], ( + f"expected replay location {first.headers['Location']!r}, " + f"got {replay.headers['Location']!r}" + ) + assert replay.headers["Retry-After"] == first.headers["Retry-After"], ( + f"expected retry delay {first.headers['Retry-After']!r}, " + f"got {replay.headers['Retry-After']!r}" + ) + assert len(launcher.run_ids) == 1, ( + f"expected one launched run, got {launcher.run_ids!r}" + ) + + +def _assert_polled_generation_run( + response: httpx.Response, + payload: dict[str, object], +) -> None: + """Assert the pending generation-run polling response contract.""" + assert response.status_code == 200, response.text + assert response.json()["qa_status"] == "skipped", ( + f"expected skipped QA status, got {response.json()['qa_status']!r}" + ) + assert response.json()["skip_qa_rationale"] == payload["skip_qa_rationale"], ( + f"expected rationale {payload['skip_qa_rationale']!r}, " + f"got {response.json()['skip_qa_rationale']!r}" + ) + assert response.headers["Retry-After"] == "1", ( + f"expected retry delay '1', got {response.headers['Retry-After']!r}" + ) + + +def _assert_generation_run_conflict(response: httpx.Response) -> None: + """Assert an idempotency conflict retains the original record identifier.""" + assert response.status_code == 409, response.text + payload = response.json() + assert set(payload) == {"code", "message", "details"}, payload + assert payload["code"] == "idempotency_conflict", payload + assert payload["message"] == "Idempotency key body mismatch.", payload + changed_details = payload["details"] + assert isinstance(changed_details, dict), changed_details + record_id = changed_details.get("record_id") + assert isinstance(record_id, str), changed_details + uuid.UUID(record_id) + _assert_error_envelope( + response, + _ExpectedError( + status=409, + code="idempotency_conflict", + message="Idempotency key body mismatch.", + details={"record_id": record_id}, + ), + ) + + +@pytest.mark.asyncio +async def test_generation_run_create_replay_and_poll( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Create once, replay response metadata, and poll the stored run.""" + launcher = RecordingLauncher() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + tracer=RecordingTracer(), + ) + headers = {"Authorization": "Bearer principal-a"} + async with httpx.AsyncClient( + transport=httpx.ASGITransport( + app=typ.cast("_ASGIApp", create_app(dependencies)) + ), + base_url="http://testserver", + ) as client: + ingestion_job_id = await create_ready_ingestion_job(client, headers) + payload = generation_payload() + first = await client.post( + f"/v1/ingestion-jobs/{ingestion_job_id}/generation-runs", + headers={**headers, "Idempotency-Key": "generation-key"}, + json=payload, + ) + replay = await client.post( + f"/v1/ingestion-jobs/{ingestion_job_id}/generation-runs", + headers={**headers, "Idempotency-Key": "generation-key"}, + json=payload, + ) + run_id = launcher.run_ids[0] + await _append_generation_events(dependencies.uow_factory, run_id) + run_response = await client.get( + first.headers.get("Location", "/missing"), headers=headers + ) + events_response = await client.get( + f"/v1/generation-runs/{run_id}/events", + headers=headers, + params={"after_seq": 1, "limit": 1}, + ) + + _assert_generation_run_replay(first, replay, launcher) + _assert_polled_generation_run(run_response, payload) + _assert_generation_event_page(events_response) + _assert_trace_span( + typ.cast("RecordingTracer", dependencies.tracer), + "generation_run.read", + {"operation": "generation_run.read", "outcome": "success"}, + ) + _assert_trace_span( + typ.cast("RecordingTracer", dependencies.tracer), + "generation_run.events.list", + { + "operation": "generation_run.events.list", + "pagination": "cursor", + "outcome": "success", + }, + ) + + +@pytest.mark.asyncio +async def test_generation_run_get_routes_trace_rejected_and_missing_requests( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Record bounded read-route outcomes without request data or identifiers.""" + tracer = RecordingTracer() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + tracer=tracer, + ) + headers = {"Authorization": "Bearer principal-a"} + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + invalid = await client.get("/v1/generation-runs/not-a-uuid", headers=headers) + missing = await client.get( + f"/v1/generation-runs/{uuid.uuid7()}", headers=headers + ) + invalid_events = await client.get( + f"/v1/generation-runs/{uuid.uuid7()}/events", + headers=headers, + params={"after_seq": "1", "offset": "1"}, + ) + + assert invalid.status_code == 400, invalid.text + assert missing.status_code == 404, missing.text + assert invalid_events.status_code == 400, invalid_events.text + actual_spans = [span.attributes for span in tracer.spans] + expected_spans = [ + { + "operation": "generation_run.read", + "outcome": "rejected", + "failure_category": "invalid_input", + }, + { + "operation": "generation_run.read", + "outcome": "not_found", + "failure_category": "run.not_found", + }, + { + "operation": "generation_run.events.list", + "outcome": "rejected", + "failure_category": "invalid_input", + }, + ] + assert actual_spans == expected_spans, tracer.spans + + +@pytest.mark.asyncio +async def test_generation_run_resources_enforce_principal_ownership( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Derive run actors from the authenticated principal and hide other runs.""" + launcher = RecordingLauncher() + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + principal_a = {"Authorization": "Bearer principal-a"} + principal_b = {"Authorization": "Bearer principal-b"} + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + ingestion_job_id = await create_ready_ingestion_job(client, principal_a) + response = await client.post( + f"/v1/ingestion-jobs/{ingestion_job_id}/generation-runs", + headers={**principal_a, "Idempotency-Key": "principal-a-run"}, + json={**generation_payload(), "actor": "spoofed@example.test"}, + ) + assert response.status_code == 202, response.text + assert response.json()["actor"] == "principal-a", response.json() + run_location = response.headers.get("Location", "/missing") + response = await client.get(run_location, headers=principal_a) + assert response.status_code == 200, response.text + response = await client.get(run_location) + assert response.status_code == 401, response.text + response = await client.post( + f"/v1/ingestion-jobs/{ingestion_job_id}/generation-runs", + headers={**principal_b, "Idempotency-Key": "principal-b-run"}, + json=generation_payload(), + ) + assert response.status_code == 404, response.text + response = await client.get(run_location, headers=principal_b) + assert response.status_code == 404, response.text + response = await client.get( + f"/v1/generation-runs/{launcher.run_ids[0]}/events", + headers=principal_b, + ) + assert response.status_code == 404, response.text + + +def _assert_trace_span( + tracer: RecordingTracer, + name: str, + attributes: dict[str, str], +) -> None: + """Assert one completed trace span with the safe bounded attributes.""" + span = next(record for record in tracer.spans if record.name == name) + assert span.attributes == attributes, span + assert span.is_completed, span + + +@pytest.mark.asyncio +async def test_generation_run_validation_and_idempotency_conflict( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Reject invalid quality metadata and changed idempotent bodies.""" + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=RecordingLauncher(), + ) + headers = {"Authorization": "Bearer principal-a"} + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + ingestion_job_id = await create_ready_ingestion_job(client, headers) + endpoint = f"/v1/ingestion-jobs/{ingestion_job_id}/generation-runs" + accepted = await client.post( + endpoint, + headers={**headers, "Idempotency-Key": "conflict-key"}, + json=generation_payload(), + ) + changed = await client.post( + endpoint, + headers={**headers, "Idempotency-Key": "conflict-key"}, + json={**generation_payload(), "skip_qa_rationale": "Changed."}, + ) + missing_rationale = await client.post( + endpoint, + headers={**headers, "Idempotency-Key": "missing-key"}, + json={"quality_mode": "draft_without_qa", "actor": "editor"}, + ) + unsupported_mode = await client.post( + endpoint, + headers={**headers, "Idempotency-Key": "mode-key"}, + json={**generation_payload(), "quality_mode": "qa_gated"}, + ) + + assert accepted.status_code == 202, accepted.text + _assert_generation_run_conflict(changed) + _assert_error_envelope( + missing_rationale, + _ExpectedError( + status=400, + code="validation_error", + message="Missing required field: skip_qa_rationale", + details={"field": "skip_qa_rationale", "constraint": "required"}, + ), + ) + _assert_error_envelope( + unsupported_mode, + _ExpectedError( + status=422, + code="quality_mode_unsupported", + message="Unsupported quality_mode: qa_gated.", + details={"quality_mode": "qa_gated"}, + ), + ) diff --git a/tests/test_generation_run_api_admission.py b/tests/test_generation_run_api_admission.py new file mode 100644 index 00000000..8165c7ff --- /dev/null +++ b/tests/test_generation_run_api_admission.py @@ -0,0 +1,133 @@ +"""Generation-run API admission failure integration tests.""" + +import dataclasses as dc +import typing as typ +import uuid + +import httpx +import pytest + +from episodic.api import create_app +from episodic.canonical.domain import GenerationRunStatus +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.generation import GenerationRunAdmissionError +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import ( + HeaderPrincipalAuthorization, + RecordingLauncher, + create_ready_ingestion_job, + generation_payload, +) + +if typ.TYPE_CHECKING: + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +async def _post_generation_run( + client: httpx.AsyncClient, + job_id: str, +) -> httpx.Response: + """Submit one authenticated generation request for a ready job.""" + return await client.post( + f"/v1/ingestion-jobs/{job_id}/generation-runs", + headers={ + "Authorization": "Bearer principal-a", + "Idempotency-Key": "admission-key", + }, + json=generation_payload(), + ) + + +@pytest.mark.asyncio +async def test_generation_run_admission_failure_marks_persisted_run_failed( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Capacity rejection records one terminal run before returning HTTP 503.""" + launcher = RecordingLauncher() + launcher.launch.side_effect = GenerationRunAdmissionError("capacity exhausted") + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + headers = {"Authorization": "Bearer principal-a"} + job_id = await create_ready_ingestion_job(client, headers) + response = await _post_generation_run(client, job_id) + + assert response.status_code == 503, response.text + assert response.json()["code"] == "generation_overloaded", response.text + assert launcher.launch.await_count == 1, launcher.launch.await_args_list + async with SqlAlchemyUnitOfWork(session_factory) as uow: + job = await uow.ingestion_jobs.get(uuid.UUID(job_id)) + assert job is not None, f"missing job {job_id}" + assert job.target_episode_id is not None, job + runs = await uow.generation_runs.list_runs(job.target_episode_id) + assert len(runs) == 1, runs + assert runs[0].status is GenerationRunStatus.FAILED, runs[0] + assert runs[0].error_category == "launcher.overloaded", runs[0] + + +@pytest.mark.asyncio +async def test_generation_run_scheduling_failure_marks_persisted_run_failed( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Unexpected launcher failures leave the durable run terminal.""" + launcher = RecordingLauncher() + launcher.launch.side_effect = RuntimeError("task allocation failed") + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=launcher, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + headers = {"Authorization": "Bearer principal-a"} + job_id = await create_ready_ingestion_job(client, headers) + response = await _post_generation_run(client, job_id) + + assert response.status_code == 500, response.text + assert launcher.launch.await_count == 1, launcher.launch.await_args_list + async with SqlAlchemyUnitOfWork(session_factory) as uow: + job = await uow.ingestion_jobs.get(uuid.UUID(job_id)) + assert job is not None, f"missing job {job_id}" + assert job.target_episode_id is not None, job + runs = await uow.generation_runs.list_runs(job.target_episode_id) + assert len(runs) == 1, runs + assert runs[0].status is GenerationRunStatus.FAILED, runs[0] + assert runs[0].error_category == "launcher.scheduling", runs[0] + + +@pytest.mark.asyncio +async def test_generation_run_rejects_requests_without_a_launcher( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """An unavailable launcher rejects creation before materialisation begins.""" + dependencies = dc.replace( + build_api_dependencies(session_factory), + authorization=HeaderPrincipalAuthorization(), + launcher=None, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + headers = {"Authorization": "Bearer principal-a"} + job_id = await create_ready_ingestion_job(client, headers) + response = await _post_generation_run(client, job_id) + + assert response.status_code == 503, response.text + assert response.json()["code"] == "service_unavailable", response.text + async with SqlAlchemyUnitOfWork(session_factory) as uow: + job = await uow.ingestion_jobs.get(uuid.UUID(job_id)) + assert job is not None, f"missing job {job_id}" + assert job.target_episode_id is None, job diff --git a/tests/test_generation_run_claim_hydration.py b/tests/test_generation_run_claim_hydration.py new file mode 100644 index 00000000..021b4d6b --- /dev/null +++ b/tests/test_generation_run_claim_hydration.py @@ -0,0 +1,96 @@ +"""Transaction-boundary tests for generation-run claim hydration.""" + +import asyncio +import contextlib +import datetime as dt +import typing as typ + +import pytest + +import episodic.generation.launcher as launcher_module +from episodic.canonical.domain import GenerationRunStatus, SourceDocument +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from tests.generation_run_launcher_support import ( + RecordingDraftGenerator, + draft_result, + launcher, + prepare_pending_run, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.object_store import ObjectStorePort + from episodic.generation.launcher_support import GenerationSourceLimits + + +@pytest.mark.asyncio +async def test_claim_commits_before_blocked_hydration( + session_factory: async_sessionmaker[AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Allow another unit of work to mutate a claimed run during hydration.""" + run_id, _ = await prepare_pending_run(session_factory) + hydration_started = asyncio.Event() + release_hydration = asyncio.Event() + + original_source_from_document = launcher_module.source_from_document + + async def block_source_loading( + document: SourceDocument, + object_store: object, + limits: object = None, + *, + remaining_aggregate_bytes: int | None = None, + ) -> object: + """Block source hydration after canonical reads have released their UOW.""" + hydration_started.set() + await release_hydration.wait() + return await original_source_from_document( + document, + typ.cast("ObjectStorePort | None", object_store), + typ.cast("GenerationSourceLimits | None", limits), + remaining_aggregate_bytes=remaining_aggregate_bytes, + ) + + monkeypatch.setattr(launcher_module, "source_from_document", block_source_loading) + run_launcher = launcher( + session_factory, + RecordingDraftGenerator(draft_result("")), + ) + claim_task = asyncio.create_task(run_launcher._claim(run_id)) + try: + await asyncio.wait_for(hydration_started.wait(), timeout=1) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + claimed_run = await uow.generation_runs.get_run(run_id) + assert claimed_run is not None, f"expected claimed run {run_id} to persist" + assert claimed_run.status is GenerationRunStatus.RUNNING, ( + f"expected run {run_id} to be running before hydration, " + f"got {claimed_run.status.value}" + ) + await uow.generation_runs.append_event( + run_id, + kind="hydration.observed", + payload={}, + occurred_at=dt.datetime(2026, 8, 20, tzinfo=dt.UTC), + ) + await uow.commit() + + release_hydration.set() + claimed = await claim_task + finally: + if not release_hydration.is_set(): + release_hydration.set() + if not claim_task.done(): + claim_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await claim_task + + assert claimed is not None, f"expected claimed run {run_id} after hydration" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + events = await uow.generation_runs.list_events(run_id) + assert [event.kind for event in events] == [ + "run.started", + "hydration.observed", + ], events diff --git a/tests/test_generation_run_domain.py b/tests/test_generation_run_domain.py index 7301c777..de068c10 100644 --- a/tests/test_generation_run_domain.py +++ b/tests/test_generation_run_domain.py @@ -27,6 +27,7 @@ GenerationRun, GenerationRunStatus, ) +from episodic.canonical.generation_quality import QaStatus, QualityMode from episodic.canonical.generation_run_errors import ( CheckpointAlreadyTerminal, CheckpointNotFound, @@ -62,6 +63,9 @@ def generation_run() -> GenerationRun: started_at=None, ended_at=None, error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", ) @@ -97,6 +101,67 @@ def test_generation_run_status_terminal_states_are_explicit() -> None: assert GenerationRunStatus.CANCELLED.is_terminal(), "CANCELLED must be terminal." +def test_quality_enums_pin_no_qa_slice_values() -> None: + """Quality metadata uses stable wire values for the no-QA slice.""" + assert QualityMode.DRAFT_WITHOUT_QA == "draft_without_qa", ( + f"expected draft_without_qa wire value, got {QualityMode.DRAFT_WITHOUT_QA!r}" + ) + assert QaStatus.SKIPPED == "skipped", ( + f"expected skipped wire value, got {QaStatus.SKIPPED!r}" + ) + + +def test_generation_run_records_no_qa_quality_metadata( + generation_run: GenerationRun, +) -> None: + """Draft-without-QA runs record their QA bypass rationale.""" + run = dc.replace( + generation_run, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="Initial vertical-slice draft.", + ) + + assert run.quality_mode is QualityMode.DRAFT_WITHOUT_QA, ( + f"expected draft-without-QA mode, got {run.quality_mode!r}" + ) + assert run.qa_status is QaStatus.SKIPPED, ( + f"expected skipped QA status, got {run.qa_status!r}" + ) + assert run.skip_qa_rationale == "Initial vertical-slice draft.", ( + f"expected no-QA rationale, got {run.skip_qa_rationale!r}" + ) + + +@pytest.mark.parametrize( + ("changes", "match"), + [ + pytest.param( + {"qa_status": None, "skip_qa_rationale": "Valid rationale."}, + "qa_status", + id="missing-qa-status", + ), + pytest.param( + {"qa_status": QaStatus.SKIPPED, "skip_qa_rationale": " "}, + "skip_qa_rationale", + id="blank-rationale", + ), + ], +) +def test_generation_run_rejects_incomplete_no_qa_metadata( + generation_run: GenerationRun, + changes: dict[str, object], + match: str, +) -> None: + """Draft-without-QA runs require skipped status and rationale.""" + with pytest.raises(ValueError, match=match): + dc.replace( + generation_run, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + **changes, + ) + + def test_checkpoint_status_terminal_states_are_explicit() -> None: """Only checkpoint end states should be terminal.""" assert not CheckpointStatus.CREATED.is_terminal(), "CREATED must not be terminal." @@ -268,6 +333,9 @@ def test_generation_run_and_checkpoint_repr_snapshot( started_at=None, ended_at=None, error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", ) checkpoint = Checkpoint( id=FIXED_CHECKPOINT_ID, diff --git a/tests/test_generation_run_idempotency.py b/tests/test_generation_run_idempotency.py new file mode 100644 index 00000000..4f3b138b --- /dev/null +++ b/tests/test_generation_run_idempotency.py @@ -0,0 +1,86 @@ +"""Principal-scoped generation-run idempotency tests.""" + +import typing as typ + +import pytest + +from episodic.canonical.adapters.generation_runs import InMemoryGenerationRunStore +from episodic.canonical.storage import GenerationRunRecord, SqlAlchemyUnitOfWork +from tests.canonical_storage._generation_run_support import ( + count_records, + make_generation_run, + persist_generation_run_prerequisites, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_in_memory_generation_runs_scope_keys_to_principals() -> None: + """Distinct principals may use the same generation-run idempotency key.""" + store = InMemoryGenerationRunStore() + first = await store.create_run( + make_generation_run(), + idempotency_key="run-key", + idempotency_principal_id="principal-a", + ) + second = await store.create_run( + make_generation_run(), + idempotency_key="run-key", + idempotency_principal_id="principal-b", + ) + + assert first.id != second.id, "principal scopes must not share a run" + + +@pytest.mark.asyncio +async def test_in_memory_generation_runs_reject_negative_event_limit() -> None: + """Event pagination limits must be non-negative.""" + store = InMemoryGenerationRunStore() + run = await store.create_run(make_generation_run()) + + with pytest.raises(ValueError, match="limit"): + await store.list_events(run.id, limit=-1) + + +@pytest.mark.asyncio +async def test_sql_generation_runs_scope_keys_to_principals( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """The SQL adapter persists a distinct run for each principal scope.""" + first_request = make_generation_run() + second_request = make_generation_run() + replay_request = make_generation_run() + await persist_generation_run_prerequisites( + session_factory, + first_request, + second_request, + replay_request, + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + first = await uow.generation_runs.create_run( + first_request, + idempotency_key="run-key", + idempotency_principal_id="principal-a", + ) + second = await uow.generation_runs.create_run( + second_request, + idempotency_key="run-key", + idempotency_principal_id="principal-b", + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + replayed = await uow.generation_runs.create_run( + replay_request, + idempotency_key="run-key", + idempotency_principal_id="principal-a", + ) + await uow.commit() + + assert first.id != second.id, "principal scopes must not share a run" + assert replayed == first, "same-principal replay must return the first run" + record_count = await count_records(session_factory, GenerationRunRecord) + assert record_count == 2, f"expected two generation runs, got {record_count}" diff --git a/tests/test_generation_run_launcher.py b/tests/test_generation_run_launcher.py new file mode 100644 index 00000000..6343f94a --- /dev/null +++ b/tests/test_generation_run_launcher.py @@ -0,0 +1,393 @@ +"""Tests for in-process generation-run launching.""" + +import asyncio +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import ( + GenerationEvent, + GenerationRun, + GenerationRunStatus, + SourceDocument, +) +from episodic.canonical.generation_quality import QaStatus +from episodic.canonical.storage import FilesystemObjectStore, SqlAlchemyUnitOfWork +from episodic.generation.draft_script import DraftScriptTransientProviderError +from episodic.generation.launcher import ( + GenerationRunAdmissionError, + InProcessGenerationRunLauncher, +) +from episodic.generation.launcher_support import source_from_document +from episodic.observability import RecordingTracer +from tests.generation_run_launcher_support import ( + BlockingDraftGenerator, + FailingDraftGenerator, + LauncherOptions, + RecordingCostRecorder, + RecordingDraftGenerator, + draft_result, + launcher, + prepare_pending_run, + valid_tei, +) + +if typ.TYPE_CHECKING: + import collections.abc as cabc + from pathlib import Path + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +async def _uploaded_chunks() -> cabc.AsyncIterator[bytes]: + """Yield uploaded source bytes with mixed line endings.""" + await asyncio.sleep(0) + yield b"Uploaded source line one.\r\n" + yield b"Line two.\n" + + +@dc.dataclass(slots=True) +class _RecordingMetrics: + """Capture bounded launcher metrics for assertions.""" + + counters: list[tuple[str, dict[str, str]]] = dc.field(default_factory=list) + values: list[tuple[str, float, dict[str, str]]] = dc.field(default_factory=list) + + def increment_counter(self, name: str, *, labels: cabc.Mapping[str, str]) -> None: + """Capture one counter increment.""" + self.counters.append((name, dict(labels))) + + def observe_latency_ms( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Accept latency observations outside this test's scope.""" + _ = (name, value, labels) + + def observe_value( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Capture a scalar observation.""" + self.values.append((name, value, dict(labels))) + + +async def _launch_and_load_run( + factory: async_sessionmaker[AsyncSession], + run_id: uuid.UUID, + run_launcher: InProcessGenerationRunLauncher, +) -> tuple[GenerationRun, tuple[GenerationEvent, ...]]: + """Launch a run and return its persisted terminal state and events.""" + await run_launcher.launch(run_id) + await run_launcher.drain() + async with SqlAlchemyUnitOfWork(factory) as uow: + run = await uow.generation_runs.get_run(run_id) + events = await uow.generation_runs.list_events(run_id) + assert run is not None, f"run {run_id} was not persisted; events={events!r}" + return run, events + + +@pytest.mark.asyncio +async def test_upload_backed_source_reads_object_content(tmp_path: Path) -> None: + """Upload provenance should resolve to normalized text before generation.""" + store = FilesystemObjectStore(tmp_path) + await store.put("uploads/source", _uploaded_chunks(), max_bytes=1_000) + document = SourceDocument( + id=uuid.uuid4(), + ingestion_job_id=uuid.uuid4(), + canonical_episode_id=uuid.uuid4(), + reference_document_revision_id=None, + source_type="research_brief", + source_uri="upload:uploads/source", + weight=1.0, + content_hash="sha256:source", + metadata={}, + created_at=dt.datetime(2026, 6, 24, tzinfo=dt.UTC), + ) + + source = await source_from_document(document, store) + + assert source.content == "Uploaded source line one.\nLine two.", ( + f"unexpected uploaded source content: {source.content!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_completes_run_and_records_cost( + session_factory: object, +) -> None: + """Successful launches should persist TEI, lifecycle events, and cost usage.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + run_id, episode_id = await prepare_pending_run(factory) + cost_recorder = RecordingCostRecorder() + generator = RecordingDraftGenerator(draft_result(valid_tei())) + run_launcher = launcher(factory, generator, cost_recorder) + + await run_launcher.launch(run_id) + await run_launcher.drain() + + async with SqlAlchemyUnitOfWork(factory) as uow: + run = await uow.generation_runs.get_run(run_id) + episode = await uow.episodes.get(episode_id) + events = await uow.generation_runs.list_events(run_id) + + assert run is not None, f"run {run_id} was not persisted; events={events!r}" + assert run.status is GenerationRunStatus.SUCCEEDED, ( + f"run {run_id} status={run.status}; events={events!r}" + ) + assert run.current_node == "complete", ( + f"run {run_id} current_node={run.current_node!r}" + ) + assert episode is not None, f"episode {episode_id} was not persisted" + assert episode.tei_xml == valid_tei(), ( + f"episode {episode_id} TEI={episode.tei_xml!r}" + ) + assert episode.qa_status is QaStatus.SKIPPED, ( + f"episode {episode_id} qa_status={episode.qa_status!r}" + ) + assert [event.kind for event in events] == [ + "run.started", + "draft.generated", + "tei.persisted", + "run.succeeded", + ], f"run {run_id} events={events!r}" + assert ( + generator.requests[0].sources[0].content == "Bridgewater launch source text." + ), f"run {run_id} source={generator.requests[0].sources[0]!r}" + assert [ + profile.display_name for profile in generator.requests[0].presenter_profiles + ] == ["Host One", "Guest One"], ( + f"run {run_id} profiles={generator.requests[0].presenter_profiles!r}" + ) + assert [profile.role for profile in generator.requests[0].presenter_profiles] == [ + "host", + "guest", + ], f"run {run_id} profiles={generator.requests[0].presenter_profiles!r}" + assert cost_recorder.provider_calls[0].workflow_run_id == str(run_id), ( + f"run {run_id} provider calls={cost_recorder.provider_calls!r}" + ) + assert cost_recorder.provider_calls[0].usage == { + "input_tokens": 10, + "output_tokens": 20, + }, f"run {run_id} usage={cost_recorder.provider_calls[0].usage!r}" + assert cost_recorder.finalized_runs == [(str(run_id), "draft")], ( + f"run {run_id} finalized runs={cost_recorder.finalized_runs!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_records_provider_failure( + session_factory: object, +) -> None: + """Draft-generation provider errors should become failed runs.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + run_id, _ = await prepare_pending_run(factory) + run_launcher = launcher( + factory, + FailingDraftGenerator(DraftScriptTransientProviderError("try again later")), + ) + tracer = RecordingTracer() + run_launcher.tracer = tracer + + run, events = await _launch_and_load_run(factory, run_id, run_launcher) + + assert run.status is GenerationRunStatus.FAILED, ( + f"run {run_id} status={run.status}; events={events!r}" + ) + assert run.error_message == "try again later", ( + f"run {run_id} error={run.error_message!r}" + ) + assert run.current_node == "failed", ( + f"run {run_id} current_node={run.current_node!r}" + ) + assert [event.kind for event in events] == ["run.started", "run.failed"], ( + f"run {run_id} events={events!r}" + ) + assert events[-1].payload["error_category"] == "provider.transient", ( + f"run {run_id} final event={events[-1]!r}" + ) + attributes = tracer.spans[0].attributes + assert attributes["operation"] == "generation_run.execute", attributes + assert attributes["run_id"] == str(run_id), attributes + assert attributes["outcome"] == "failed", attributes + assert attributes["failure_category"] == "provider.transient", attributes + + +@pytest.mark.asyncio +async def test_launcher_records_invalid_tei_failure( + session_factory: object, +) -> None: + """Invalid generated TEI should be recorded as a terminal run failure.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + run_id, _ = await prepare_pending_run(factory) + run_launcher = launcher( + factory, RecordingDraftGenerator(draft_result("broken")) + ) + + run, events = await _launch_and_load_run(factory, run_id, run_launcher) + + assert run.status is GenerationRunStatus.FAILED, ( + f"run {run_id} status={run.status}; events={events!r}" + ) + assert [event.kind for event in events] == [ + "run.started", + "draft.generated", + "tei.invalid", + "run.failed", + ], f"run {run_id} events={events!r}" + assert events[-1].payload["error_category"] == "tei.invalid", ( + f"run {run_id} final event={events[-1]!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_uses_detached_unit_of_work( + session_factory: object, +) -> None: + """Background launches should keep working after a request UoW closes.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + run_id, episode_id = await prepare_pending_run(factory) + run_launcher = launcher(factory, RecordingDraftGenerator(draft_result(valid_tei()))) + + async with SqlAlchemyUnitOfWork(factory) as request_uow: + assert await request_uow.generation_runs.get_run(run_id) is not None, ( + f"run {run_id} was unavailable in the request unit of work" + ) + + await run_launcher.launch(run_id) + await run_launcher.drain() + + async with SqlAlchemyUnitOfWork(factory) as uow: + run = await uow.generation_runs.get_run(run_id) + episode = await uow.episodes.get(episode_id) + + assert run is not None, f"run {run_id} was not persisted" + assert run.status is GenerationRunStatus.SUCCEEDED, ( + f"run {run_id} status={run.status}" + ) + assert episode is not None, f"episode {episode_id} was not persisted" + assert episode.tei_xml == valid_tei(), ( + f"episode {episode_id} TEI={episode.tei_xml!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_immediate_shutdown_marks_pending_task_failed( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Shutdown should persist cancellation before a task first executes.""" + run_id, _ = await prepare_pending_run(session_factory) + run_launcher = launcher( + session_factory, + RecordingDraftGenerator(draft_result(valid_tei())), + ) + + await run_launcher.launch(run_id) + await run_launcher.shutdown() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + run = await uow.generation_runs.get_run(run_id) + events = await uow.generation_runs.list_events(run_id) + + assert run is not None, f"run {run_id} was not persisted; events={events!r}" + assert run.status is GenerationRunStatus.FAILED, ( + f"run {run_id} status={run.status}; events={events!r}" + ) + assert run.error_category == "launcher.shutdown", ( + f"run {run_id} error category={run.error_category!r}; events={events!r}" + ) + assert [event.kind for event in events].count("run.failed") == 1, ( + f"run {run_id} events={events!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_shutdown_marks_running_task_failed( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Shutdown should fail a still-running background generation task.""" + run_id, _ = await prepare_pending_run(session_factory) + generator = BlockingDraftGenerator() + run_launcher = launcher(session_factory, generator) + + await run_launcher.launch(run_id) + await generator.started.wait() + await run_launcher.shutdown() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + run = await uow.generation_runs.get_run(run_id) + events = await uow.generation_runs.list_events(run_id) + + assert run is not None, f"run {run_id} was not persisted; events={events!r}" + assert run.status is GenerationRunStatus.FAILED, ( + f"run {run_id} status={run.status}; events={events!r}" + ) + assert run.error_message == "Generation task cancelled during shutdown.", ( + f"run {run_id} error={run.error_message!r}" + ) + assert events[-1].kind == "run.failed", f"run {run_id} final event={events[-1]!r}" + assert events[-1].payload["error_category"] == "launcher.shutdown", ( + f"run {run_id} final event={events[-1]!r}" + ) + assert [event.kind for event in events].count("run.failed") == 1, ( + f"run {run_id} events={events!r}" + ) + + +@pytest.mark.asyncio +async def test_launcher_returns_while_concurrency_slot_is_busy( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Scheduling should not wait for an execution semaphore permit.""" + run_id, _ = await prepare_pending_run(session_factory) + generator = BlockingDraftGenerator() + run_launcher = launcher( + session_factory, + generator, + options=LauncherOptions(max_concurrency=1, max_pending_runs=1), + ) + metrics = _RecordingMetrics() + run_launcher.metrics = metrics + + await run_launcher.launch(run_id) + await generator.started.wait() + await asyncio.wait_for(run_launcher.launch(uuid.uuid7()), timeout=0.1) + with pytest.raises(GenerationRunAdmissionError, match="capacity"): + await run_launcher.launch(uuid.uuid7()) + + assert run_launcher.scheduled_run_count == 2, ( + "expected one running and one pending task, got " + f"{run_launcher.scheduled_run_count}" + ) + assert ( + "generation_run_admission_rejected_total", + {"reason": "capacity"}, + ) in metrics.counters, metrics.counters + assert metrics.values[-1] == ( + "generation_run_pending_depth", + 1.0, + {}, + ), metrics.values + + await run_launcher.shutdown() + + +def test_launcher_rejects_negative_pending_capacity(session_factory: object) -> None: + """Pending admission capacity should be non-negative.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + + with pytest.raises(ValueError, match="max_pending_runs"): + launcher( + factory, + RecordingDraftGenerator(draft_result(valid_tei())), + options=LauncherOptions(max_pending_runs=-1), + ) diff --git a/tests/test_generation_run_launcher_admission.py b/tests/test_generation_run_launcher_admission.py new file mode 100644 index 00000000..e228b399 --- /dev/null +++ b/tests/test_generation_run_launcher_admission.py @@ -0,0 +1,60 @@ +"""Tests for in-process generation-run admission capacity.""" + +import dataclasses as dc +import typing as typ +import uuid + +import pytest + +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.generation.launcher import GenerationRunAdmissionError +from tests.generation_run_launcher_support import ( + LauncherOptions, + ReleasableDraftGenerator, + draft_result, + launcher, + prepare_pending_run, + valid_tei, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_launcher_releases_admission_capacity_after_completion( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """A completed run should release capacity for a later launch.""" + first_run_id, _ = await prepare_pending_run(session_factory) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + first_run = await uow.generation_runs.get_run(first_run_id) + assert first_run is not None, f"run {first_run_id} was not persisted" + second_run_id = uuid.uuid7() + await uow.generation_runs.create_run(dc.replace(first_run, id=second_run_id)) + await uow.commit() + generator = ReleasableDraftGenerator(draft_result(valid_tei())) + run_launcher = launcher( + session_factory, + generator, + options=LauncherOptions(max_concurrency=1, max_pending_runs=0), + ) + + await run_launcher.launch(first_run_id) + await generator.started.wait() + with pytest.raises(GenerationRunAdmissionError, match="capacity"): + await run_launcher.launch(second_run_id) + + generator.release.set() + await run_launcher.drain() + + assert run_launcher.scheduled_run_count == 0, ( + f"retained runs: {run_launcher.scheduled_run_count}" + ) + + await run_launcher.launch(second_run_id) + await run_launcher.drain() + + assert run_launcher.scheduled_run_count == 0, ( + f"retained runs: {run_launcher.scheduled_run_count}" + ) diff --git a/tests/test_generation_run_launcher_properties.py b/tests/test_generation_run_launcher_properties.py new file mode 100644 index 00000000..e1ddbff1 --- /dev/null +++ b/tests/test_generation_run_launcher_properties.py @@ -0,0 +1,137 @@ +"""Generated lifecycle invariants for the in-process generation launcher.""" + +import datetime as dt +import enum +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import GenerationRun, GenerationRunStatus, SourceDocument +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.generation.draft_script import ( + DraftScriptGenerator, + DraftScriptTransientProviderError, +) +from tests.canonical_storage._generation_run_support import ( + make_generation_run, + persist_generation_run_prerequisites, +) +from tests.generation_run_launcher_support import ( + BlockingDraftGenerator, + FailingDraftGenerator, + RecordingDraftGenerator, + draft_result, + launcher, + valid_tei, +) + +if typ.TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + type SessionFactory = async_sessionmaker[AsyncSession] +else: + type SessionFactory = object + + +class _Outcome(enum.StrEnum): + """Generated terminal launcher lifecycle outcomes.""" + + SUCCESS = "success" + FAILURE = "failure" + CANCELLED = "cancelled" + + +async def _persist_generated_lifecycle_run( + factory: SessionFactory, +) -> GenerationRun: + """Persist one launchable run with its generated-lifecycle source.""" + run = make_generation_run() + await persist_generation_run_prerequisites(factory, run) + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.generation_runs.create_run(run) + await uow.source_documents.add( + SourceDocument( + id=uuid.uuid7(), + ingestion_job_id=run.source_bundle_id, + canonical_episode_id=run.episode_id, + reference_document_revision_id=None, + source_type="research_note", + source_uri="https://example.test/source", + weight=1.0, + content_hash="sha256:source", + metadata={"content": "generated lifecycle source"}, + created_at=dt.datetime(2026, 8, 20, tzinfo=dt.UTC), + ) + ) + await uow.commit() + return run + + +def _generator_for_outcome(outcome: _Outcome) -> DraftScriptGenerator: + """Return the draft-generator double for one lifecycle outcome.""" + match outcome: + case _Outcome.SUCCESS: + return RecordingDraftGenerator(draft_result(valid_tei())) + case _Outcome.FAILURE: + return FailingDraftGenerator(DraftScriptTransientProviderError("retry")) + case _Outcome.CANCELLED: + return BlockingDraftGenerator() + + +def _assert_terminal_event_kinds(event_kinds: tuple[str, ...]) -> None: + """Assert that the lifecycle begins and ends with its expected events.""" + assert event_kinds, "expected lifecycle events" + assert event_kinds[0] == "run.started", event_kinds + assert event_kinds[-1] in {"run.succeeded", "run.failed"}, event_kinds + + +def _assert_outcome(run: GenerationRun, outcome: _Outcome) -> None: + """Assert the persisted terminal state for one generated lifecycle.""" + match outcome: + case _Outcome.SUCCESS: + assert run.status is GenerationRunStatus.SUCCEEDED, ( + f"expected successful run {run.id}, got {run.status.value}" + ) + assert run.error_category is None, run.error_category + case _Outcome.FAILURE: + assert run.status is GenerationRunStatus.FAILED, ( + f"expected failed run {run.id}, got {run.status.value}" + ) + assert run.error_category == "provider.transient", run.error_category + case _Outcome.CANCELLED: + assert run.status is GenerationRunStatus.FAILED, ( + f"expected cancelled run {run.id}, got {run.status.value}" + ) + assert run.error_category == "launcher.shutdown", run.error_category + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", list(_Outcome)) +async def test_launcher_generated_lifecycles_end_with_one_terminal_event( + session_factory: SessionFactory, + outcome: _Outcome, +) -> None: + """Success, provider failure, and shutdown cancellation reach one terminal state.""" + run = await _persist_generated_lifecycle_run(session_factory) + generator = _generator_for_outcome(outcome) + run_launcher = launcher(session_factory, generator) + + await run_launcher.launch(run.id) + match outcome: + case _Outcome.CANCELLED: + await typ.cast("BlockingDraftGenerator", generator).started.wait() + await run_launcher.shutdown() + case _: + await run_launcher.drain() + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + persisted_run = await uow.generation_runs.get_run(run.id) + events = await uow.generation_runs.list_events(run.id) + + assert persisted_run is not None, f"expected persisted run {run.id}" + assert persisted_run.status.is_terminal(), ( + f"expected terminal run {run.id}, got {persisted_run.status.value}" + ) + _assert_terminal_event_kinds(tuple(event.kind for event in events)) + _assert_outcome(persisted_run, outcome) diff --git a/tests/test_generation_run_port_contract.py b/tests/test_generation_run_port_contract.py index 7fe09b71..ac787e76 100644 --- a/tests/test_generation_run_port_contract.py +++ b/tests/test_generation_run_port_contract.py @@ -1,17 +1,13 @@ """Contract tests for generation-run port implementations. -These tests define the behavioural contract for adapters implementing -`GenerationRunRepository`, `GenerationEventLog`, `GenerationCheckpointPort`, -and the composite `GenerationRunPort`. They validate protocol compliance, -lifecycle guarantees, error and edge-case behaviour, idempotency, pagination -guardrails, event sequence allocation, and checkpoint response persistence. -Use the `store` fixture plus `make_generation_run()` and `make_checkpoint()` -when adding scenarios for another implementation. +These tests define the behavioural contract for generation-run adapters. Use +the `store` fixture plus `make_generation_run()` and `make_checkpoint()` when +adding scenarios for another implementation. """ +import asyncio import dataclasses as dc import datetime as dt -import typing as typ import uuid import pytest @@ -19,22 +15,20 @@ from episodic.canonical.adapters.generation_runs import InMemoryGenerationRunStore from episodic.canonical.domain import ( Checkpoint, - CheckpointResponse, CheckpointStatus, - GenerationEvent, GenerationRun, GenerationRunStatus, - JsonMapping, ) -from episodic.canonical.generation_run_errors import CheckpointNotFound, RunNotFound +from episodic.canonical.generation_quality import QaStatus, QualityMode +from episodic.canonical.generation_run_errors import RunAlreadyTerminal, RunNotFound from episodic.canonical.generation_run_ports import ( - EventSeq, GenerationCheckpointPort, GenerationEventLog, GenerationRunPort, GenerationRunRepository, event_seq, ) +from tests.test_generation_run_port_contract_support import NoopGenerationRunPort NOW = dt.datetime(2026, 6, 4, 8, 0, tzinfo=dt.UTC) @@ -75,6 +69,9 @@ def make_generation_run( started_at=None, ended_at=None, error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", ) @@ -114,105 +111,6 @@ def store() -> InMemoryGenerationRunStore: return InMemoryGenerationRunStore(time_provider=lambda: NOW) -# Protocol arity is fixed by the port contract; this is a minimal test stub. -class NoopGenerationRunPort: # pylint: disable=too-many-arguments - """No-op implementation used for composite protocol type checking.""" - - async def create_run( - self, - run: GenerationRun, - *, - idempotency_key: str | None = None, - ) -> GenerationRun: - """Return the supplied run.""" - return run - - async def get_run(self, run_id: uuid.UUID) -> GenerationRun | None: - """Return no run.""" - return None - - async def list_runs( - self, - episode_id: uuid.UUID, - *, - status: GenerationRunStatus | None = None, - limit: int = 50, - offset: int = 0, - ) -> tuple[GenerationRun, ...]: - """Return no runs.""" - return () - - async def update_run_status( - self, - run_id: uuid.UUID, - *, - status: GenerationRunStatus, - current_node: str | None, - ended_at: dt.datetime | None, - ) -> GenerationRun: - """Raise for all updates.""" - raise RunNotFound(run_id) - - async def append_event( - self, - run_id: uuid.UUID, - *, - kind: str, - payload: JsonMapping, - occurred_at: dt.datetime | None = None, - ) -> GenerationEvent: - """Raise for all event appends.""" - raise RunNotFound(run_id) - - async def list_events( - self, - run_id: uuid.UUID, - *, - after_seq: EventSeq | None = None, - limit: int = 100, - ) -> tuple[GenerationEvent, ...]: - """Return no events.""" - return () - - async def create_checkpoint(self, checkpoint: Checkpoint) -> Checkpoint: - """Return the supplied checkpoint.""" - return checkpoint - - async def get_checkpoint( - self, - checkpoint_id: uuid.UUID, - ) -> Checkpoint | None: - """Return no checkpoint.""" - return None - - async def respond_to_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - response: CheckpointResponse, - ) -> Checkpoint: - """Raise for all responses.""" - raise CheckpointNotFound(checkpoint_id) - - async def time_out_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - at: dt.datetime, - ) -> Checkpoint: - """Raise for all timeouts.""" - raise CheckpointNotFound(checkpoint_id) - - async def cancel_checkpoint( - self, - checkpoint_id: uuid.UUID, - *, - at: dt.datetime, - ) -> Checkpoint: - """Raise for all cancellations.""" - raise CheckpointNotFound(checkpoint_id) - - class TestGenerationRunRepository: """Contract tests for generation-run repository operations.""" @@ -234,25 +132,45 @@ def test_in_memory_store_satisfies_generation_run_ports(self) -> None: ) @pytest.mark.asyncio - async def test_create_run_reuses_idempotency_key( + async def test_create_run_scopes_idempotency_key_to_principal( self, store: InMemoryGenerationRunStore, ) -> None: - """Creating with the same idempotency key returns the first run.""" + """A principal replays its run while another principal gets a new run.""" first = make_generation_run() - duplicate = make_generation_run() + cross_principal = make_generation_run() + replay = make_generation_run() - stored_first = await store.create_run(first, idempotency_key="run-key") - stored_duplicate = await store.create_run(duplicate, idempotency_key="run-key") + stored_first = await store.create_run( + first, + idempotency_key="run-key", + idempotency_principal_id="principal-a", + ) + stored_cross_principal = await store.create_run( + cross_principal, + idempotency_key="run-key", + idempotency_principal_id="principal-b", + ) + stored_replay = await store.create_run( + replay, + idempotency_key="run-key", + idempotency_principal_id="principal-a", + ) - assert stored_duplicate == stored_first, ( - "duplicate idempotency key must return first run" + assert stored_cross_principal == cross_principal, ( + "cross-principal key reuse must create a distinct run" + ) + assert stored_replay == stored_first, ( + "same-principal key replay must return first run" ) assert await store.get_run(first.id) == stored_first, ( "first run must be retrievable by ID" ) - assert await store.get_run(duplicate.id) is None, ( - "duplicate run must not be persisted" + assert await store.get_run(cross_principal.id) == stored_cross_principal, ( + "cross-principal run must be persisted" + ) + assert await store.get_run(replay.id) is None, ( + "same-principal replay run must not be persisted" ) @pytest.mark.asyncio @@ -299,6 +217,62 @@ async def test_list_runs_rejects_negative_offset( with pytest.raises(ValueError, match="offset"): await store.list_runs(uuid.uuid7(), offset=-1) + @pytest.mark.asyncio + async def test_claim_run_for_execution_is_first_writer_wins( + self, + store: InMemoryGenerationRunStore, + ) -> None: + """A pending run can be claimed once for execution.""" + run = await store.create_run(make_generation_run()) + + results = await asyncio.gather( + store.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=NOW + dt.timedelta(minutes=5), + ), + store.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=NOW + dt.timedelta(minutes=5), + ), + ) + claim_count = sum(result is not None for result in results) + assert claim_count == 1, f"expected one successful claim, got {claim_count}" + claimed = next(result for result in results if result is not None) + assert claimed.status is GenerationRunStatus.RUNNING, ( + f"status: {claimed.status}" + ) + assert claimed.current_node == "draft", f"node: {claimed.current_node!r}" + assert claimed.started_at == NOW, f"started_at: {claimed.started_at!r}" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "status", + [ + GenerationRunStatus.SUCCEEDED, + GenerationRunStatus.FAILED, + GenerationRunStatus.CANCELLED, + ], + ) + async def test_claim_run_for_execution_rejects_terminal_runs( + self, + store: InMemoryGenerationRunStore, + status: GenerationRunStatus, + ) -> None: + """Terminal runs cannot be reclaimed for execution.""" + run = await store.create_run(dc.replace(make_generation_run(), status=status)) + + with pytest.raises(RunAlreadyTerminal, match="generation run is already"): + await store.claim_run_for_execution( + run.id, + current_node="draft", + started_at=NOW, + lease_expires_at=NOW + dt.timedelta(minutes=5), + ) + class TestGenerationEventLog: """Contract tests for generation event-log operations.""" @@ -321,6 +295,8 @@ async def test_append_event_allocates_gap_free_sequences( assert await store.list_events(run.id, after_seq=event_seq(1)) == (second,), ( "list_events after seq 1 should return only the second event" ) + with pytest.raises(ValueError, match="after_seq and offset cannot be combined"): + await store.list_events(run.id, after_seq=event_seq(1), offset=1) @pytest.mark.asyncio async def test_append_event_rejects_unknown_run( @@ -331,26 +307,13 @@ async def test_append_event_rejects_unknown_run( with pytest.raises(RunNotFound, match=r"unknown generation run:"): await store.append_event(uuid.uuid7(), kind="created", payload={}) - @pytest.mark.asyncio - async def test_list_events_rejects_negative_limit( - self, - store: InMemoryGenerationRunStore, - ) -> None: - """Event pagination limits must be non-negative.""" - run = await store.create_run(make_generation_run()) - - with pytest.raises(ValueError, match="limit"): - await store.list_events(run.id, limit=-1) - class TestCompositeProtocol: """Contract tests for the composite generation-run protocol.""" def test_noop_composite_protocol_stub_typechecks(self) -> None: """A class implementing every method should satisfy the composite port.""" - # Static type checkers validate NoopGenerationRunPort against - # GenerationRunPort through this assignment. - _port: GenerationRunPort = typ.cast( - "GenerationRunPort", - NoopGenerationRunPort(), + noop_port: GenerationRunPort = NoopGenerationRunPort() + assert isinstance(noop_port, GenerationRunPort), ( + f"expected no-op port to satisfy GenerationRunPort: {noop_port!r}" ) diff --git a/tests/test_generation_run_port_contract_support.py b/tests/test_generation_run_port_contract_support.py new file mode 100644 index 00000000..f023f369 --- /dev/null +++ b/tests/test_generation_run_port_contract_support.py @@ -0,0 +1,337 @@ +"""No-op implementations for generation-run protocol contract tests.""" + +import typing as typ + +from episodic.canonical.generation_run_errors import CheckpointNotFound, RunNotFound + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + + from episodic.canonical.domain import ( + Checkpoint, + CheckpointResponse, + GenerationEvent, + GenerationRun, + GenerationRunStatus, + JsonMapping, + ) + from episodic.canonical.generation_run_ports import ( + EventSeq, + GenerationRunStatusUpdate, + ) + + +# Protocol arity is fixed by the port contract; this is a minimal test stub. +class NoopGenerationRunPort: # pylint: disable=too-many-arguments + """No-op composite port implementation used for protocol type checking. + + Read operations return empty values, creation operations return their input, + and mutation operations raise the corresponding not-found exception. The + implementation exists only to exercise the complete composite protocol + surface without persistence. + """ + + async def create_run( + self, + run: GenerationRun, + *, + idempotency_key: str | None = None, + idempotency_principal_id: str | None = None, + ) -> GenerationRun: + """Return the supplied run without persisting it. + + Parameters + ---------- + run + Generation run to return. + idempotency_key + Idempotency key accepted by the port contract and ignored here. + idempotency_principal_id + Principal identifier accepted by the port contract and ignored here. + + Returns + ------- + GenerationRun + The supplied generation run. + """ + return run + + async def get_run(self, run_id: uuid.UUID) -> GenerationRun | None: + """Return no run for the supplied identifier. + + Parameters + ---------- + run_id + Run identifier, which is ignored by this no-op implementation. + + Returns + ------- + GenerationRun or None + Always ``None`` because this implementation stores no runs. + """ + return None + + async def list_runs( + self, + episode_id: uuid.UUID, + *, + status: GenerationRunStatus | None = None, + limit: int = 50, + offset: int = 0, + ) -> tuple[GenerationRun, ...]: + """Return no runs for the supplied episode. + + Parameters + ---------- + episode_id + Episode identifier, which is ignored by this no-op implementation. + status + Optional lifecycle filter accepted by the port contract and ignored + here. + limit + Maximum number of runs accepted by the port contract and ignored + here. + offset + Number of runs to skip accepted by the port contract and ignored + here. + + Returns + ------- + tuple of GenerationRun + Always an empty tuple because this implementation stores no runs. + """ + return () + + async def update_run_status( + self, + run_id: uuid.UUID, + *, + update: GenerationRunStatusUpdate, + ) -> GenerationRun: + """Reject every lifecycle update because no run is stored. + + Parameters + ---------- + run_id + Identifier of the run to update. + update + Lifecycle update to apply. + + Raises + ------ + RunNotFound + Always, because this implementation stores no runs. + """ + _ = update + raise RunNotFound(run_id) + + async def claim_run_for_execution( + self, + run_id: uuid.UUID, + *, + current_node: str | None, + started_at: dt.datetime, + lease_expires_at: dt.datetime | None, + ) -> GenerationRun | None: + """Reject every execution claim because no run is stored. + + Parameters + ---------- + run_id + Identifier of the run to claim. + current_node + Worker node that would own the claim. + started_at + Timestamp at which execution would start. + lease_expires_at + Optional timestamp at which the execution lease would expire. + + Raises + ------ + RunNotFound + Always, because this implementation stores no runs. + """ + raise RunNotFound(run_id) + + async def append_event( + self, + run_id: uuid.UUID, + *, + kind: str, + payload: JsonMapping, + occurred_at: dt.datetime | None = None, + ) -> GenerationEvent: + """Reject every event append because no run is stored. + + Parameters + ---------- + run_id + Identifier of the run to which the event would be appended. + kind + Event kind accepted by the port contract. + payload + Event payload accepted by the port contract. + occurred_at + Optional event timestamp accepted by the port contract. + + Raises + ------ + RunNotFound + Always, because this implementation stores no runs. + """ + raise RunNotFound(run_id) + + async def list_events( + self, + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + limit: int = 100, + offset: int = 0, + ) -> tuple[GenerationEvent, ...]: + """Return no events for the supplied run. + + Parameters + ---------- + run_id + Run identifier, which is ignored by this no-op implementation. + after_seq + Optional event-sequence cursor accepted by the port contract and + ignored here. + limit + Maximum number of events accepted by the port contract and ignored + here. + offset + Number of events to skip accepted by the port contract and ignored + here. + + Returns + ------- + tuple of GenerationEvent + Always an empty tuple because this implementation stores no events. + """ + return () + + async def count_events( + self, + run_id: uuid.UUID, + *, + after_seq: EventSeq | None = None, + ) -> int: + """Return zero for the supplied run's event count. + + Parameters + ---------- + run_id + Run identifier, which is ignored by this no-op implementation. + after_seq + Optional event-sequence cursor accepted by the port contract and + ignored here. + + Returns + ------- + int + Always zero because this implementation stores no events. + """ + return 0 + + async def create_checkpoint(self, checkpoint: Checkpoint) -> Checkpoint: + """Return the supplied checkpoint without persisting it. + + Parameters + ---------- + checkpoint + Checkpoint to return. + + Returns + ------- + Checkpoint + The supplied checkpoint. + """ + return checkpoint + + async def get_checkpoint( + self, + checkpoint_id: uuid.UUID, + ) -> Checkpoint | None: + """Return no checkpoint for the supplied identifier. + + Parameters + ---------- + checkpoint_id + Checkpoint identifier, which is ignored by this no-op + implementation. + + Returns + ------- + Checkpoint or None + Always ``None`` because this implementation stores no checkpoints. + """ + return None + + async def respond_to_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + response: CheckpointResponse, + ) -> Checkpoint: + """Reject every checkpoint response because none is stored. + + Parameters + ---------- + checkpoint_id + Identifier of the checkpoint to respond to. + response + Reviewer response to record. + + Raises + ------ + CheckpointNotFound + Always, because this implementation stores no checkpoints. + """ + raise CheckpointNotFound(checkpoint_id) + + async def time_out_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + at: dt.datetime, + ) -> Checkpoint: + """Reject every checkpoint timeout because none is stored. + + Parameters + ---------- + checkpoint_id + Identifier of the checkpoint to time out. + at + Timestamp at which the timeout would be recorded. + + Raises + ------ + CheckpointNotFound + Always, because this implementation stores no checkpoints. + """ + raise CheckpointNotFound(checkpoint_id) + + async def cancel_checkpoint( + self, + checkpoint_id: uuid.UUID, + *, + at: dt.datetime, + ) -> Checkpoint: + """Reject every checkpoint cancellation because none is stored. + + Parameters + ---------- + checkpoint_id + Identifier of the checkpoint to cancel. + at + Timestamp at which the cancellation would be recorded. + + Raises + ------ + CheckpointNotFound + Always, because this implementation stores no checkpoints. + """ + raise CheckpointNotFound(checkpoint_id) diff --git a/tests/test_generation_run_properties.py b/tests/test_generation_run_properties.py index c5503b27..4269dc7e 100644 --- a/tests/test_generation_run_properties.py +++ b/tests/test_generation_run_properties.py @@ -12,7 +12,6 @@ import dataclasses import datetime as dt import itertools -import typing as typ import uuid import hypothesis.strategies as st @@ -21,8 +20,12 @@ from episodic.canonical.adapters.generation_runs import InMemoryGenerationRunStore from episodic.canonical.domain import GenerationRun, GenerationRunStatus +from episodic.canonical.generation_quality import QaStatus, QualityMode from episodic.canonical.generation_run_errors import RunAlreadyTerminal -from episodic.canonical.generation_run_ports import event_seq +from episodic.canonical.generation_run_ports import ( + GenerationRunStatusUpdate, + event_seq, +) NOW = dt.datetime(2026, 6, 4, 8, 0, tzinfo=dt.UTC) type EventInput = tuple[str, dict[str, object]] @@ -101,6 +104,9 @@ def make_generation_run( started_at=None, ended_at=None, error_message=None, + quality_mode=QualityMode.DRAFT_WITHOUT_QA, + qa_status=QaStatus.SKIPPED, + skip_qa_rationale="No-QA vertical-slice draft.", ) @@ -218,18 +224,22 @@ async def test_terminal_runs_reject_status_updates_and_events( terminal = await store.update_run_status( run.id, - status=terminal_status, - current_node=None, - ended_at=NOW, + update=GenerationRunStatusUpdate( + status=terminal_status, + current_node=None, + ended_at=NOW, + ), ) assert terminal.status == terminal_status, "Run must enter the terminal state." with pytest.raises(RunAlreadyTerminal, match="generation run is already terminal"): await store.update_run_status( run.id, - status=GenerationRunStatus.RUNNING, - current_node="planner", - ended_at=None, + update=GenerationRunStatusUpdate( + status=GenerationRunStatus.RUNNING, + current_node="planner", + ended_at=None, + ), ) with pytest.raises(RunAlreadyTerminal, match="generation run is already terminal"): await store.append_event(run.id, kind="node_started", payload={}) @@ -290,109 +300,18 @@ async def test_list_events_pagination_is_stable( assert listed == expected, "Event page must match the sequence slice." -async def _apply_adapter_operation( - state: AdapterExerciseState, - operation_item: AdapterOperation, - *, - is_terminal: bool, - limit: int, -) -> bool: - """Apply one generated adapter operation and return terminal state.""" - match operation_item: - case ("append", (kind, payload)): - if is_terminal: - with pytest.raises( - RunAlreadyTerminal, - match="generation run is already terminal", - ): - await state.store.append_event( - state.run_id, - kind=kind, - payload=payload, - ) - return is_terminal - state.appended.append( - await state.store.append_event( - state.run_id, - kind=kind, - payload=payload, - ) - ) - return is_terminal - case ("terminal", status): - status = typ.cast("GenerationRunStatus", status) - if is_terminal: - with pytest.raises( - RunAlreadyTerminal, - match="generation run is already terminal", - ): - await state.store.update_run_status( - state.run_id, - status=status, - current_node=None, - ended_at=NOW, - ) - return is_terminal - await state.store.update_run_status( - state.run_id, - status=status, - current_node=None, - ended_at=NOW, - ) - return True - case ("list", None): - listed = await state.store.list_events(state.run_id, limit=limit) - assert listed == tuple(state.appended[:limit]), ( - "Listed events must preserve adapter sequence order." - ) - return is_terminal - msg = f"Unknown adapter operation: {operation_item!r}" - raise AssertionError(msg) - - -@given( - idempotency_key=st.text(min_size=1, max_size=24), - operations=ADAPTER_OPERATIONS, - limit=PAGE_LIMITS, -) -@settings(max_examples=35, deadline=None) @pytest.mark.asyncio -async def test_adapter_invariants_hold_across_generated_operation_sequences( - idempotency_key: str, - operations: list[AdapterOperation], - limit: int, +async def test_list_events_offset_pagination_selects_the_filtered_page( monotonic_time_provider: cabc.Callable[[], dt.datetime], ) -> None: - """Generated adapter operation sequences should preserve core invariants.""" + """Offset pagination returns the second event without a cursor.""" store = InMemoryGenerationRunStore(time_provider=monotonic_time_provider) - run = await store.create_run( - make_generation_run(), - idempotency_key=idempotency_key, - ) - retried = await store.create_run( - make_generation_run(), - idempotency_key=idempotency_key, - ) - state = AdapterExerciseState(store=store, run_id=run.id, appended=[]) - is_terminal = False - - assert retried.id == run.id, "Idempotency retry must return first run." + run = await store.create_run(make_generation_run()) + appended = [ + await store.append_event(run.id, kind=f"event.{index}", payload={}) + for index in range(3) + ] - for operation_item in operations: - is_terminal = await _apply_adapter_operation( - state, - operation_item, - is_terminal=is_terminal, - limit=limit, - ) + listed = await store.list_events(run.id, offset=1, limit=1) - listed = await store.list_events(run.id) - assert [event.seq for event in listed] == list(range(1, len(state.appended) + 1)), ( - "Generated operation sequences must leave gap-free event sequences." - ) - for after_index in range(len(state.appended) + 1): - after_seq = event_seq(after_index) if after_index else None - page = await store.list_events(run.id, after_seq=after_seq, limit=limit) - assert page == tuple(state.appended[after_index : after_index + limit]), ( - "Event cursor pages must match the appended event slice." - ) + assert listed == (appended[1],), f"offset event page: {listed!r}" diff --git a/tests/test_generation_run_resource_factories.py b/tests/test_generation_run_resource_factories.py new file mode 100644 index 00000000..5e00c654 --- /dev/null +++ b/tests/test_generation_run_resource_factories.py @@ -0,0 +1,72 @@ +"""Deterministic factory seams for generation-run HTTP resource creation.""" + +import datetime as dt +import typing as typ +import uuid + +import httpx +import pytest + +from episodic.api import create_app +from episodic.api.resources.generation_runs import ( + GenerationRunsResource, + _CreateGenerationRun, +) +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import ( + HeaderPrincipalAuthorization, + RecordingLauncher, + create_ready_ingestion_job, +) + +if typ.TYPE_CHECKING: + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@pytest.mark.asyncio +async def test_generation_run_resource_uses_injected_factories( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Run creation should use deterministic IDs and lifecycle timestamps.""" + dependencies = build_api_dependencies( + session_factory, + authorization=HeaderPrincipalAuthorization(), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport( + app=typ.cast("_ASGIApp", create_app(dependencies)) + ), + base_url="http://testserver", + ) as client: + ingestion_job_id = uuid.UUID( + await create_ready_ingestion_job( + client, {"Authorization": "Bearer principal-a"} + ) + ) + + ids = tuple(uuid.uuid7() for _ in range(4)) + now = dt.datetime(2026, 7, 22, tzinfo=dt.UTC) + resource = GenerationRunsResource( + dependencies.uow_factory, + launcher=RecordingLauncher(), + clock=lambda: now, + uuid_factory=iter(ids).__next__, + ) + run = await resource._create_run( + ingestion_job_id, + _CreateGenerationRun( + skip_qa_rationale="Deterministic resource construction.", + configuration={}, + budget_snapshot={}, + ), + actor="principal-a", + idempotency_key="factory-key", + ) + + assert run.id == ids[2], f"expected factory run id {ids[2]}, got {run.id}" + assert run.episode_id == ids[0], ( + f"expected factory episode id {ids[0]}, got {run.episode_id}" + ) + assert run.created_at == now, f"expected fixed creation time, got {run.created_at}" + assert run.updated_at == now, f"expected fixed update time, got {run.updated_at}" diff --git a/tests/test_generation_run_state_machine_properties.py b/tests/test_generation_run_state_machine_properties.py new file mode 100644 index 00000000..6d9e27f6 --- /dev/null +++ b/tests/test_generation_run_state_machine_properties.py @@ -0,0 +1,135 @@ +"""Generated state-machine invariants for the in-memory generation-run store.""" + +import datetime as dt +import itertools + +import hypothesis.strategies as st +import pytest +from hypothesis import given, settings + +from episodic.canonical.adapters.generation_runs import InMemoryGenerationRunStore +from episodic.canonical.domain import GenerationRunStatus +from episodic.canonical.generation_run_errors import RunAlreadyTerminal +from episodic.canonical.generation_run_ports import GenerationRunStatusUpdate, event_seq +from tests.test_generation_run_properties import ( + ADAPTER_OPERATIONS, + NOW, + PAGE_LIMITS, + AdapterExerciseState, + AdapterOperation, + make_generation_run, +) + + +async def _apply_adapter_operation( + state: AdapterExerciseState, + operation_item: AdapterOperation, + *, + is_terminal: bool, + limit: int, +) -> bool: + """Apply one generated adapter operation and return terminal state.""" + match operation_item: + case ("append", (kind, payload)): + if is_terminal: + with pytest.raises( + RunAlreadyTerminal, + match="generation run is already terminal", + ): + await state.store.append_event( + state.run_id, + kind=kind, + payload=payload, + ) + return is_terminal + state.appended.append( + await state.store.append_event( + state.run_id, + kind=kind, + payload=payload, + ) + ) + return is_terminal + case ("terminal", GenerationRunStatus() as status): + if is_terminal: + with pytest.raises( + RunAlreadyTerminal, + match="generation run is already terminal", + ): + await state.store.update_run_status( + state.run_id, + update=GenerationRunStatusUpdate( + status=status, + current_node=None, + ended_at=NOW, + ), + ) + return is_terminal + await state.store.update_run_status( + state.run_id, + update=GenerationRunStatusUpdate( + status=status, + current_node=None, + ended_at=NOW, + ), + ) + return True + case ("list", None): + listed = await state.store.list_events(state.run_id, limit=limit) + assert listed == tuple(state.appended[:limit]), ( + "Listed events must preserve adapter sequence order." + ) + return is_terminal + msg = f"Unknown adapter operation: {operation_item!r}" + raise AssertionError(msg) + + +@given( + idempotency_key=st.text(min_size=1, max_size=24), + operations=ADAPTER_OPERATIONS, + limit=PAGE_LIMITS, +) +@settings(max_examples=35, deadline=None) +@pytest.mark.asyncio +async def test_adapter_invariants_hold_across_generated_operation_sequences( + idempotency_key: str, + operations: list[AdapterOperation], + limit: int, +) -> None: + """Generated adapter operation sequences should preserve core invariants.""" + counter = itertools.count() + + def monotonic_time_provider() -> dt.datetime: + return NOW + dt.timedelta(microseconds=next(counter)) + + store = InMemoryGenerationRunStore(time_provider=monotonic_time_provider) + run = await store.create_run( + make_generation_run(), + idempotency_key=idempotency_key, + ) + retried = await store.create_run( + make_generation_run(), + idempotency_key=idempotency_key, + ) + state = AdapterExerciseState(store=store, run_id=run.id, appended=[]) + is_terminal = False + + assert retried.id == run.id, "Idempotency retry must return first run." + for operation_item in operations: + is_terminal = await _apply_adapter_operation( + state, + operation_item, + is_terminal=is_terminal, + limit=limit, + ) + + listed = await store.list_events(run.id) + assert [event.seq for event in listed] == list(range(1, len(state.appended) + 1)), ( + "Generated operation sequences must leave gap-free event sequences." + ) + for after_index in range(len(state.appended) + 1): + after_seq = event_seq(after_index) if after_index else None + page = await store.list_events(run.id, after_seq=after_seq, limit=limit) + assert page == tuple(state.appended[after_index : after_index + limit]), ( + "Event cursor pages must match the appended event slice." + ) diff --git a/tests/test_generation_source_limits.py b/tests/test_generation_source_limits.py new file mode 100644 index 00000000..5537ac65 --- /dev/null +++ b/tests/test_generation_source_limits.py @@ -0,0 +1,216 @@ +"""Tests for bounded source hydration before draft generation.""" + +import asyncio +import contextlib +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import SourceDocument +from episodic.canonical.storage.filesystem_object_store import FilesystemObjectStore +from episodic.generation.draft_script import DraftScriptGenerationError +from episodic.generation.launcher import InProcessGenerationRunLauncher +from episodic.generation.launcher_support import ( + GenerationSourceLimitError, + GenerationSourceLimits, + source_from_document, +) +from tests.generation_run_launcher_support import RecordingDraftGenerator, draft_result + +if typ.TYPE_CHECKING: + import collections.abc as cabc + from pathlib import Path + + from episodic.canonical.object_store import ObjectStorePort + from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork + + +@dc.dataclass(slots=True) +class _ChunkStore: + """Object-store fake that records how many source chunks were consumed.""" + + chunks: tuple[bytes, ...] + yielded_chunks: int = 0 + + @contextlib.asynccontextmanager + async def open( + self, + key: str, + ) -> cabc.AsyncIterator[cabc.AsyncIterator[bytes]]: + """Yield the configured source bytes without retaining additional data.""" + del key + + async def iterator() -> cabc.AsyncIterator[bytes]: + """Yield one configured chunk at a time.""" + for chunk in self.chunks: + self.yielded_chunks += 1 + await asyncio.sleep(0) + yield chunk + + yield iterator() + + +def _source_document( + *, content: str | None = None, source_uri: str = "source" +) -> SourceDocument: + """Return one canonical source document with optional inline content.""" + return SourceDocument( + id=uuid.uuid7(), + ingestion_job_id=uuid.uuid7(), + canonical_episode_id=uuid.uuid7(), + reference_document_revision_id=None, + source_type="research_note", + source_uri=source_uri, + weight=1.0, + content_hash="sha256:source", + metadata={} if content is None else {"content": content}, + created_at=dt.datetime(2026, 8, 20, tzinfo=dt.UTC), + ) + + +def _launcher(*, limits: GenerationSourceLimits) -> InProcessGenerationRunLauncher: + """Build a launcher only for its source-loading boundary.""" + return InProcessGenerationRunLauncher( + uow_factory=lambda: typ.cast("CanonicalUnitOfWork", object()), + draft_generator=RecordingDraftGenerator(draft_result("")), + source_limits=limits, + ) + + +@pytest.mark.asyncio +async def test_source_loading_rejects_count_above_configured_limit() -> None: + """Reject source bundles before loading content beyond the source-count bound.""" + run_launcher = _launcher( + limits=GenerationSourceLimits( + max_source_count=1, + max_source_bytes=10, + max_aggregate_source_bytes=10, + max_normalized_source_bytes=10, + ) + ) + + with pytest.raises(GenerationSourceLimitError, match="count"): + await run_launcher._load_sources([ + _source_document(content="one"), + _source_document(content="two"), + ]) + + +@pytest.mark.asyncio +async def test_uploaded_source_stops_before_retaining_over_limit_chunk() -> None: + """Stop streaming when the next uploaded chunk exceeds the source limit.""" + store = _ChunkStore((b"1234", b"5678", b"not-read")) + limits = GenerationSourceLimits( + max_source_count=2, + max_source_bytes=6, + max_aggregate_source_bytes=10, + max_normalized_source_bytes=10, + ) + + with pytest.raises(GenerationSourceLimitError, match="source exceeds byte"): + await source_from_document( + _source_document(source_uri="upload:uploads/source"), + typ.cast("ObjectStorePort", store), + limits, + ) + + assert store.yielded_chunks == 2, store.yielded_chunks + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("chunks", "expected_message"), + [ + pytest.param( + (b"\xff",), + "Uploaded source 'uploads/source' is not valid UTF-8 text.", + id="invalid_utf8", + ), + pytest.param( + (b" \n\t ",), + "Uploaded source 'uploads/source' contains no text.", + id="whitespace_only", + ), + ], +) +async def test_uploaded_source_rejects_invalid_text( + chunks: tuple[bytes, ...], + expected_message: str, +) -> None: + """Report stable diagnostics for invalid uploaded source text.""" + with pytest.raises(DraftScriptGenerationError) as raised: + await source_from_document( + _source_document(source_uri="upload:uploads/source"), + typ.cast("ObjectStorePort", _ChunkStore(chunks)), + ) + assert str(raised.value) == expected_message, ( + f"expected malformed upload diagnostic {expected_message!r}, " + f"got {str(raised.value)!r}" + ) + + +@pytest.mark.asyncio +async def test_source_loading_enforces_aggregate_and_normalized_limits() -> None: + """Distinguish aggregate input capacity from normalized-text capacity.""" + aggregate_limits = GenerationSourceLimits( + max_source_count=2, + max_source_bytes=10, + max_aggregate_source_bytes=4, + max_normalized_source_bytes=10, + ) + normalized_limits = GenerationSourceLimits( + max_source_count=2, + max_source_bytes=10, + max_aggregate_source_bytes=10, + max_normalized_source_bytes=4, + ) + + with pytest.raises(GenerationSourceLimitError, match="aggregate"): + await source_from_document( + _source_document(content="12345"), + None, + aggregate_limits, + remaining_aggregate_bytes=4, + ) + with pytest.raises(GenerationSourceLimitError, match="normalized"): + await source_from_document( + _source_document(content="12345"), + None, + normalized_limits, + ) + + +@pytest.mark.asyncio +async def test_filesystem_object_store_offloads_open_read_and_close( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Perform filesystem reads through threads rather than the event loop.""" + import episodic.canonical.storage.filesystem_object_store as object_store_module + + store = FilesystemObjectStore(tmp_path) + path = tmp_path / "uploads" / "source" + path.parent.mkdir() + path.write_bytes(b"source") + offloaded: list[str] = [] + original_to_thread = object_store_module.asyncio.to_thread + + async def recording_to_thread( + function: cabc.Callable[..., object], + /, + *args: object, + **kwargs: object, + ) -> object: + """Record the blocking operation delegated by the storage adapter.""" + offloaded.append(getattr(function, "__name__", type(function).__name__)) + return await original_to_thread(function, *args, **kwargs) + + monkeypatch.setattr(object_store_module.asyncio, "to_thread", recording_to_thread) + async with store.open("uploads/source") as chunks: + content = b"".join([chunk async for chunk in chunks]) + + assert content == b"source", f"expected stored source bytes, got {content!r}" + assert offloaded == ["open", "read", "read", "close"], offloaded diff --git a/tests/test_lifespan_hooks.py b/tests/test_lifespan_hooks.py index 11a3ff0c..eaaf442f 100644 --- a/tests/test_lifespan_hooks.py +++ b/tests/test_lifespan_hooks.py @@ -8,25 +8,35 @@ import tests.test_http_service_scaffold_support as scaffold_support if typ.TYPE_CHECKING: + from pathlib import Path + import httpx from httpx._transports.asgi import _ASGIApp @pytest.mark.asyncio async def test_create_app_runs_shutdown_hooks_during_asgi_shutdown() -> None: - """Expose a cleanup seam for runtime-managed resources like DB engines.""" + """Run lifecycle cleanup hooks sequentially in their supplied order.""" from episodic.api import ApiDependencies, create_app hook_calls: list[str] = [] + first_hook_completed = False + + async def first_shutdown_hook() -> None: + nonlocal first_hook_completed + await asyncio.sleep(0) + hook_calls.append("first") + first_hook_completed = True - async def shutdown_hook() -> None: + async def second_shutdown_hook() -> None: await asyncio.sleep(0) - hook_calls.append("shutdown") + assert first_hook_completed, "shutdown hooks must not run concurrently" + hook_calls.append("second") app = create_app( ApiDependencies( uow_factory=scaffold_support.unexpected_uow_factory, - shutdown_hooks=(shutdown_hook,), + shutdown_hooks=(first_shutdown_hook, second_shutdown_hook), ) ) sent_events = await scaffold_support.run_asgi_lifespan( @@ -37,13 +47,114 @@ async def shutdown_hook() -> None: ), ) - assert hook_calls == ["shutdown"], "hook_calls must contain only shutdown" + assert hook_calls == ["first", "second"], ( + "shutdown hooks must run in supplied order" + ) assert sent_events == [ {"type": "lifespan.startup.complete"}, {"type": "lifespan.shutdown.complete"}, ], "sent_events must contain completed startup and shutdown events" +@pytest.mark.asyncio +async def test_shutdown_runs_remaining_hooks_after_failure() -> None: + """Attempt all ordered shutdown hooks before surfacing the first failure.""" + from episodic.api.app import _ShutdownHooksMiddleware + + calls: list[str] = [] + + async def failing_hook() -> None: + await asyncio.sleep(0) + calls.append("failing") + msg = "first hook failed" + raise RuntimeError(msg) + + async def later_hook() -> None: + await asyncio.sleep(0) + calls.append("later") + + middleware = _ShutdownHooksMiddleware((failing_hook, later_hook)) + + with pytest.raises(RuntimeError, match="first hook failed"): + await middleware.process_shutdown({}, {}) + + assert calls == ["failing", "later"], calls + + +@pytest.mark.asyncio +async def test_runtime_lifespan_shuts_down_generation_before_database( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Ensure runtime teardown cancels generation before disposing its database.""" + from unittest import mock + + from episodic.api import runtime as runtime_module + + monkeypatch.setenv("DATABASE_URL", "postgresql://example.test/episodic") + monkeypatch.setenv("SOURCE_INTAKE_OBJECT_STORE_ROOT", str(tmp_path)) + monkeypatch.setenv("API_AUTHORIZATION_BEARER_TOKEN", "test-token") + monkeypatch.setenv("API_AUTHORIZATION_PRINCIPAL_ID", "test-principal") + events: list[str] = [] + + async def shutdown_database() -> None: + await asyncio.sleep(0) + events.append("database.dispose") + + async def check_database() -> bool: + await asyncio.sleep(0) + return True + + async def shutdown_launcher() -> None: + await asyncio.sleep(0) + events.append("launcher.shutdown") + + async def close_llm() -> None: + await asyncio.sleep(0) + events.append("llm_port.aclose") + + def unit_of_work_factory() -> object: + return object() + + probe = runtime_module.ReadinessProbe(name="database", check=check_database) + launcher = mock.Mock() + launcher.shutdown.side_effect = shutdown_launcher + llm_port = mock.Mock() + llm_port.aclose.side_effect = close_llm + with ( + mock.patch.object( + runtime_module, + "_build_database_probe", + return_value=(probe, unit_of_work_factory, shutdown_database), + ), + mock.patch.object(runtime_module, "_build_llm_port", return_value=llm_port), + mock.patch.object( + runtime_module, + "_build_generation_launcher", + return_value=launcher, + ), + ): + app = runtime_module.create_app_from_env() + + sent_events = await scaffold_support.run_asgi_lifespan( + typ.cast("_ASGIApp", app), + ( + scaffold_support.LifespanEvent(type="lifespan.startup"), + scaffold_support.LifespanEvent(type="lifespan.shutdown"), + ), + ) + + assert events == [ + "launcher.shutdown", + "llm_port.aclose", + "database.dispose", + ], f"unexpected runtime shutdown order: {events!r}" + assert sent_events == [ + {"type": "lifespan.startup.complete"}, + {"type": "lifespan.shutdown.complete"}, + ], f"unexpected lifespan events: {sent_events!r}" + + @pytest.mark.asyncio async def test_create_app_keeps_existing_canonical_routes_working( canonical_api_async_client: httpx.AsyncClient, diff --git a/tests/test_observability.py b/tests/test_observability.py new file mode 100644 index 00000000..09771fad --- /dev/null +++ b/tests/test_observability.py @@ -0,0 +1,123 @@ +"""Focused tests for synchronous tracing adapters.""" + +import dataclasses as dc +import typing as typ + +from episodic.observability import ( + NoopTracer, + RecordingTracer, + StructuredLogMetrics, + StructuredLogTracer, +) + +if typ.TYPE_CHECKING: + from collections import abc as cabc + + +@dc.dataclass(slots=True) +class _RecordingLogger: + """Capture structured logger events without process-wide configuration.""" + + events: list[tuple[str, dict[str, str]]] = dc.field(default_factory=list) + + def info(self, message: str, /, *, extra: cabc.Mapping[str, str]) -> None: + """Record a structured INFO event.""" + self.events.append((message, dict(extra))) + + +def test_recording_tracer_preserves_span_details_and_completion() -> None: + """Recording spans keep copied attributes and complete after context exit.""" + tracer = RecordingTracer() + attributes = {"run_id": "run-42", "operation": "admit"} + + with tracer.start_span("generation_run.admission", attributes=attributes): + attributes["run_id"] = "mutated-after-start" + + assert tracer.spans[0].name == "generation_run.admission", tracer.spans + assert tracer.spans[0].attributes == { + "run_id": "run-42", + "operation": "admit", + }, tracer.spans + assert tracer.spans[0].is_completed, tracer.spans + + +def test_structured_log_tracer_allows_bounded_operation_attributes() -> None: + """Structured spans retain only allow-listed bounded operation attributes.""" + logger = _RecordingLogger() + tracer = StructuredLogTracer(logger=logger) + + with tracer.start_span( + "generation_run.admission", + attributes={ + "operation": "generation_run.admission", + "run_id": "run-42", + "access_token": "secret-value", + }, + ) as span: + span.set_attribute("outcome", "rejected") + span.set_attribute("failure_category", "launcher.overloaded") + + expected_events = [ + ( + "trace_span_started", + { + "span_name": "generation_run.admission", + "operation": "generation_run.admission", + }, + ), + ( + "trace_span_completed", + { + "span_name": "generation_run.admission", + "operation": "generation_run.admission", + "outcome": "rejected", + "failure_category": "launcher.overloaded", + }, + ), + ] + assert logger.events == expected_events, logger.events + + +def test_structured_log_metrics_emits_latency_and_value_events() -> None: + """Structured metrics retain their event names and payload fields.""" + logger = _RecordingLogger() + metrics = StructuredLogMetrics(logger=logger) + labels = {"operation": "generation_run.execute"} + + metrics.observe_latency_ms( + "generation_run.duration_ms", + 123.45, + labels=labels, + ) + metrics.observe_value( + "generation_run.queue_depth", + 2.0, + labels=labels, + ) + + expected_latency_event = ( + "metric_latency", + { + "metric_name": "generation_run.duration_ms", + "value": "123.45", + "operation": "generation_run.execute", + }, + ) + expected_value_event = ( + "metric_value", + { + "metric_name": "generation_run.queue_depth", + "value": "2.0", + "operation": "generation_run.execute", + }, + ) + + assert logger.events == [expected_latency_event, expected_value_event], ( + logger.events + ) + + +def test_noop_tracer_supports_synchronous_span_contexts() -> None: + """No-op tracing leaves synchronous operation control flow unchanged.""" + with NoopTracer().start_span("generation_run.admission", attributes={}): + pass diff --git a/tests/test_protocol_stubs.py b/tests/test_protocol_stubs.py index 01b13268..fde99e2b 100644 --- a/tests/test_protocol_stubs.py +++ b/tests/test_protocol_stubs.py @@ -75,6 +75,12 @@ async def _assert_async_stub_raises( (EpisodeRepository, "add", (None,), {}), (EpisodeRepository, "get", (None,), {}), (EpisodeRepository, "list_by_ids", ((),), {}), + ( + EpisodeRepository, + "update", + (None,), + {"update": None}, + ), (IngestionJobRepository, "add", (None,), {}), (IngestionJobRepository, "get", (None,), {}), (SourceDocumentRepository, "add", (None,), {}), diff --git a/tests/test_runtime_configuration.py b/tests/test_runtime_configuration.py new file mode 100644 index 00000000..14b043ff --- /dev/null +++ b/tests/test_runtime_configuration.py @@ -0,0 +1,124 @@ +"""Focused runtime configuration tests.""" + +import typing as typ + +import pytest + +if typ.TYPE_CHECKING: + from pathlib import Path + + +def test_load_runtime_config_uses_configured_pricing_directory( + tmp_path: "Path", # noqa: UP037 # Imported only during type checking. +) -> None: + """Pricing snapshots should be loaded from a validated configured directory.""" + from episodic.api.runtime import _load_runtime_config + + pricing_directory = tmp_path / "pricing" + pricing_directory.mkdir() + config = _load_runtime_config({ + "DATABASE_URL": "postgresql://example.test/episodic", + "SOURCE_INTAKE_OBJECT_STORE_ROOT": str(tmp_path / "objects"), + "PRICING_SNAPSHOT_DIRECTORY": str(pricing_directory), + "API_AUTHORIZATION_BEARER_TOKEN": "test-token", + "API_AUTHORIZATION_PRINCIPAL_ID": "test-principal", + "GENERATION_MAX_SOURCE_COUNT": "3", + "GENERATION_MAX_SOURCE_BYTES": "400", + "GENERATION_MAX_AGGREGATE_SOURCE_BYTES": "800", + "GENERATION_MAX_NORMALIZED_SOURCE_BYTES": "200", + "GENERATION_MAX_OUTPUT_TOKENS": "512", + "GENERATION_MAX_RESPONSE_BYTES": "4096", + }) + + assert config.pricing_snapshot_directory == pricing_directory.resolve(), ( + f"expected configured pricing path, got {config.pricing_snapshot_directory}" + ) + assert config.generation_source_limits.max_source_count == 3, ( + "expected configured source count 3, got " + f"{config.generation_source_limits.max_source_count}" + ) + assert config.generation_source_limits.max_source_bytes == 400, ( + "expected configured source bytes 400, got " + f"{config.generation_source_limits.max_source_bytes}" + ) + assert config.generation_source_limits.max_aggregate_source_bytes == 800, ( + "expected configured aggregate source bytes 800, got " + f"{config.generation_source_limits.max_aggregate_source_bytes}" + ) + assert config.generation_source_limits.max_normalized_source_bytes == 200, ( + "expected configured normalized source bytes 200, got " + f"{config.generation_source_limits.max_normalized_source_bytes}" + ) + assert config.generation_max_output_tokens == 512, ( + "expected configured output token limit 512, got " + f"{config.generation_max_output_tokens}" + ) + assert config.generation_max_response_bytes == 4096, ( + "expected configured response limit 4096, got " + f"{config.generation_max_response_bytes}" + ) + + +@pytest.mark.parametrize("value", ["0", "-1", "not-an-integer"]) +def test_load_runtime_config_rejects_invalid_generation_source_limit( + tmp_path: "Path", # noqa: UP037 # Imported only during type checking. + value: str, +) -> None: + """Generation source limits must be positive integer runtime settings.""" + from episodic.api.runtime import RuntimeConfigurationError, _load_runtime_config + + pricing_directory = tmp_path / "pricing" + pricing_directory.mkdir() + with pytest.raises( + RuntimeConfigurationError, + match="GENERATION_MAX_SOURCE_COUNT must be a positive integer", + ): + _load_runtime_config({ + "DATABASE_URL": "postgresql://example.test/episodic", + "SOURCE_INTAKE_OBJECT_STORE_ROOT": str(tmp_path / "objects"), + "PRICING_SNAPSHOT_DIRECTORY": str(pricing_directory), + "API_AUTHORIZATION_BEARER_TOKEN": "test-token", + "API_AUTHORIZATION_PRINCIPAL_ID": "test-principal", + "GENERATION_MAX_SOURCE_COUNT": value, + }) + + +@pytest.mark.parametrize( + "setting", + ["GENERATION_MAX_OUTPUT_TOKENS", "GENERATION_MAX_RESPONSE_BYTES"], +) +def test_load_runtime_config_rejects_invalid_generation_output_limit( + tmp_path: "Path", # noqa: UP037 # Imported only during type checking. + setting: str, +) -> None: + """Generation output limits must be positive integer runtime settings.""" + from episodic.api.runtime import RuntimeConfigurationError, _load_runtime_config + + pricing_directory = tmp_path / "pricing" + pricing_directory.mkdir() + with pytest.raises( + RuntimeConfigurationError, + match=f"{setting} must be a positive integer", + ): + _load_runtime_config({ + "DATABASE_URL": "postgresql://example.test/episodic", + "SOURCE_INTAKE_OBJECT_STORE_ROOT": str(tmp_path / "objects"), + "PRICING_SNAPSHOT_DIRECTORY": str(pricing_directory), + "API_AUTHORIZATION_BEARER_TOKEN": "test-token", + "API_AUTHORIZATION_PRINCIPAL_ID": "test-principal", + setting: "0", + }) + + +def test_load_runtime_config_rejects_missing_pricing_directory( + tmp_path: "Path", # noqa: UP037 # Imported only during type checking. +) -> None: + """Pricing configuration should fail before launcher construction.""" + from episodic.api.runtime import RuntimeConfigurationError, _load_runtime_config + + with pytest.raises(RuntimeConfigurationError, match="PRICING_SNAPSHOT_DIRECTORY"): + _load_runtime_config({ + "DATABASE_URL": "postgresql://example.test/episodic", + "SOURCE_INTAKE_OBJECT_STORE_ROOT": str(tmp_path / "objects"), + "PRICING_SNAPSHOT_DIRECTORY": str(tmp_path / "missing"), + }) diff --git a/tests/test_runtime_metrics_wiring.py b/tests/test_runtime_metrics_wiring.py new file mode 100644 index 00000000..0866280d --- /dev/null +++ b/tests/test_runtime_metrics_wiring.py @@ -0,0 +1,145 @@ +"""Tests for production metrics wiring in the runtime composition root.""" + +import typing as typ +from unittest import mock + +import httpx +import pytest + +import tests.test_http_service_scaffold_support as scaffold_support + +if typ.TYPE_CHECKING: + import collections.abc as cabc + from pathlib import Path + + from httpx._transports.asgi import _ASGIApp + + from episodic.api.dependencies import ApiDependencies + + +class _RecordingMetrics: + """Capture bounded route observations for assertion.""" + + def __init__(self) -> None: + self.counters: list[tuple[str, dict[str, str]]] = [] + self.latencies: list[tuple[str, float, dict[str, str]]] = [] + + def increment_counter( + self, + name: str, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Record one bounded counter increment.""" + self.counters.append((name, dict(labels))) + + def observe_latency_ms( + self, + name: str, + value: float, + *, + labels: cabc.Mapping[str, str], + ) -> None: + """Record one bounded latency observation.""" + self.latencies.append((name, value, dict(labels))) + + +class _SteppingMonotonicClock: + """Return deterministic monotonic timestamps for request timing tests.""" + + def __init__(self, timestamps: cabc.Iterator[float]) -> None: + self._timestamps = timestamps + + def monotonic_seconds(self) -> float: + """Return the next configured timestamp.""" + return next(self._timestamps) + + +@pytest.mark.asyncio +async def test_create_app_from_env_shares_production_metrics_sink( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Runtime-created UoWs and launchers should share the production sink.""" + from episodic.api import runtime as runtime_module + from episodic.generation import InProcessGenerationRunLauncher + from episodic.observability import NoopMetrics, StructuredLogMetrics + + monkeypatch.setenv("DATABASE_URL", "postgresql://example.test/episodic") + monkeypatch.setenv("SOURCE_INTAKE_OBJECT_STORE_ROOT", str(tmp_path)) + monkeypatch.setenv("API_AUTHORIZATION_BEARER_TOKEN", "runtime-test-token") + monkeypatch.setenv("API_AUTHORIZATION_PRINCIPAL_ID", "runtime-test-principal") + monkeypatch.setenv("OPENAI_BASE_URL", "https://llm.example.test/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + captured_dependencies: ApiDependencies | None = None + + def capture_dependencies(dependencies: ApiDependencies) -> object: + nonlocal captured_dependencies + captured_dependencies = dependencies + return object() + + with ( + mock.patch.object( + runtime_module, + "create_app", + side_effect=capture_dependencies, + ), + mock.patch.object( + runtime_module, + "SqlAlchemyUnitOfWork", + autospec=True, + ) as unit_of_work_constructor, + ): + runtime_module.create_app_from_env() + assert captured_dependencies is not None, ( + "expected captured dependencies, got None" + ) + captured_dependencies.uow_factory() + uow_metrics = unit_of_work_constructor.call_args.kwargs["metrics"] + + assert captured_dependencies is not None, "expected captured dependencies, got None" + assert isinstance(captured_dependencies.launcher, InProcessGenerationRunLauncher), ( + "expected an in-process launcher, got " + f"{type(captured_dependencies.launcher).__name__}" + ) + + assert isinstance(uow_metrics, StructuredLogMetrics), ( + f"expected structured-log metrics, got {type(uow_metrics).__name__}" + ) + assert not isinstance(uow_metrics, NoopMetrics), ( + "expected a production metrics sink" + ) + assert captured_dependencies.launcher.metrics is uow_metrics, ( + "runtime-created UoWs and launcher should share one metrics sink" + ) + + await captured_dependencies.shutdown_hooks[0]() + + +@pytest.mark.asyncio +async def test_generation_route_metrics_use_injected_monotonic_clock() -> None: + """Generation-route latency uses the dependency-injected clock seam.""" + from episodic.api import ApiDependencies, create_app + + metrics = _RecordingMetrics() + clock = _SteppingMonotonicClock(iter((10.0, 10.25))) + dependencies = ApiDependencies( + uow_factory=scaffold_support.unexpected_uow_factory, + metrics=metrics, + monotonic_clock=clock, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + response = await client.get("/v1/generation-runs/not-a-uuid") + + expected_labels = {"operation": "generation_run.read", "outcome": "rejected"} + assert response.status_code == 400, response.text + assert metrics.counters == [("generation_api_request_total", expected_labels)], ( + metrics.counters + ) + assert metrics.latencies == [ + ("generation_api_request_latency_ms", 250.0, expected_labels) + ], metrics.latencies diff --git a/tests/test_runtime_object_store_wiring.py b/tests/test_runtime_object_store_wiring.py new file mode 100644 index 00000000..2b87c8ab --- /dev/null +++ b/tests/test_runtime_object_store_wiring.py @@ -0,0 +1,70 @@ +"""Integration coverage for runtime-configured upload storage.""" + +import hashlib +import typing as typ + +import httpx +import pytest + +import tests.test_http_service_scaffold_support as scaffold_support + +if typ.TYPE_CHECKING: + from pathlib import Path + + from httpx._transports.asgi import _ASGIApp + + +@pytest.mark.asyncio +async def test_create_app_from_env_wires_object_store_for_uploads( + migrated_database_url: str, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Runtime-created apps accept uploads when object storage is configured.""" + object_store_root = tmp_path / "objects" + monkeypatch.setenv("DATABASE_URL", migrated_database_url) + monkeypatch.setenv("SOURCE_INTAKE_OBJECT_STORE_ROOT", str(object_store_root)) + monkeypatch.setenv("API_AUTHORIZATION_BEARER_TOKEN", "runtime-test-token") + monkeypatch.setenv("API_AUTHORIZATION_PRINCIPAL_ID", "runtime-test-principal") + + from episodic.api.runtime import create_app_from_env + + app = create_app_from_env() + try: + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", app)) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + payload = b"runtime upload\n" + response = await client.post( + "/v1/uploads", + headers={ + "Authorization": "Bearer runtime-test-token", + "Idempotency-Key": "runtime-upload", + }, + files={ + "file": ("source.txt", payload, "text/plain"), + "content_type": (None, "text/plain"), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + }, + ) + finally: + await scaffold_support.run_asgi_lifespan( + typ.cast("_ASGIApp", app), + ( + scaffold_support.LifespanEvent(type="lifespan.startup"), + scaffold_support.LifespanEvent(type="lifespan.shutdown"), + ), + ) + + assert response.status_code == 201, response.text + response_body = response.json() + expected_hash = hashlib.sha256(payload).hexdigest() + stored_path = object_store_root / "uploads" / response_body["id"] + assert response_body["content_hash"] == f"sha256:{expected_hash}", ( + "Expected values to match" + ) + assert stored_path.is_file(), f"expected upload payload at {stored_path}" + assert stored_path.read_bytes() == payload, "Expected values to match" diff --git a/tests/test_source_idempotency_encoding.py b/tests/test_source_idempotency_encoding.py new file mode 100644 index 00000000..fefc8205 --- /dev/null +++ b/tests/test_source_idempotency_encoding.py @@ -0,0 +1,16 @@ +"""Unit tests for source-intake idempotency response encoding.""" + +import pytest + +from episodic.api.source_idempotency import IdempotentResponse, _encode_outcome + + +def test_idempotency_outcome_encoding_rejects_large_payloads() -> None: + """Idempotency replay envelopes are capped at 64 KiB.""" + response = IdempotentResponse( + "201 Created", + {"content": "x" * (64 * 1024)}, + ) + + with pytest.raises(ValueError, match="64 KiB"): + _encode_outcome(response) diff --git a/tests/test_source_intake_api.py b/tests/test_source_intake_api.py index 25f0cec6..aff01251 100644 --- a/tests/test_source_intake_api.py +++ b/tests/test_source_intake_api.py @@ -1,6 +1,7 @@ """Integration tests for the source-intake REST workflow.""" import asyncio +import contextlib import dataclasses import datetime as dt import hashlib @@ -17,7 +18,6 @@ ) from episodic.api.source_idempotency import ( IdempotentResponse, - _encode_outcome, _idempotent_response, ) from episodic.canonical.idempotency import Acquired, IdempotencyAcquireRequest @@ -26,6 +26,7 @@ from tests.fixtures.api import build_api_dependencies if typ.TYPE_CHECKING: + import collections.abc as cabc from pathlib import Path from httpx._transports.asgi import _ASGIApp @@ -45,19 +46,35 @@ async def decide(self, context: AuthorizationContext) -> AuthorizationResult: ) -@pytest.mark.asyncio -async def test_source_intake_upload_job_and_attach_flow( +@contextlib.asynccontextmanager +async def _source_intake_client( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, -) -> None: - """Client can upload bytes, create a job, attach the upload, and poll ready.""" - object_store = FilesystemObjectStore(tmp_path / "objects") - dependencies = build_api_dependencies(session_factory, object_store=object_store) + *, + headers: dict[str, str] | None = None, +) -> cabc.AsyncIterator[httpx.AsyncClient]: + """Yield an authenticated source-intake API client.""" + dependencies = build_api_dependencies( + session_factory, + authorization=HeaderPrincipalAuthorization(), + object_store=FilesystemObjectStore(tmp_path / "objects"), + ) transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) async with httpx.AsyncClient( transport=transport, base_url="http://testserver", + headers={"Authorization": "principal-a"} if headers is None else headers, ) as client: + yield client + + +@pytest.mark.asyncio +async def test_source_intake_upload_job_and_attach_flow( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """Client can upload bytes, create a job, attach the upload, and poll ready.""" + async with _source_intake_client(session_factory, tmp_path) as client: profile_id = await _create_series_profile(client) upload_response = await _post_text_upload( client, @@ -121,13 +138,7 @@ async def test_source_intake_idempotency_conflict( tmp_path: Path, ) -> None: """Same idempotency key with different upload body returns 409.""" - object_store = FilesystemObjectStore(tmp_path / "objects") - dependencies = build_api_dependencies(session_factory, object_store=object_store) - transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) - async with httpx.AsyncClient( - transport=transport, - base_url="http://testserver", - ) as client: + async with _source_intake_client(session_factory, tmp_path) as client: first_request = _TextUploadRequest(key="conflict-key", payload=b"hello\n") second_request = _TextUploadRequest(key="conflict-key", payload=b"bye\n") first = await _post_text_upload(client, first_request) @@ -144,16 +155,10 @@ async def test_source_intake_idempotency_is_scoped_by_authorized_principal( tmp_path: Path, ) -> None: """The authorization principal scopes idempotency records for upload replay.""" - object_store = FilesystemObjectStore(tmp_path / "objects") - dependencies = build_api_dependencies( + async with _source_intake_client( session_factory, - authorization=HeaderPrincipalAuthorization(), - object_store=object_store, - ) - transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) - async with httpx.AsyncClient( - transport=transport, - base_url="http://testserver", + tmp_path, + headers={}, ) as client: principal_a_request = _TextUploadRequest( key="principal-key", @@ -184,11 +189,16 @@ async def test_source_intake_response_envelope_snapshot( ) -> None: """Snapshot stable fields from source-intake response envelopes.""" object_store = FilesystemObjectStore(tmp_path / "objects") - dependencies = build_api_dependencies(session_factory, object_store=object_store) + dependencies = build_api_dependencies( + session_factory, + authorization=HeaderPrincipalAuthorization(), + object_store=object_store, + ) transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) async with httpx.AsyncClient( transport=transport, base_url="http://testserver", + headers={"Authorization": "principal-a"}, ) as client: profile_id = await _create_series_profile(client) upload_response = await _post_text_upload( @@ -285,17 +295,6 @@ async def test_idempotent_response_allows_retry_after_work_failure( ) -def test_idempotency_outcome_encoding_rejects_large_payloads() -> None: - """Idempotency replay envelopes are capped at 64 KiB.""" - response = IdempotentResponse( - "201 Created", - {"content": "x" * (64 * 1024)}, - ) - - with pytest.raises(ValueError, match="64 KiB"): - _encode_outcome(response) - - async def _create_series_profile(client: httpx.AsyncClient) -> str: """Create a series profile through the public API and return its id.""" response = await client.post( diff --git a/tests/test_source_intake_api_contract.py b/tests/test_source_intake_api_contract.py index 00d75845..8e507f44 100644 --- a/tests/test_source_intake_api_contract.py +++ b/tests/test_source_intake_api_contract.py @@ -1,27 +1,50 @@ """Contract tests for source-intake REST error paths and read endpoints.""" -import contextlib -import datetime as dt +import dataclasses as dc import hashlib import typing as typ import uuid import httpx import pytest +import sqlalchemy as sa from episodic.api import create_app -from episodic.canonical.storage import FilesystemObjectStore, SqlAlchemyUnitOfWork -from episodic.canonical.uploads import Upload, UploadState +from episodic.canonical.storage import ( + FilesystemObjectStore, + IngestionJobRecord, +) from tests.fixtures.api import build_api_dependencies +from tests.test_source_intake_api_contract_support import ( + _create_ingestion_job, + _create_pending_upload, + _create_profile_and_job, + _create_ready_upload, + _create_series_profile, + _post_attach_source, + _post_text_upload, + _source_intake_client, + _source_uri_payload, + _upload_payload, +) if typ.TYPE_CHECKING: - import collections.abc as cabc from pathlib import Path from httpx._transports.asgi import _ASGIApp from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +@dc.dataclass(frozen=True, slots=True) +class _InvalidUploadExpectation: + """Expected result for a rejected source-upload attachment.""" + + create_pending_upload: bool + idempotency_key: str + status_code: int + code: str + + def _assert_api_error( response: httpx.Response, *, @@ -122,40 +145,53 @@ async def test_attach_source_reports_missing_ingestion_job( @pytest.mark.asyncio -async def test_attach_upload_reports_missing_upload( - session_factory: async_sessionmaker[AsyncSession], - tmp_path: Path, -) -> None: - """Attaching an unknown upload to a known job returns upload_not_found.""" - job_id = await _create_profile_and_job(session_factory, tmp_path) - async with _source_intake_client(session_factory, tmp_path) as client: - response = await _post_attach_source( - client, - job_id, - idempotency_key="missing-upload", - payload=_upload_payload(str(uuid.uuid4())), - ) - - _assert_api_error(response, status_code=404, code="upload_not_found") - - -@pytest.mark.asyncio -async def test_attach_upload_reports_not_ready_upload( +@pytest.mark.parametrize( + "expected", + [ + pytest.param( + _InvalidUploadExpectation( + create_pending_upload=False, + idempotency_key="missing-upload", + status_code=404, + code="upload_not_found", + ), + id="missing_upload", + ), + pytest.param( + _InvalidUploadExpectation( + create_pending_upload=True, + idempotency_key="pending-upload", + status_code=409, + code="upload_not_ready", + ), + id="not_ready_upload", + ), + ], +) +async def test_attach_upload_reports_invalid_upload( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, + expected: _InvalidUploadExpectation, ) -> None: - """Attaching a pending upload returns upload_not_ready.""" + """Upload attachment rejects missing and non-ready uploads.""" job_id = await _create_profile_and_job(session_factory, tmp_path) - upload_id = await _create_pending_upload(session_factory) + if expected.create_pending_upload: + upload_id = await _create_pending_upload(session_factory) + else: + upload_id = uuid.uuid4() async with _source_intake_client(session_factory, tmp_path) as client: response = await _post_attach_source( client, job_id, - idempotency_key="pending-upload", + idempotency_key=expected.idempotency_key, payload=_upload_payload(str(upload_id)), ) - _assert_api_error(response, status_code=409, code="upload_not_ready") + _assert_api_error( + response, + status_code=expected.status_code, + code=expected.code, + ) @pytest.mark.asyncio @@ -212,184 +248,134 @@ async def test_upload_get_endpoint_reports_missing_upload( @pytest.mark.asyncio -async def test_ingestion_job_sources_get_endpoint_lists_sources( +async def test_source_intake_hides_other_principals_resources( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, ) -> None: - """GET /v1/ingestion-jobs/{job_id}/sources returns a paged source list.""" - async with _source_intake_client(session_factory, tmp_path) as client: - profile_id = await _create_series_profile(client) - upload = await _post_text_upload(client, key="list-source-upload", payload=b"x") - job_id = await _create_ingestion_job(client, profile_id) - attach = await client.post( - f"/v1/ingestion-jobs/{job_id}/sources", - headers={"Idempotency-Key": "list-source"}, - json=_upload_payload(typ.cast("str", upload.json()["id"])), + """A different principal cannot read or attach another owner's intake data.""" + async with _source_intake_client(session_factory, tmp_path) as owner: + profile_id = await _create_series_profile(owner) + upload = await _post_text_upload(owner, key="owned-upload", payload=b"x") + job_id = await _create_ingestion_job(owner, profile_id) + attached = await _post_attach_source( + owner, + job_id, + idempotency_key="owned-source", + payload=_upload_payload(typ.cast("str", upload.json()["id"])), ) - response = await client.get(f"/v1/ingestion-jobs/{job_id}/sources") + assert attached.status_code == 201, attached.text - assert attach.status_code == 201, attach.text - assert response.status_code == 200, "source-list GET must return HTTP 200" - assert response.json()["total"] == 1, "source-list total must equal one" - assert response.json()["items"][0]["upload_id"] == upload.json()["id"], ( - "listed source upload_id must identify the attached upload" - ) + async with _source_intake_client( + session_factory, + tmp_path, + principal="principal-b", + ) as other: + upload_response = await other.get(f"/v1/uploads/{upload.json()['id']}") + job_response = await other.get(f"/v1/ingestion-jobs/{job_id}") + sources_response = await other.get(f"/v1/ingestion-jobs/{job_id}/sources") + attach_response = await _post_attach_source( + other, + job_id, + idempotency_key="other-source", + payload=_source_uri_payload(), + ) + listing_response = await other.get("/v1/ingestion-jobs") + + for response, code in ( + (upload_response, "upload_not_found"), + (job_response, "ingestion_job_not_found"), + (sources_response, "ingestion_job_not_found"), + (attach_response, "ingestion_job_not_found"), + ): + _assert_api_error(response, status_code=404, code=code) + assert listing_response.status_code == 200, listing_response.text + assert listing_response.json()["items"] == [], listing_response.json() @pytest.mark.asyncio -async def test_ingestion_job_sources_get_reports_missing_job( +async def test_source_intake_rejects_client_target_episode_id( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, ) -> None: - """GET /v1/ingestion-jobs/{job_id}/sources reports unknown jobs.""" + """Client-selected episode identifiers do not create ingestion jobs.""" async with _source_intake_client(session_factory, tmp_path) as client: - response = await client.get(f"/v1/ingestion-jobs/{uuid.uuid4()}/sources") + profile_id = await _create_series_profile(client) + response = await client.post( + "/v1/ingestion-jobs", + headers={"Idempotency-Key": "client-target"}, + json={ + "series_profile_id": profile_id, + "target_episode_id": str(uuid.uuid7()), + }, + ) - _assert_api_error( - response, - status_code=404, - code="ingestion_job_not_found", - ) + _assert_api_error(response, status_code=400, code="validation_error") + assert response.json()["details"] == { + "field": "target_episode_id", + "constraint": "unsupported", + }, response.text + async with session_factory() as session: + count = await session.scalar(sa.select(sa.func.count(IngestionJobRecord.id))) + assert count == 0, f"ingestion jobs created after rejected request: {count}" -@contextlib.asynccontextmanager -async def _source_intake_client( +@pytest.mark.asyncio +async def test_permit_all_composition_hides_named_upload_without_principal( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, - *, - upload_max_bytes: int | None = None, -) -> cabc.AsyncIterator[httpx.AsyncClient]: - """Yield an async client with source-intake object storage configured.""" - object_store = FilesystemObjectStore(tmp_path / "objects") +) -> None: + """A missing principal cannot read a named upload under PermitAll tests.""" + upload_id = await _create_ready_upload(session_factory, owner="principal-a") dependencies = build_api_dependencies( session_factory, - object_store=object_store, - upload_max_bytes=upload_max_bytes, + object_store=FilesystemObjectStore(tmp_path / "objects"), ) transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) async with httpx.AsyncClient( transport=transport, base_url="http://testserver", ) as client: - yield client - - -async def _create_series_profile(client: httpx.AsyncClient) -> str: - """Create a series profile through the public API and return its id.""" - response = await client.post( - "/v1/series-profiles", - json={ - "slug": f"source-intake-{uuid.uuid4()}", - "title": "Source Intake", - "description": "Created for intake contract tests.", - "configuration": {"tone": "clear"}, - "guardrails": {"instruction": "Keep claims sourced."}, - "actor": "api-user@example.com", - "note": "Initial profile", - }, - ) - assert response.status_code == 201, response.text - return typ.cast("str", response.json()["id"]) - + response = await client.get(f"/v1/uploads/{upload_id}") -async def _create_ingestion_job(client: httpx.AsyncClient, profile_id: str) -> str: - """Create an ingestion job through the public API and return its id.""" - response = await client.post( - "/v1/ingestion-jobs", - headers={"Idempotency-Key": f"job-{uuid.uuid4()}"}, - json={"series_profile_id": profile_id, "target_episode_id": None}, - ) - assert response.status_code == 201, response.text - return typ.cast("str", response.json()["id"]) + _assert_api_error(response, status_code=404, code="upload_not_found") -async def _create_profile_and_job( +@pytest.mark.asyncio +async def test_ingestion_job_sources_get_endpoint_lists_sources( session_factory: async_sessionmaker[AsyncSession], tmp_path: Path, -) -> str: - """Create a series profile and an ingestion job; return the job id.""" +) -> None: + """GET /v1/ingestion-jobs/{job_id}/sources returns a paged source list.""" async with _source_intake_client(session_factory, tmp_path) as client: profile_id = await _create_series_profile(client) - return await _create_ingestion_job(client, profile_id) - + upload = await _post_text_upload(client, key="list-source-upload", payload=b"x") + job_id = await _create_ingestion_job(client, profile_id) + attach = await client.post( + f"/v1/ingestion-jobs/{job_id}/sources", + headers={"Idempotency-Key": "list-source"}, + json=_upload_payload(typ.cast("str", upload.json()["id"])), + ) + response = await client.get(f"/v1/ingestion-jobs/{job_id}/sources") -async def _create_pending_upload( - session_factory: async_sessionmaker[AsyncSession], -) -> uuid.UUID: - """Persist one pending upload for not-ready attach tests.""" - now = dt.datetime.now(dt.UTC) - upload = Upload( - id=uuid.uuid4(), - owner_principal_id="api-user", - content_type="text/plain", - declared_size=1, - actual_size=None, - declared_sha256=None, - content_hash=None, - storage_key=f"uploads/{uuid.uuid4()}", - state=UploadState.PENDING, - metadata={}, - created_at=now, - updated_at=now, + assert attach.status_code == 201, attach.text + assert response.status_code == 200, "source-list GET must return HTTP 200" + assert response.json()["total"] == 1, "source-list total must equal one" + assert response.json()["items"][0]["upload_id"] == upload.json()["id"], ( + "listed source upload_id must identify the attached upload" ) - async with SqlAlchemyUnitOfWork(session_factory) as uow: - await uow.uploads.add(upload) - await uow.commit() - return upload.id -async def _post_attach_source( - client: httpx.AsyncClient, - job_id: str, - *, - idempotency_key: str, - payload: dict[str, object], -) -> httpx.Response: - """POST a source attachment to an ingestion job and return the response.""" - return await client.post( - f"/v1/ingestion-jobs/{job_id}/sources", - headers={"Idempotency-Key": idempotency_key}, - json=payload, - ) - +@pytest.mark.asyncio +async def test_ingestion_job_sources_get_reports_missing_job( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """GET /v1/ingestion-jobs/{job_id}/sources reports unknown jobs.""" + async with _source_intake_client(session_factory, tmp_path) as client: + response = await client.get(f"/v1/ingestion-jobs/{uuid.uuid4()}/sources") -async def _post_text_upload( - client: httpx.AsyncClient, - *, - key: str, - payload: bytes, - content_type: str = "text/plain", -) -> httpx.Response: - """Post a deterministic text upload multipart request.""" - return await client.post( - "/v1/uploads", - headers={"Idempotency-Key": key}, - files={ - "file": ("source.txt", payload, "text/plain"), - "content_type": (None, content_type), - "declared_size": (None, str(len(payload))), - "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), - }, + _assert_api_error( + response, + status_code=404, + code="ingestion_job_not_found", ) - - -def _upload_payload(upload_id: str) -> dict[str, object]: - """Return a valid upload-source attachment payload.""" - return { - "type": "upload", - "upload_id": upload_id, - "source_type": "research_paper", - "weight": 1.0, - "metadata": {"language": "en"}, - } - - -def _source_uri_payload() -> dict[str, object]: - """Return a valid URI-source attachment payload.""" - return { - "type": "source_uri", - "source_uri": "https://example.test/source.txt", - "source_type": "research_paper", - "weight": 1.0, - "metadata": {"language": "en"}, - } diff --git a/tests/test_source_intake_api_contract_support.py b/tests/test_source_intake_api_contract_support.py new file mode 100644 index 00000000..6370cf66 --- /dev/null +++ b/tests/test_source_intake_api_contract_support.py @@ -0,0 +1,215 @@ +"""Support fixtures for source-intake API contract tests.""" + +import contextlib +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +import httpx + +from episodic.api import create_app +from episodic.canonical.storage import FilesystemObjectStore, SqlAlchemyUnitOfWork +from episodic.canonical.uploads import Upload, UploadState +from tests.fixtures.api import build_api_dependencies +from tests.fixtures.generation_run_api import HeaderPrincipalAuthorization + +if typ.TYPE_CHECKING: + import collections.abc as cabc + from pathlib import Path + + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@dc.dataclass(frozen=True, slots=True) +class _UploadFixtureState: + """Storage values that distinguish pending and ready upload fixtures.""" + + state: UploadState + actual_size: int | None + content_hash: str | None + + +@contextlib.asynccontextmanager +async def _source_intake_client( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, + *, + upload_max_bytes: int | None = None, + principal: str = "principal-a", +) -> cabc.AsyncIterator[httpx.AsyncClient]: + """Yield an async client with source-intake object storage configured.""" + dependencies = build_api_dependencies( + session_factory, + authorization=HeaderPrincipalAuthorization(), + object_store=FilesystemObjectStore(tmp_path / "objects"), + upload_max_bytes=upload_max_bytes, + ) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + headers={"Authorization": f"Bearer {principal}"}, + ) as client: + yield client + + +async def _create_series_profile(client: httpx.AsyncClient) -> str: + """Create a series profile through the public API and return its id.""" + response = await client.post( + "/v1/series-profiles", + headers={"Idempotency-Key": f"profile-{uuid.uuid4()}"}, + json={ + "slug": f"source-intake-{uuid.uuid4()}", + "title": "Source Intake", + "description": "Created for intake contract tests.", + "configuration": {"tone": "clear"}, + "guardrails": {"instruction": "Keep claims sourced."}, + "actor": "api-user@example.com", + "note": "Initial profile", + }, + ) + assert response.status_code == 201, response.text + return typ.cast("str", response.json()["id"]) + + +async def _create_ingestion_job(client: httpx.AsyncClient, profile_id: str) -> str: + """Create an ingestion job through the public API and return its id.""" + response = await client.post( + "/v1/ingestion-jobs", + headers={"Idempotency-Key": f"job-{uuid.uuid4()}"}, + json={"series_profile_id": profile_id, "target_episode_id": None}, + ) + assert response.status_code == 201, response.text + return typ.cast("str", response.json()["id"]) + + +async def _create_profile_and_job( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> str: + """Create a series profile and ingestion job; return the job id.""" + async with _source_intake_client(session_factory, tmp_path) as client: + return await _create_ingestion_job( + client, + await _create_series_profile(client), + ) + + +async def _create_pending_upload( + session_factory: async_sessionmaker[AsyncSession], +) -> uuid.UUID: + """Persist one pending upload for not-ready attach tests.""" + return await _create_upload( + session_factory, + owner="principal-a", + expected=_UploadFixtureState( + state=UploadState.PENDING, + actual_size=None, + content_hash=None, + ), + ) + + +async def _create_upload( + session_factory: async_sessionmaker[AsyncSession], + *, + owner: str, + expected: _UploadFixtureState, +) -> uuid.UUID: + """Persist one upload fixture and return its identifier.""" + now = dt.datetime.now(dt.UTC) + upload = Upload( + id=uuid.uuid4(), + owner_principal_id=owner, + content_type="text/plain", + declared_size=1, + actual_size=expected.actual_size, + declared_sha256=None, + content_hash=expected.content_hash, + storage_key=f"uploads/{uuid.uuid4()}", + state=expected.state, + metadata={}, + created_at=now, + updated_at=now, + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + await uow.uploads.add(upload) + await uow.commit() + return upload.id + + +async def _create_ready_upload( + session_factory: async_sessionmaker[AsyncSession], + *, + owner: str, +) -> uuid.UUID: + """Persist one owner-bound upload for metadata access checks.""" + return await _create_upload( + session_factory, + owner=owner, + expected=_UploadFixtureState( + state=UploadState.READY, + actual_size=1, + content_hash="sha256:upload", + ), + ) + + +async def _post_attach_source( + client: httpx.AsyncClient, + job_id: str, + *, + idempotency_key: str, + payload: dict[str, object], +) -> httpx.Response: + """POST a source attachment to an ingestion job and return the response.""" + return await client.post( + f"/v1/ingestion-jobs/{job_id}/sources", + headers={"Idempotency-Key": idempotency_key}, + json=payload, + ) + + +async def _post_text_upload( + client: httpx.AsyncClient, + *, + key: str, + payload: bytes, + content_type: str = "text/plain", +) -> httpx.Response: + """Post a deterministic text upload multipart request.""" + return await client.post( + "/v1/uploads", + headers={"Idempotency-Key": key}, + files={ + "file": ("source.txt", payload, content_type), + "content_type": (None, content_type), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + }, + ) + + +def _upload_payload(upload_id: str) -> dict[str, object]: + """Return a valid upload-source attachment payload.""" + return { + "type": "upload", + "upload_id": upload_id, + "source_type": "research_paper", + "weight": 1.0, + "metadata": {"language": "en"}, + } + + +def _source_uri_payload() -> dict[str, object]: + """Return a valid URI-source attachment payload.""" + return { + "type": "source_uri", + "source_uri": "https://example.test/source.txt", + "source_type": "research_paper", + "weight": 1.0, + "metadata": {"language": "en"}, + } diff --git a/tests/test_workflow_test_utils.py b/tests/test_workflow_test_utils.py index ca5a9cd9..106346a4 100644 --- a/tests/test_workflow_test_utils.py +++ b/tests/test_workflow_test_utils.py @@ -1,6 +1,11 @@ """Tests for workflow integration test helpers.""" -from tests.test_workflow_utils import artifact_server_addr, artifact_server_port +from tests.test_workflow_utils import ( + _failed_act_steps, + _has_unsupported_artifact_protocol, + artifact_server_addr, + artifact_server_port, +) def test_artifact_server_binds_for_rootless_podman_job_containers() -> None: @@ -14,3 +19,56 @@ def test_artifact_server_binds_for_rootless_podman_job_containers() -> None: assert 0 < port < 65536, ( f"artifact_server_port() returned invalid port {port}; expected 1-65535." ) + + +def test_artifact_protocol_detection_is_narrow() -> None: + """Skip only when the artifact protocol is the sole failed act step.""" + unsupported_logs = ( + '{"level":"error","job":"validate","step":"Upload artifact",' + r'"msg":"Error decode request body: unknown field \"mime_type\""}' + ) + assert _has_unsupported_artifact_protocol(unsupported_logs), ( + "Expected the upload-artifact mime_type incompatibility to be recognized." + ) + assert not _has_unsupported_artifact_protocol( + '{"level":"error","job":"validate","step":"Upload artifact",' + r'"msg":"Error decode request body: unknown field \"content_type\""}' + ), "An unrelated artifact-server error must remain a workflow failure." + + +def test_artifact_protocol_detection_preserves_other_failed_steps() -> None: + """A second failed step must prevent an artifact compatibility skip.""" + logs = "\n".join(( + ( + '{"level":"error","job":"validate","step":"Upload artifact",' + r'"msg":"Error decode request body: unknown field \"mime_type\""}' + ), + ( + '{"level":"error","job":"validate","step":"Run validation",' + '"msg":"validation command failed"}' + ), + )) + + assert not _has_unsupported_artifact_protocol(logs), ( + "A separate failed step must remain visible as a workflow failure." + ) + + +def test_failed_act_steps_preserves_message_fallback_type_validation() -> None: + """Structured act parsing should retain the existing msg fallback semantics.""" + logs = "\n".join(( + '{"level":"error","job":"build","step":"compile","message":"fallback message"}', + ( + '{"level":"error","job":"build","step":"compile",' + '"msg":"","message":"empty-msg fallback"}' + ), + ( + '{"level":"error","job":"build","step":"compile",' + '"msg":7,"message":"must be ignored"}' + ), + '{"level":"error","job":7,"step":"compile","msg":"must be ignored"}', + )) + + assert _failed_act_steps(logs) == { + ("build", "compile"): ["fallback message", "empty-msg fallback"] + }, _failed_act_steps(logs) diff --git a/tests/test_workflow_utils.py b/tests/test_workflow_utils.py index 875de402..232582d0 100644 --- a/tests/test_workflow_utils.py +++ b/tests/test_workflow_utils.py @@ -14,7 +14,7 @@ from tests.utils import podman_socket_path ACT_RUNNER_IMAGE = "catthehacker/ubuntu:act-latest" - +_UNSUPPORTED_ARTIFACT_FIELD = 'unknown field "mime_type"' if typ.TYPE_CHECKING: from pathlib import Path @@ -233,6 +233,45 @@ def _run_act_subprocess(cmd: list[str], env: dict[str, str]) -> tuple[int, str]: return completed.returncode, completed.stdout + "\n" + completed.stderr +def _failed_act_steps(logs: str) -> dict[tuple[str, str], list[str]]: + """Return structured act error messages grouped by failed job step.""" + failed_steps: dict[tuple[str, str], list[str]] = {} + for line in logs.splitlines(): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + match entry: + case { + "level": "error", + "job": str() as job, + "step": str() as step, + **fields, + }: + match fields.get("msg"): + case str() as message if message: + failed_steps.setdefault((job, step), []).append(message) + case value if value: + continue + case _: + match fields.get("message"): + case str() as message: + failed_steps.setdefault((job, step), []).append(message) + return failed_steps + + +def _has_unsupported_artifact_protocol(logs: str) -> bool: + """Return whether the only failed act step is the artifact protocol error.""" + failed_steps = _failed_act_steps(logs) + if len(failed_steps) != 1: + return False + return any( + _UNSUPPORTED_ARTIFACT_FIELD in message + for messages in failed_steps.values() + for message in messages + ) + + def run_act( *, job_name: str, @@ -268,7 +307,13 @@ def run_act( ] env = os.environ.copy() env["DOCKER_HOST"] = socket_uri - return _run_act_subprocess(cmd, env) + returncode, logs = _run_act_subprocess(cmd, env) + if returncode != 0 and _has_unsupported_artifact_protocol(logs): + pytest.skip( + "installed act artifact server does not support " + "actions/upload-artifact's current request schema." + ) + return returncode, logs def _find_in_zips( diff --git a/typos.local.toml b/typos.local.toml index a4fc03c3..91b1f840 100644 --- a/typos.local.toml +++ b/typos.local.toml @@ -28,6 +28,8 @@ ignore = [ # The documentation style guide cites this API field as its own example of # keeping United States spelling inside code. "`color`", + # Public service name whose British spelling is part of the Python API. + "\\bmaterialise_episode_from_ingestion\\b", ] [files] diff --git a/typos.toml b/typos.toml index d0c21156..daf54207 100644 --- a/typos.toml +++ b/typos.toml @@ -34,6 +34,7 @@ extend-ignore-re = [ "--artifact-server-path\\b", "", "\\bact-artifacts\\b", + "\\bmaterialise_episode_from_ingestion\\b", "\\brust-analyzer\\b", "\\bserialised_outcome\\b", "\\bspeech_render_artifacts\\b",