diff --git a/Makefile b/Makefile index a87a332e..1122699b 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,11 @@ TOOLS = $(MDFORMAT_ALL) VENV_TOOLS = pytest UV_ENV = PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 UV_CACHE_DIR=.uv-cache UV_TOOL_DIR=.uv-tools PYTEST_XDIST_WORKERS ?= 1 +ifeq ($(PYTEST_XDIST_WORKERS),1) +PYTEST_XDIST_ARGS := +else +PYTEST_XDIST_ARGS := -n $(PYTEST_XDIST_WORKERS) +endif PYLINT_PYTHON ?= pypy PYLINT_TARGETS ?= alembic episodic openai_test_types.py tests PYLINT_PYPY_SHIM_REF ?= 726d09f968b4d729ee4b29c71fc732e744854f3b @@ -92,7 +97,7 @@ nixie: ## Validate Mermaid diagrams $(NIXIE) --no-sandbox test: build crosshair $(VENV_TOOLS) ## Run tests - $(UV_ENV) $(UV) run pytest -v -n $(PYTEST_XDIST_WORKERS) + $(UV_ENV) $(UV) run pytest -v $(PYTEST_XDIST_ARGS) check-migrations: build $(VENV_TOOLS) ## Check for schema drift between models and migrations $(UV_ENV) $(UV) run python -m episodic.canonical.storage.migration_check diff --git a/alembic/versions/20260610_000009_add_source_intake_tables.py b/alembic/versions/20260610_000009_add_source_intake_tables.py new file mode 100644 index 00000000..8236e900 --- /dev/null +++ b/alembic/versions/20260610_000009_add_source_intake_tables.py @@ -0,0 +1,212 @@ +"""Add source-intake upload and idempotency tables.""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "20260610_000009" +down_revision = "20260508_000008" +branch_labels = None +depends_on = None + + +def _enum(name: str, *values: str) -> postgresql.ENUM: + """Return an existing-or-creatable PostgreSQL enum.""" + return postgresql.ENUM(*values, name=name, create_type=False) + + +def upgrade() -> None: + """Create source-intake tables and intake-state columns.""" + _enum( + "intake_state", "awaiting_sources", "ready_for_generation", "cancelled" + ).create( + op.get_bind(), + checkfirst=True, + ) + _enum("upload_state", "pending", "ready", "failed", "expired").create( + op.get_bind(), + checkfirst=True, + ) + _enum("attachment_kind", "upload", "source_uri").create( + op.get_bind(), + checkfirst=True, + ) + _enum("idempotency_state", "in_flight", "completed").create( + op.get_bind(), + checkfirst=True, + ) + + op.add_column( + "ingestion_jobs", + sa.Column( + "intake_state", + _enum( + "intake_state", "awaiting_sources", "ready_for_generation", "cancelled" + ), + server_default="awaiting_sources", + nullable=False, + ), + ) + op.create_table( + "uploads", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("owner_principal_id", sa.String(length=200), nullable=True), + sa.Column("content_type", sa.String(length=255), nullable=False), + sa.Column("declared_size", sa.BigInteger(), nullable=False), + sa.Column("actual_size", sa.BigInteger(), nullable=True), + sa.Column("declared_sha256", sa.String(length=64), nullable=True), + sa.Column("content_hash", sa.String(length=80), nullable=True), + sa.Column("storage_key", sa.Text(), nullable=False, unique=True), + sa.Column( + "state", + _enum("upload_state", "pending", "ready", "failed", "expired"), + nullable=False, + ), + sa.Column("metadata", postgresql.JSONB(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.CheckConstraint("declared_size >= 0", name="ck_uploads_declared_size"), + sa.CheckConstraint( + "actual_size IS NULL OR actual_size >= 0", + name="ck_uploads_actual_size", + ), + ) + op.create_table( + "ingestion_job_sources", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + "ingestion_job_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("ingestion_jobs.id"), + nullable=False, + ), + sa.Column( + "attachment_kind", + _enum("attachment_kind", "upload", "source_uri"), + nullable=False, + ), + sa.Column( + "upload_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("uploads.id"), + nullable=True, + ), + sa.Column("source_uri", sa.Text(), nullable=True), + sa.Column("source_type", sa.String(length=120), nullable=False), + sa.Column("weight", sa.Float(), nullable=False), + sa.Column("metadata", postgresql.JSONB(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.CheckConstraint( + "weight >= 0 AND weight <= 1", + name="ck_ingestion_job_sources_weight", + ), + sa.CheckConstraint( + "(upload_id IS NOT NULL AND source_uri IS NULL) OR " + "(upload_id IS NULL AND source_uri IS NOT NULL)", + name="ck_ingestion_job_sources_exactly_one_source", + ), + ) + op.create_index( + "ix_ingestion_job_sources_ingestion_job_id", + "ingestion_job_sources", + ["ingestion_job_id"], + ) + op.create_index( + "ix_ingestion_job_sources_upload_id", + "ingestion_job_sources", + ["upload_id"], + ) + op.create_table( + "idempotency_records", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("principal_id", sa.String(length=200), nullable=False), + sa.Column("operation", sa.String(length=120), nullable=False), + sa.Column("idempotency_key", sa.String(length=512), nullable=False), + sa.Column("body_hash", sa.String(length=128), nullable=False), + sa.Column( + "state", + _enum("idempotency_state", "in_flight", "completed"), + nullable=False, + ), + sa.Column("serialised_outcome", postgresql.BYTEA(), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint( + "principal_id", + "operation", + "idempotency_key", + name="uq_idempotency_records_principal_operation_key", + ), + sa.CheckConstraint( + "state != 'completed' OR serialised_outcome IS NOT NULL", + name="ck_idempotency_records_completed_outcome", + ), + ) + op.create_index( + "ix_idempotency_records_expires_at", + "idempotency_records", + ["expires_at"], + ) + + +def downgrade() -> None: + """Drop source-intake tables and intake-state columns.""" + op.drop_index( + "ix_idempotency_records_expires_at", + table_name="idempotency_records", + ) + op.drop_table("idempotency_records") + op.drop_index( + "ix_ingestion_job_sources_upload_id", + table_name="ingestion_job_sources", + ) + op.drop_index( + "ix_ingestion_job_sources_ingestion_job_id", + table_name="ingestion_job_sources", + ) + op.drop_table("ingestion_job_sources") + op.drop_table("uploads") + op.drop_column("ingestion_jobs", "intake_state") + _enum("idempotency_state", "in_flight", "completed").drop( + op.get_bind(), + checkfirst=True, + ) + _enum("attachment_kind", "upload", "source_uri").drop( + op.get_bind(), + checkfirst=True, + ) + _enum("upload_state", "pending", "ready", "failed", "expired").drop( + op.get_bind(), + checkfirst=True, + ) + _enum("intake_state", "awaiting_sources", "ready_for_generation", "cancelled").drop( + op.get_bind(), + checkfirst=True, + ) diff --git a/docs/adr/adr-015-upload-and-idempotency-ports.md b/docs/adr/adr-015-upload-and-idempotency-ports.md new file mode 100644 index 00000000..9a054582 --- /dev/null +++ b/docs/adr/adr-015-upload-and-idempotency-ports.md @@ -0,0 +1,235 @@ +# ADR-015: Upload and idempotency ports + +## Status + +Accepted for roadmap item `4.3.1`. + +## Context + +ADR 009 defines the source-to-script REST vertical slice. The first delivery +task needs a small intake path before generation exists: clients upload a +source document, create an ingestion job, attach the uploaded or URI-backed +source, and bind presenter profiles through existing reusable reference +documents. + +The implementation must preserve the hexagonal boundary in ADR 014. Upload +bytes and retryable `POST` requests are side effects, so the domain layer needs +ports for object storage and idempotency without importing Falcon, SQLAlchemy, +or any cloud provider Software Development Kit (SDK). + +## Decision + +Introduce two driven ports under `episodic.canonical.*`: + +- `ObjectStorePort` stores opaque byte streams by server-generated keys. The + first adapter is a local filesystem adapter. Client filenames never become + storage paths, and the port rejects absolute paths, leading slashes, and `..` + components before an adapter touches the filesystem. +- `IdempotencyStore` records retryable side-effecting `POST` requests by + `(principal_id, operation, idempotency_key)`, where `operation` is a stable + adapter-defined domain operation string such as `upload.create` or + `ingestion_job.create`. The backing SQL table enforces that tuple with a + unique constraint. `acquire` returns one of the domain-only outcomes + `Acquired(record_id: UUID)`, `Replay(serialised_outcome: bytes)`, + `Conflict(record_id: UUID)`, or `InFlight(record_id: UUID)`. `complete` + accepts `(record_id: UUID, serialised_outcome: bytes)` and stores only the + opaque bytes and expiry timestamp. The HTTP adapter is responsible for + serialising status, body, and headers before completion and deserialising + them when replaying. + +The intake slice also introduces: + +- `Upload`, a metadata record for uploaded bytes. +- `IngestionJobSource`, a pre-generation source attachment that references + either an upload or a remote source Uniform Resource Identifier (URI). +- `IdempotencyRecord`, the stored request hash and opaque replay payload. Its + domain fields are `id`, `principal_id`, `operation`, `idempotency_key`, + `body_hash`, `state`, `serialised_outcome: bytes | None`, `expires_at`, + `created_at`, and `updated_at`. The adapter layer selects the codec used for + `serialised_outcome`. +- `IngestionJob.intake_state`, an orthogonal state for REST intake progress. + +The `idempotency_records` table stores the same domain vocabulary: + +```sql +CREATE TABLE idempotency_records ( + id UUID PRIMARY KEY, + principal_id TEXT, + operation TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + body_hash TEXT NOT NULL, + state TEXT NOT NULL, + serialised_outcome BYTEA, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE (principal_id, operation, idempotency_key) +); +``` + +The existing `SourceDocument` entity remains post-merge provenance. Reusing it +for pre-generation attachment would mix the queue of proposed inputs with the +canonical episode provenance that ingestion creates after merging. + +`POST /v1/uploads` is the only upload creation route in this slice. +`POST /v1/uploads/init` and `PUT /v1/uploads/{upload_id}/bytes` are deferred +until an S3-compatible or resumable upload adapter has a concrete consumer. + +## Request hashes + +JSON request hashes use canonical UTF-8 JSON with stable key ordering and no +insignificant whitespace. Multipart upload hashes are derived from the streamed +body hash and the metadata fields that affect the created upload. + +For `POST /v1/uploads`, the metadata allowlist is: + +```plaintext +content_type +declared_sha256 +declared_size +``` + +The worked vector below is normative for tests. With body bytes `hello\n`, +`body_sha256` is: + +```plaintext +5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +``` + +With metadata: + +```json +{"content_type":"text/plain","declared_sha256":null,"declared_size":6} +``` + +the canonical metadata bytes are exactly: + +```plaintext +{"content_type":"text/plain","declared_sha256":null,"declared_size":6} +``` + +The multipart request fingerprint is SHA-256 over: + +```plaintext +5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03:{"content_type":"text/plain","declared_sha256":null,"declared_size":6} +``` + +which yields: + +```plaintext +sha256:f03f8d4c738536bcd1c13cc34d6816f8ea0672c3e2d47c2cbbaf5c8ecbda5e2c +``` + +## Observability contract + +The intake adapters must emit metrics, traces, and structured logs at the +storage and idempotency boundaries. Metrics use bounded-cardinality labels +only; do not label metrics with upload ids, idempotency keys, object-store +keys, filenames, source URIs, document hashes, or principal ids. + +Required metrics: + +- `source_intake_upload_requests_total{operation,outcome,content_type_family}`: + counter for upload requests. `content_type_family` is one of `pdf`, `docx`, + `text`, `markdown`, `html`, or `other`. +- `source_intake_upload_duration_seconds{operation,outcome}`: histogram covering + request receipt through upload row persistence. +- `source_intake_upload_bytes{content_type_family,outcome}`: histogram of + accepted and rejected upload byte counts. +- `source_intake_upload_errors_total{operation,error_code}`: counter for + documented intake error-code outcomes. +- `source_intake_object_store_operations_total{operation,outcome,error_class}`: + counter for object-store `put`, `open`, and `delete` operations. The + `error_class` label is a stable adapter-defined category such as `permission`, + `not_found`, `io`, or `none`. +- `source_intake_object_store_operation_duration_seconds{operation,outcome}`: + histogram around each storage-port call. +- `source_intake_idempotency_outcomes_total{operation,outcome}`: counter for + `acquired`, `replay`, `conflict`, `in_flight`, and `complete_failed`. +- `source_intake_orphan_uploads_total{state}` and + `source_intake_stuck_idempotency_records_total{state}`: counters incremented + by manual or automated recovery sweeps. +- `source_intake_stream_errors_total{operation,error_class}`: counter for failed + multipart reads, client disconnects, payload-limit aborts, and hash + mismatches. + +Required trace spans: + +- `source_intake.upload.register` wraps metadata validation, stream hashing, + object-store write, upload-row persistence, and idempotency completion. +- `source_intake.object_store.put`, `source_intake.object_store.open`, and + `source_intake.object_store.delete` wrap the storage adapter boundary. +- `source_intake.idempotency.acquire` and + `source_intake.idempotency.complete` wrap idempotency-store calls. +- `source_intake.ingestion_job.create` and + `source_intake.ingestion_job.attach_source` wrap job and source-attachment + service operations. + +Span attributes are limited to `operation`, `outcome`, `error_code`, +`content_type_family`, `upload_state`, `intake_state`, and `target_kind`. +Correlation to a specific request belongs in logs and trace context, not in +metric labels. + +Required alerts: + +- Page when `source_intake_upload_errors_total` exceeds 5 percent of upload + requests over 15 minutes, excluding `unsupported_content_type` and + `payload_too_large`. +- Page when any object-store operation failure rate exceeds 1 percent over + 10 minutes or any `permission` error is observed. +- Page when a recovery sweep reports non-zero pending or failed uploads older + than one hour. +- Page when a recovery sweep reports in-flight idempotency records older than + 15 minutes. +- Warn when `payload_too_large` or `unsupported_content_type` exceeds 20 + occurrences in five minutes for one operation. + +Structured log levels are fixed as follows: + +- `INFO`: successful upload registration, idempotency replay, ingestion-job + creation, source attachment, and `awaiting_sources → ready_for_generation` + transitions. +- `WARN`: client-correctable validation failures, idempotency conflicts, + in-flight duplicate requests, hash or size mismatches, and recovery sweeps + that find orphan uploads or stale idempotency records. +- `ERROR`: object-store failures, database transaction failures, stream read + failures after request acceptance, and idempotency completion failures that + may make a committed side effect non-replayable. + +## Consequences + +### Positive + +- Upload storage and idempotency are replaceable adapters behind domain ports. +- Retried `POST` requests can be replayed or rejected consistently before a + client creates duplicate resources. +- Intake status can move to `ready_for_generation` without changing the + existing ingestion merge lifecycle. +- The filesystem adapter keeps local development and Continuous Integration + (CI) self-contained. + +### Negative + +- The local filesystem adapter can leave orphan blobs if a database transaction + fails after bytes are written. Operators must sweep stale `pending` or + `failed` upload rows until an automated recovery worker lands. +- Idempotency records grow linearly with side-effecting requests. A purge job + is deferred, so operators need a manual retention recipe. +- Trusting declared content types is weaker than magic-byte sniffing. Sniffing + is deferred because it would require a new runtime dependency. + +### Neutral + +- S3-compatible pre-signed uploads, resumable uploads, and Vidai Mock inference + fixtures belong to later roadmap items. The intake slice has no inference + call and does not need a mock inference service. + +## References + +ADR 009 defines the source-to-script REST slice.[^1] ADR 014 defines the +architecture boundary that these ports preserve.[^2] The system design records +the current table relationships.[^3] + +[^1]: [ADR 009: Source-to-script REST vertical slice](adr-009-source-to-script-rest-vertical-slice.md) +[^2]: [ADR 014: Hexagonal architecture enforcement](adr-014-hexagonal-architecture-enforcement.md) +[^3]: [Episodic podcast generation system design](../episodic-podcast-generation-system-design.md) diff --git a/docs/contents.md b/docs/contents.md index 17c71339..86d4c811 100644 --- a/docs/contents.md +++ b/docs/contents.md @@ -95,6 +95,8 @@ or delivery planning. - speech synthesis adapter boundaries. - [ADR 014: Hexagonal architecture enforcement](adr/adr-014-hexagonal-architecture-enforcement.md) - import-boundary enforcement model. +- [ADR 015: Upload and idempotency ports](adr/adr-015-upload-and-idempotency-ports.md) + - source-intake upload storage and idempotency port decisions. ## Execution plans diff --git a/docs/developers-guide.md b/docs/developers-guide.md index babadde3..1b3dc6b9 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -394,17 +394,25 @@ Key expectations: runtime. - Tests run against Postgres semantics, not SQLite. - The shared fixture stack is fully asynchronous: - - `_pglite_sqlalchemy_manager(tmp_path)` starts + - `_pglite_sqlalchemy_manager(work_dir)` starts `SQLAlchemyAsyncPGliteManager`, waits for the helper-managed engine to - accept connections, and shuts the manager down after the test. - - `pglite_sqlalchemy_manager` is the public function-scoped manager fixture. + accept connections, and shuts the manager down after the test session. + - `pglite_sqlalchemy_manager` is the public session-scoped manager fixture. - `pglite_engine` yields the helper-managed `AsyncEngine`. - - `migrated_engine` applies Alembic migrations to that engine. + - `migrated_engine` resets the shared py-pglite database's `public` schema + and applies Alembic migrations to that engine for each database-backed + test. - `session_factory` returns `async_sessionmaker[AsyncSession]` with `expire_on_commit=False`. - `pglite_session` yields a ready-to-use `AsyncSession`. -- Because the stack depends on pytest's function-scoped `tmp_path`, each - database-backed test gets an isolated ephemeral database by default. +- Because `migrated_engine` drops and recreates the `public` schema before + applying migrations, each database-backed test gets an isolated schema while + the expensive py-pglite Node process is shared for the pytest session. +- A session-scoped `pglite_node_environment` fixture owns the py-pglite work + root. The helper installs py-pglite's Node dependencies once for the session + and retries startup up to three times with a fresh run directory before + failing, because the external Node process can occasionally time out during + startup on shared hosts. - Most database-backed tests should use `session_factory` or `pglite_session`. Use `pglite_engine` only for lower-level engine assertions, and use `pglite_sqlalchemy_manager` only when a test genuinely needs direct manager @@ -419,9 +427,14 @@ Key expectations: database tests. Synchronous Falcon API tests should use `canonical_api_client`, which is already wired to `SqlAlchemyUnitOfWork` backed by the shared `session_factory`. -- `make test` uses `PYTEST_XDIST_WORKERS=1` by default to avoid py-pglite - cross-worker process termination. Override with - `PYTEST_XDIST_WORKERS= make test` when debugging worker-count behaviour. +- `make test` uses `PYTEST_XDIST_WORKERS=1` by default and does not load xdist + in that mode. Override with `PYTEST_XDIST_WORKERS= make test` when + deliberately debugging worker-count behaviour; values above one add + `pytest -n `. +- The global pytest timeout is 180 seconds. Keep it high enough for + function-scoped py-pglite startup and Alembic migration application under + shared Continuous Integration (CI) or multi-agent host load, but investigate + any individual database test that approaches the limit repeatedly. - `EPISODIC_TEST_DB=sqlite` disables the py-pglite fixtures (tests that depend on them will be skipped). - If a non-SQLite backend is requested while py-pglite is unavailable, the @@ -687,6 +700,85 @@ When an episode context is present, the same resolution algorithm is exposed through `ResolvedBindingsResource` and reused by ingestion to snapshot the resolved reference revisions into provenance `source_documents`. +### Source-intake idempotency and errors + +Roadmap item `4.3.1` reserves the source-intake `POST` contract for idempotent +uploads, ingestion jobs, and source attachments. Each side-effecting request in +the slice accepts `Idempotency-Key`; the server scopes the key by authenticated +principal, route, and request-body hash. A repeated request with the same key +and same canonical body replays the stored response. A repeated request with +the same key and a different canonical body returns `409 Conflict`. + +The following error codes are reserved for the source-intake implementation: + +| Error code | HTTP status | Meaning | +| -------------------------- | ----------- | ------------------------------------------------------------- | +| `idempotency_conflict` | 409 | Same key, different request-body hash. | +| `idempotency_in_progress` | 409 | Same key and body while the first request is still in flight. | +| `upload_not_found` | 404 | Referenced `upload_id` does not exist. | +| `upload_not_ready` | 409 | Referenced `upload_id` is not yet in `ready` state. | +| `upload_hash_mismatch` | 400 | Server-computed SHA-256 differs from the client declaration. | +| `upload_size_mismatch` | 400 | Server-observed byte count differs from the declared size. | +| `unsupported_content_type` | 415 | Declared content type is outside the allowlist. | +| `payload_too_large` | 413 | Streamed body exceeds the configured cap. | +| `source_payload_invalid` | 422 | Source-attachment payload fails discriminator validation. | +| `ingestion_job_not_found` | 404 | Referenced ingestion job does not exist. | +| `series_profile_not_found` | 404 | Referenced series profile does not exist. | + +_Table 4: Reserved source-intake API error codes._ + +Source-intake observability follows +[ADR 015](adr/adr-015-upload-and-idempotency-ports.md). Implement the metrics +through the shared `MetricsPort` boundary and keep labels bounded: route, +outcome, content-type family, operation, error code, error class, upload state, +intake state, and target kind are allowed. Never use upload ids, idempotency +keys, object-store keys, filenames, Uniform Resource Identifiers (URIs), +document hashes, or principal ids as metric labels. + +Required metrics are: + +- `source_intake_upload_requests_total` +- `source_intake_upload_duration_seconds` +- `source_intake_upload_bytes` +- `source_intake_upload_errors_total` +- `source_intake_object_store_operations_total` +- `source_intake_object_store_operation_duration_seconds` +- `source_intake_idempotency_outcomes_total` +- `source_intake_orphan_uploads_total` +- `source_intake_stuck_idempotency_records_total` +- `source_intake_stream_errors_total` + +Trace these service and adapter boundaries with the span names from ADR 015: +upload registration, object-store `put`/`open`/`delete`, idempotency `acquire`/ +`complete`, ingestion-job creation, and source attachment. Span attributes must +use the same bounded vocabulary as metrics. + +Use log levels consistently: + +- `INFO` for successful upload registration, idempotency replay, ingestion-job + creation, source attachment, and ready-for-generation transitions. +- `WARN` for client-correctable validation failures, idempotency conflicts, + in-flight duplicate requests, hash or size mismatches, and recovery sweeps + that find orphan uploads or stale idempotency records. +- `ERROR` for object-store failures, database transaction failures, accepted + request stream failures, and idempotency completion failures that may make a + committed side effect non-replayable. + +Production alerting must page when upload errors exceed 5 percent of requests +over 15 minutes after excluding expected client rejections, when object-store +operation failures exceed 1 percent over 10 minutes, when any object-store +permission error occurs, when recovery finds pending or failed uploads older +than one hour, or when in-flight idempotency records are older than 15 minutes. +Emit warning alerts for high-volume `payload_too_large` or +`unsupported_content_type` responses so integrators can correct clients before +they become incidents. + +Until the automated purge worker lands, operators can recover stale upload and +idempotency state manually. Stale `uploads` rows in `pending` or `failed` state +identify blobs that can be deleted through the configured object-store adapter. +Expired idempotency rows can be purged with a bounded SQL delete against +`idempotency_records.expires_at`. + ### Prompt scaffolding for generators Use the canonical prompt helpers to build deterministic generation scaffolds diff --git a/docs/episodic-podcast-generation-system-design.md b/docs/episodic-podcast-generation-system-design.md index 100a31c0..217f1806 100644 --- a/docs/episodic-podcast-generation-system-design.md +++ b/docs/episodic-podcast-generation-system-design.md @@ -20,6 +20,7 @@ Accepted decision records: - [ADR 012: Pronunciation repository](adr/adr-012-pronunciation-repository.md) - [ADR 013: Speech synthesis adapters](adr/adr-013-speech-synthesis-adapters.md) - [ADR 014: Hexagonal architecture enforcement](adr/adr-014-hexagonal-architecture-enforcement.md) +- [ADR 015: Upload and idempotency ports](adr/adr-015-upload-and-idempotency-ports.md) ## Overview @@ -981,6 +982,13 @@ operations that can outlive a request return pollable resources with `POST` requests accept `Idempotency-Key` so clients can retry network failures without duplicating uploads, source attachments, or generation runs. +ADR 015 records the implementation ports for the first task. Upload bytes go +through an `ObjectStorePort`, retryable `POST` requests go through an +`IdempotencyStore`, and pre-generation attachments are stored separately from +post-merge `source_documents`. The first implementation serves only +`POST /v1/uploads`; resumable upload initialisation and direct byte `PUT` +routes are deferred until a concrete S3-compatible adapter lands. + TEI retrieval remains attached to the episode resource rather than to export jobs. `GET /v1/episodes/{episode_id}/tei` returns a JSON envelope by default, including TEI XML, content hash, revision, last generation run id, quality @@ -1471,7 +1479,13 @@ Agentic workflow behaviour is configurable per series profile: limits, storage location, and idempotency keys before uploaded material is attached to ingestion jobs. - `ingestion_jobs` tracks each ingestion run, including status, timestamps, and - targeted episodes. + targeted episodes. Source-intake readiness is tracked separately from merge + status through `intake_state`. +- `ingestion_job_sources` records pre-generation attachments for an ingestion + job. Each row references either an `upload` or a remote `source_uri`, but not + both. +- `idempotency_records` stores accepted retryable operation keys, body hashes, + opaque serialised outcomes, and expiry timestamps. - `source_documents` records ingestion-run inputs, document types, weighting factors, and original files in object storage. - `episodes` holds canonical TEI, generation status, QA verdicts, and approval @@ -1564,6 +1578,62 @@ the same provenance contract when implemented. The diagram below summarizes the canonical content tables and their relationships. +For screen readers: the following entity-relationship diagram shows uploads, +ingestion jobs, source attachments, source provenance, and idempotency records. + +```mermaid +erDiagram + UPLOADS { + uuid id + string owner_principal_id + string content_type + string content_hash + string storage_key + string state + } + + INGESTION_JOBS { + uuid id + uuid series_profile_id + uuid target_episode_id + string status + string intake_state + } + + INGESTION_JOB_SOURCES { + uuid id + uuid ingestion_job_id + uuid upload_id + string source_uri + string attachment_kind + string source_type + } + + SOURCE_DOCUMENTS { + uuid id + uuid ingestion_job_id + uuid episode_id + string document_type + } + + IDEMPOTENCY_RECORDS { + uuid id + string principal_id + string operation + string idempotency_key + string body_hash + string state + bytes serialised_outcome + } + + INGESTION_JOBS ||--o{ INGESTION_JOB_SOURCES : queues + UPLOADS ||--o{ INGESTION_JOB_SOURCES : attaches + INGESTION_JOBS ||--o{ SOURCE_DOCUMENTS : produces +``` + +_Figure 18: Source-intake tables and their relationship to post-merge +provenance._ + ```mermaid erDiagram SERIES_PROFILES ||--o{ EPISODES : has diff --git a/docs/execplans/2-3-3-generate-guest-bios-from-reference-document-bindings.md b/docs/execplans/2-3-3-generate-guest-bios-from-reference-document-bindings.md index 92e6afa5..973a5270 100644 --- a/docs/execplans/2-3-3-generate-guest-bios-from-reference-document-bindings.md +++ b/docs/execplans/2-3-3-generate-guest-bios-from-reference-document-bindings.md @@ -173,11 +173,12 @@ enrichment. brittle raw XML assertions around entity escaping; the tests now inspect parsed `tei_rapporteur` payloads. - [x] 2026-05-14: Ran focused validation for milestone 2/3 with - `set -o pipefail`, including - `uv run pytest tests/test_guest_bios.py`, - `tests/test_guest_bios_properties.py -q`, `make check-fmt`, - `make typecheck`, `make lint`, `make markdownlint`, and `make nixie`. - Ran `coderabbit review --agent`; it completed with zero findings. + `set -o pipefail`, including + `uv run pytest tests/test_guest_bios.py + tests/test_guest_bios_properties.py -q`, + `make check-fmt`, `make typecheck`, + `make lint`, `make markdownlint`, and `make nixie`. Ran + `coderabbit review --agent`; it completed with zero findings. - [x] 2026-05-14: Added `generate_guest_bios_from_reference_bindings(...)` to resolve existing reference bindings through the canonical unit of work, project only diff --git a/docs/execplans/4-1-2-finalize-rest-surfaces.md b/docs/execplans/4-1-2-finalize-rest-surfaces.md index bc91410d..e5605b24 100644 --- a/docs/execplans/4-1-2-finalize-rest-surfaces.md +++ b/docs/execplans/4-1-2-finalize-rest-surfaces.md @@ -768,20 +768,25 @@ Test-first changes: Production changes: -1. Add - - `count_for_series(self, owner_series_profile_id: str, kind: ReferenceDocumentKind | None) -> int` - to `ReferenceDocumentRepository` Protocol at +1. Add this method to `ReferenceDocumentRepository` Protocol at `episodic/canonical/reference_protocols.py:30-39` and its SQLAlchemy implementation at `episodic/canonical/storage/reference_repositories.py:57-81`. + + ```python + def count_for_series( + self, + owner_series_profile_id: str, + kind: ReferenceDocumentKind | None, + ) -> int: ... + ``` + 2. Add `count_for_document(self, document_id: str, owner_series_profile_id: str) -> int` to `ReferenceDocumentRevisionRepository ` Protocol (`reference_protocols.py:76-84`) and its implementation (` reference_repositories.py:158-178`). -3. Add - - `count_for_target(self, target_kind: ReferenceBindingTarget, target_id: str) -> int` +3. Add `count_for_target(self, target_kind: ReferenceBindingTarget, + target_id: str) -> int` to `ReferenceBindingRepository` Protocol (`reference_protocols.py:113-122`) and its implementation (`reference_repositories.py:236-261`). 4. Update the service-layer wrappers to return `(items, total)` tuples @@ -857,12 +862,20 @@ Test-first changes: Production changes: -1. Add - - `list_for_profile_paged(self, profile_id: uuid.UUID, *, limit: int, offset: int) -> list[SeriesProfileHistoryEntry]` - and `count_for_profile(self, profile_id: uuid.UUID) -> int` to - `_SeriesProfileHistoryRepository` +1. Add these methods to `_SeriesProfileHistoryRepository` (`episodic/canonical/profile_templates/types.py:296-300`); analogue for + + ```python + def list_for_profile_paged( + self, + profile_id: uuid.UUID, + *, + limit: int, + offset: int, + ) -> list[SeriesProfileHistoryEntry]: ... + def count_for_profile(self, profile_id: uuid.UUID) -> int: ... + ``` + episode templates. 2. Implement both in `episodic/canonical/storage/history_repositories.py:146-201`. diff --git a/docs/execplans/4-3-1-source-and-presenter-profile-intake-script-generation.md b/docs/execplans/4-3-1-source-and-presenter-profile-intake-script-generation.md new file mode 100644 index 00000000..75e0379e --- /dev/null +++ b/docs/execplans/4-3-1-source-and-presenter-profile-intake-script-generation.md @@ -0,0 +1,1438 @@ +# Source and presenter-profile intake for script generation + +This Execution Plan (ExecPlan) is a living document. The sections `Constraints`, +`Tolerances`, `Risks`, `Progress`, `Surprises & Discoveries`, `Decision Log`, +and `Outcomes & Retrospective` must be kept up to date as work proceeds. + +Status: IN PROGRESS + +## Purpose and big picture + +This change delivers roadmap item `4.3.1`, the first half of the +source-to-script vertical slice defined in +[ADR 009](../adr/adr-009-source-to-script-rest-vertical-slice.md). After this +change, an integration client can drive the narrow intake workflow that ADR 009 +calls out: upload one binary source document, create or reuse an ingestion job +attached to a series profile, attach the upload or a remote source Uniform +Resource Identifier (URI) to that job, bind reusable host and guest profile +reference-document revisions for the run, and poll the job until the source +context is ready for downstream draft script generation. Roadmap item `4.3.2` +will pick up the no-Quality-Assurance (QA) generation run and the Text Encoding +Initiative Profile 5 (TEI-P5) retrieval that close the vertical slice. + +The slice is deliberately small. It introduces only the Representational State +Transfer (REST) intake surface, the two new domain ports the slice needs (an +object-store port for upload bytes and an idempotency-store port for retryable +POST requests), pre-generation source-attachment persistence, and a local +filesystem adapter sufficient for development and Continuous Integration (CI). +It deliberately defers Simple Storage Service (S3)-compatible pre-signed +adapters, magic-byte content sniffing, distributed background workers, and any +generation-run plumbing. Each deferred concern is recorded in `Risks` with the +follow-up roadmap item that should own it. + +Success is observable when: + +1. A client can `POST /v1/uploads` with `multipart/form-data` containing a + source-document byte stream of any allowlisted content type — Portable + Document Format (PDF, `application/pdf`), Office Open XML document (DOCX, + `application/vnd.openxmlformats-officedocument.wordprocessingml.document`), + plain text (`text/plain`), Markdown (`text/markdown`), or Hypertext Markup + Language (`text/html`) — an `Idempotency-Key` Hypertext Transfer Protocol + (HTTP) header, and metadata, and receive `201 Created` with body + `{"id": "", "content_hash": "sha256:", "size_bytes": ,` + `"content_type": "", "storage_key": "", ...}`. A + second identical request with the same key returns the same `201` response + verbatim; a request with the same key and a different body returns `409` + with body `{"code": "idempotency_conflict", ...}`. A request whose declared + `content_type` is outside the allowlist returns `415` with body + `{"code": "unsupported_content_type", ...}`. +2. A client can `POST /v1/ingestion-jobs` with + `{"series_profile_id": "", "target_episode_id": null}` and an + `Idempotency-Key`, and receive `201 Created` with body + `{"id": "", "series_profile_id": "", "target_episode_id": null,` + `"intake_state": "awaiting_sources", "created_at": "...", ...}`. +3. A client can `POST /v1/ingestion-jobs/{job_id}/sources` with body + `{"type": "upload", "upload_id": "", "source_type": "research_paper",` + `"weight": 1.0, "metadata": {"language": "en"}}` and an `Idempotency-Key`, + and receive `201 Created` with the new source-attachment resource. The same + endpoint accepts `{"type": "source_uri", "source_uri": "https://...", ...}` + as a substitute for the upload variant and rejects payloads with an unknown + `type` or with `type`-incompatible fields with `422`. +4. `GET /v1/ingestion-jobs/{job_id}` returns the current job status envelope + containing `intake_state` and a `next_poll_after_seconds` body field for + non-terminal states (clients should treat the field as advisory). The + polling client observes `intake_state` advancing through + `awaiting_sources → ready_for_generation` once at least one source is + attached. The terminal state is sticky: a second `pending → ready` + transition is impossible because the application-service `UPDATE` is + conditional on the current state. +5. `GET /v1/series-profiles/{profile_id}/reference-documents?kind=host_profile` + and the equivalent `kind=guest_profile` query continue to work unchanged, and + `POST /v1/reference-bindings` lets a client pin a host or guest profile + revision to the ingestion job's target context. +6. The full intake gate sequence — `make check-fmt`, `make markdownlint`, + `make nixie`, `make build`, `make lint` (which includes + `make check-architecture`), `make typecheck`, `make test` (which already + invokes `make crosshair` as a dependency in the Makefile), and + `make check-migrations` — all succeed. +7. `coderabbit review --agent` reports no unresolved actionable concerns at + each milestone closure. +8. `docs/users-guide.md` describes the intake workflow for an integration + client, `docs/developers-guide.md` documents the intake conventions + (Idempotency-Key contract, error codes, source-attachment payload + discriminator, canonical body-hash recipe, operator recovery recipe), and + `docs/episodic-podcast-generation-system-design.md` is updated where the + physical schema or port surface changes. A new + `docs/adr/adr-015-upload-and-idempotency-ports.md` records the + object-store-port plus idempotency-store-port design choices. +9. `docs/roadmap.md` marks item `4.3.1` done only after every gate above is + green. + +## Constraints + +These invariants must hold throughout implementation. They are not suggestions; +violation requires escalation, not workarounds. + +- The hexagonal boundary defined in + [ADR 014](../adr/adr-014-hexagonal-architecture-enforcement.md) and enforced + by Hecate in `pyproject.toml` `[tool.hecate]` is non-negotiable. All new port + protocols live under `episodic.canonical.*` modules that belong to the + `domain_ports` Hecate group. All new SQLAlchemy mappers, repositories, and + filesystem code live under `episodic.canonical.storage.*` (the + `outbound_adapter` group). All new Falcon resources and helpers live under + `episodic.api.*` (the `inbound_adapter` group). The only allowed + cross-adapter wiring sits in `episodic.api.runtime` and + `episodic.worker.runtime` (the `composition_root` group). New Hecate prefixes + must be added when new modules are introduced under `episodic.canonical.*` so + the rule set keeps full coverage. +- Domain entities must be frozen dataclasses with `slots=True` where the rest + of the codebase uses that convention. Repository protocols use `typ.Protocol` + and live in dedicated `*_protocols.py` (or equivalent) modules within the + `domain_ports` group. +- Logging must use `episodic.logging.get_logger` (a `femtologging` wrapper). + Logs that record idempotency outcomes, source-attachment decisions, and job + transitions must carry correlation identifiers (request id), + `Idempotency-Key`, ingestion-job id, series-profile id, and the principal id + supplied by the authorization middleware. +- All side-effecting `POST` requests in scope (`/v1/uploads`, + `/v1/ingestion-jobs`, `/v1/ingestion-jobs/{job_id}/sources`) must accept + `Idempotency-Key`. The store enforces the contract from ADR 009 §"Idempotency + implementation contract": one accepted request per key, identical body + returns the stored outcome, different body returns `409`. A SQL unique + constraint on `(principal_id, operation, idempotency_key)` is required. + `operation` is a stable adapter-defined domain operation string such as + `upload.create` or `ingestion_job.create`. The middleware order in + `episodic.api.app.create_app` is authorization first, idempotency second: the + idempotency cache key includes the authenticated principal id, so + authorization must run before the idempotency middleware can compute the + composite key. +- The request-body hash is SHA-256 over the canonical request bytes per ADR 009 + §"Request-body hash". For JSON bodies, canonical UTF-8 JSON with stable + object-key ordering and no insignificant whitespace (the helper + `canonical_json_bytes` in `episodic/canonical/idempotency_service.py` is the + only authoritative implementation). For multipart bodies, the hash is + `SHA-256(body_bytes) || ":" || canonical_json_bytes(allowlisted_metadata)`, + where `body_bytes` is the streamed binary part and the allowlisted metadata + fields per operation are fixed in the same module + (`MULTIPART_BODY_HASH_METADATA: dict[str, tuple[str, ...]]`). ADR 015 must + contain a worked example vector that integration tests assert against. +- Error responses must use the established envelope from + `docs/episodic-tui-api-design.md` §"Error contract" and the helpers in + `episodic/api/errors.py`: + `{"code": "...", "message": "...", "details": {...}}`. The new error codes + introduced by this slice and their HTTP statuses are fixed in ADR 015 and + documented in `docs/developers-guide.md`: + + | Error code | HTTP status | Used when | + | -------------------------- | ----------- | --------------------------------------------------------------- | + | `idempotency_conflict` | 409 | Same key, different request-body hash. | + | `idempotency_in_progress` | 409 | Same key, identical body, first request still in flight. | + | `upload_not_found` | 404 | Referenced `upload_id` does not exist. | + | `upload_not_ready` | 409 | Referenced `upload_id` is not yet in `ready` state. | + | `upload_hash_mismatch` | 400 | Server-computed SHA-256 differs from the client-declared value. | + | `upload_size_mismatch` | 400 | Server-observed byte count differs from the declared size. | + | `unsupported_content_type` | 415 | Declared `content_type` outside the allowlist. | + | `payload_too_large` | 413 | Streamed body exceeds the configured cap. | + | `source_payload_invalid` | 422 | Source-attachment payload fails discriminator validation. | + | `ingestion_job_not_found` | 404 | Referenced `job_id` does not exist. | + | `series_profile_not_found` | 404 | Referenced `series_profile_id` does not exist. | + +- Public REST routes must use the `/v1` prefix introduced by roadmap item + `4.1.1`. Unversioned routes are not created for any new endpoint. +- All database changes must ship as Alembic migrations under + `alembic/versions/` with `make check-migrations` clean. +- The default object-storage root is a configurable filesystem directory. + Tests use a temporary directory fixture; runtime defaults are read from + configuration through `ApiDependencies` so production deployments can swap in + a different backend without code changes. +- No new external runtime dependencies may be added beyond what already ships + in `pyproject.toml` unless documented and justified in `Decision Log` and + explicitly approved through escalation. `python-magic` and any S3-compatible + client library are out of scope for this slice. +- All Python source must satisfy `ruff format --check`, `ruff check`, and + `ty check` cleanly. Pylint runs under `make lint` and must not regress. +- Public-facing prose is British English with Oxford spelling per + `docs/documentation-style-guide.md`. + +## Tolerances (exception triggers) + +Stop and escalate when: + +- Scope: implementation requires net changes to more than 25 production files + or more than 2500 lines of code (LoC) across the slice. +- Interface: a public API contract from ADR 009 or + `docs/episodic-tui-api-design.md` §"Source-to-script vertical slice" must + change, including any HTTP status code, header name, error code, or JSON + field name. +- Dependencies: a new entry in `pyproject.toml` `[project] dependencies` or + `[dependency-groups] dev` is required. +- Iterations: tests for any single milestone still fail after three + consecutive `make test` cycles. +- Hexagonal boundary: an import would cross a forbidden direction (for + example, `episodic.canonical.domain` importing from + `episodic.canonical.storage`) and the only obvious fix is to suppress + `hecate check`. +- Database: the new schema would force a backfill of existing production + data or break a previously published migration. +- Ambiguity: ADR 009, `docs/episodic-tui-api-design.md`, and the existing + reference-document model disagree on field naming or behaviour. + +## Risks + +Known uncertainties identified upfront: + +- Risk: The existing `SourceDocument` domain entity is ingestion-scoped and + always carries `ingestion_job_id`, but it is created today by the + multi-source pipeline once the merged canonical episode is known. Reusing the + same record for pre-generation attachment would blur the two lifecycles and + quietly invert invariants in + `episodic/canonical/services.py::ingest_sources`. Severity: medium. + Likelihood: high if we cut corners. Mitigation: introduce a separate + `IngestionJobSource` entity for pre-generation attachments (the queue of + inputs) and keep the existing `SourceDocument` entity strictly for post-merge + provenance. A follow-up roadmap item can fold the two together once the + generation slice (4.3.2) consumes both. +- Risk: ADR 009 mandates content-type allowlists and content hashes, but does + not pin server-side magic-byte sniffing. Trusting the client `Content-Type` + alone is a known weak point. Severity: medium. Likelihood: medium. + Mitigation: enforce a strict allowlist (`application/pdf`, + `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, + `text/plain`, `text/markdown`, `text/html`) and a 50 MB default size cap; + verify the client-declared `sha256` (when present) by streaming hash during + ingest; reject mismatches with `400`. Schedule server-side magic-byte + sniffing as a follow-up under roadmap item `5.1` ("Establish role-based + access control and tenancy"), where multi-tenant hardening lands. +- Risk: The existing `IngestionJob.status` field is consumed by the + multi-source merge pipeline in + `episodic/canonical/services.py::ingest_sources`, which writes + `IngestionStatus.COMPLETED` at the end of the merge. Adding a new value + `READY_FOR_GENERATION` to the same enum would silently invert the merge + pipeline's "pending → running → completed" lifecycle assumption and risks + drift between the intake and merge surfaces. Severity: high. Likelihood: high + if unaddressed. Mitigation: add a new orthogonal column + `IngestionJob.intake_state: IntakeState` (separate `StrEnum` in + `episodic/canonical/domain.py` with values `awaiting_sources`, + `ready_for_generation`, `cancelled`) and leave the existing `status` field + alone. The intake REST surface reads and writes only `intake_state`. The + merge pipeline continues to read and write only `status`. ADR 015 records + this split. The two fields converge in roadmap item `4.3.2` when the + generation orchestrator unifies them. +- Risk: Orphan filesystem blobs. The single-shot `POST /v1/uploads` + streams bytes to disk before the `Upload` row is committed; a database + rollback after the bytes land leaves a blob with no row to garbage-collect + it. The two-step path is already two-phase by construction, so this risk + affects only the multipart path. Severity: medium. Likelihood: low per + request, high cumulatively. Mitigation: implement the multipart path as two + phases inside the application service. First commit inserts an `Upload` in + `pending` state with a reserved `storage_key`. The bytes then stream into + that key via `ObjectStorePort.put`. The second commit transitions the row to + `ready` with the server-computed `content_hash`. If the second commit fails, + the row remains in `pending` and the operator-recovery recipe in + `docs/developers-guide.md` (added in Milestone D7) describes the sweep: list + `Upload` rows older than the configured retention with state in (`pending`, + `failed`) and call `ObjectStorePort.delete` for each. The sweeper itself is + out of scope for the slice and tracked under roadmap item `5.1` ("Role-based + access control and tenancy hardening"). +- Risk: Idempotency-record retention cost. At sustained throughput, the + `idempotency_records` table grows at one row per side-effecting `POST` per + `Idempotency-Key`. With a 24-hour retention and even modest opaque + `serialised_outcome` payloads, the table grows fastest of any in the slice. + Severity: medium. Likelihood: medium. Mitigation: cap the serialised outcome + at 64 kilobytes (KB) in the HTTP adapter before calling + `IdempotencyStore.complete`. Add an Alembic-shipped index on `expires_at` so + a future purge job can scan efficiently. The purge job itself is deferred to + roadmap item `5.1`; the operator-recovery recipe in + `docs/developers-guide.md` includes a manual + `DELETE … WHERE expires_at < NOW()` recipe so production can drain the table + without code changes. +- Risk: Idempotency-Key storage with a strict unique constraint can collide + with parallel test execution under `pytest-xdist` if tests reuse keys. + Severity: low. Likelihood: medium. Mitigation: composite cache key + `(principal_id, operation, idempotency_key)`; test helpers generate fresh + `uuid4` keys per request; py-pglite isolates databases per worker. +- Risk: Object-storage filesystem adapter must not expose path traversal or + permit symlink escape. Severity: high. Likelihood: low. Mitigation: + server-generated `uuid4` storage keys, never trust client filenames for + paths, store under a dedicated root configured by operator, refuse + non-relative or `..` keys at the port boundary. +- Risk: ADR 009 requires Hypothesis property tests for the idempotency state + machine. Hypothesis under `pytest-xdist` can produce flakes when the database + is shared. Severity: low. Likelihood: medium. Mitigation: scope idempotency + property tests to a per-test py-pglite database fixture; pin `Hypothesis` + settings with a deterministic `derandomize` profile for CI per existing repo + convention. +- Risk: The user prompt for this slice notes that Vidai Mock should cover + inference services, but the slice itself does not call inference services — + generation lives in roadmap item `4.3.2`. Pulling Vidai Mock fixtures into + the slice would introduce a Mock-server scaffold with no consumer. Severity: + low. Likelihood: certain (no inference path exists in the slice). Mitigation: + document in `Decision Log` that Vidai Mock applies only once ingestion + adapters add an inference-backed normalizer or the generation orchestrator + lands. For 4.3.1 the required behavioural coverage uses `pytest-bdd` and + `py-pglite` alone. Roadmap item `4.3.2` owns the Vidai Mock scaffold for the + generation run. +- Risk: Two concurrent source-attachment requests for the same job both + observe `intake_state = awaiting_sources` and both attempt the transition to + `ready_for_generation`. The transition must be at-most-once because + downstream consumers in `4.3.2` listen for the transition. Severity: low. + Likelihood: medium. Mitigation: the application service performs a + conditional UPDATE inside the same transaction as the source-row insert: + + ```sql + UPDATE ingestion_jobs + SET intake_state = 'ready_for_generation' + WHERE id = :id + AND intake_state = 'awaiting_sources' + ``` + + Only the first concurrent transaction's UPDATE matches; the second is a + no-op. Document the guarantee in ADR 015 and assert it in a property test. + +## Progress + +Use this section as the authoritative status of the work. Update with +timestamps as milestones land. + +- [x] (2026-06-08T16:13Z) Milestone D1 — Documentation, ADR, and schema + sketch. Produced ADR 015, updated + `docs/episodic-podcast-generation-system-design.md` with the new intake + schema sketch, drafted the source-intake error table in + `docs/developers-guide.md`, added the `docs/users-guide.md` intake stub, and + linked ADR 015 from `docs/contents.md`. Validation passed with `make fmt`, + `make check-fmt`, `make markdownlint`, and `make nixie`. +- [x] (2026-06-10T15:05Z) Milestone D2 — Domain ports and entities. Added + `Upload`, `IngestionJobSource`, `IdempotencyRecord`, + upload/source/idempotency repository protocols, `ObjectStorePort`, canonical + JSON and multipart hash helpers, and intake-aware ingestion-job + list/count/transition protocol methods. +- [x] (2026-06-10T15:50Z) Milestone D3 — Outbound adapters. Added focused + SQLAlchemy source-intake models, mappers, repositories, Alembic revision + `20260610_000009`, `FilesystemObjectStore`, and `SqlAlchemyUnitOfWork` wiring + for `uploads`, `ingestion_job_sources`, and `idempotency`. +- [x] (2026-06-10T16:25Z) Milestone D4 — Application services. Added + `register_upload`, `create_ingestion_job`, `attach_source_to_ingestion_job`, + `get_ingestion_job_status`, and `list_ingestion_jobs`. The two-step + `initialize_upload`/`finalize_upload_bytes` path remains deferred by the + existing "Two-step upload deferred" decision. +- [x] (2026-06-10T16:50Z) Milestone D5 — Inbound adapters and HTTP wiring. Added + `UploadsResource`, `IngestionJobsResource`, `IngestionJobResource`, + `IngestionJobSourcesResource`, multipart parsing, content-type allowlisting, + size checks, idempotent replay/conflict handling, and `/v1` route + registration. Idempotency is implemented by an adapter helper rather than a + generic middleware; see the Decision Log entry dated 2026-06-10T16:45Z. +- [x] (2026-06-10T15:45Z) Milestone D6 — Tests. Added focused + unit/integration coverage for the object-store port, ADR 015 multipart + fingerprint, SQLAlchemy source-intake repositories, SQLAlchemy idempotency + replay/conflict outcomes, Hypothesis idempotency properties, an HTTP + end-to-end upload/job/source/poll flow, a pytest-bdd source-intake feature, + and a syrupy snapshot for stable source-intake response envelopes. +- [ ] Milestone D7 — Documentation final pass and roadmap toggle. Finalise + `docs/users-guide.md` and `docs/developers-guide.md` updates, refresh the + contents index, and mark `4.3.1` done in `docs/roadmap.md`. + +Update each milestone with completion timestamps and any partial-progress +notes. Use the form `[x] (YYYY-MM-DDTHH:MMZ) `. + +## Surprises & discoveries + +Record observations during implementation that were not anticipated as risks. +Each entry should follow the form: + +- Observation: the unexpected finding. + Evidence: how you know. Impact: how it affects this plan or future work. + +- Observation: implementation started from a plan still marked `DRAFT`. + Evidence: the user explicitly requested implementation of this ExecPlan on + 2026-06-08. Impact: the plan status is now `IN PROGRESS`, and the approval + gate is treated as satisfied by the implementation request. +- Observation: CodeRabbit could not complete the Milestone D1 review because + the CLI repeatedly returned `rate_limit` after three full requested retry + sleeps. Evidence: scoped review attempts used + `coderabbit review --agent --type uncommitted --dir docs`; the CLI returned + `rate_limit` after waits of 88, 64, and 84 minutes. Impact: Milestone D1 + deterministic gates are green, but the work is paused at the CodeRabbit + review gate before starting D2. +- Observation: post-review warnings identified two gaps to close before + readiness: the refactored `brief.py` and `bindings.py` façades needed + explicit replacement coverage, and source-intake observability needed more + than structured logging. Evidence: the 2026-06-10 review warning called out + deleted tests and missing metrics, tracing, and alerting. Impact: the branch + now adds loader edge-path and façade regression tests, and ADR 015 plus the + developers' guide define bounded metrics, trace spans, alert rules, and log + levels before the next validation run. +- Observation: full-suite test retries exposed py-pglite fixture setup + timeouts rather than assertion failures in the new tests. Evidence: focused + reruns passed the brief/bindings tests and most previously failed BDD tests; + the remaining failures timed out while `migrated_engine` or + `pglite_sqlalchemy_manager` started a function-scoped py-pglite database and + applied migrations. Impact: the project pytest timeout is raised from 60 to + 180 seconds; `make test` now avoids xdist in the default one-worker mode; the + py-pglite process is shared for the pytest session; `migrated_engine` resets + the `public` schema before applying migrations per test; startup is retried + up to three times with a fresh run directory; and the py-pglite docs now + describe the expected headroom and the need to investigate repeated + near-timeouts. +- Observation: follow-up review found the initial bindings façade tests still + relied on object identity checks. Evidence: the tests asserted public + attributes with `is`, which could pass if a façade attribute were replaced by + another non-callable object. Impact: the façade tests now call + `create_reference_binding`, `get_reference_binding`, + `list_reference_bindings`, and `list_reference_bindings_paged` through the + public `bindings` module using the existing async SQLAlchemy fixture stack, + and assert returned `ReferenceBinding` values and pagination totals. +- Observation: ADR 015 still described idempotency storage using HTTP-layer + routing and response-envelope concepts. Evidence: the prior + `IdempotencyStore` text mixed adapter concerns into the domain port. Impact: + ADR 015, the system-design schema, and this ExecPlan now use + `(principal_id, operation, idempotency_key)` and a single opaque + `serialised_outcome` payload owned by the adapter codec. +- Observation: the ADR 015 multipart worked-vector digest conflicts with the + prose algorithm material. Evidence: hashing the exact material shown in ADR + 015 produces + `b80f8d35a5298a757877270595160d69334f21e902f94ad2775bda2e8c9d6d12`, while the + review gate and ADR both require + `f03f8d4c738536bcd1c13cc34d6816f8ea0672c3e2d47c2cbbaf5c8ecbda5e2c`. Impact: + `multipart_request_hash` preserves the published ADR vector as a + compatibility contract, and the focused unit test pins the required digest. +- Observation: Falcon ASGI multipart parsing does not expose the WSGI + `MultipartForm` concrete class in endpoint tests. Evidence: the first upload + test returned the adapter's "multipart/form-data payload is required" error + until parsing switched to the iterable body-part interface. Impact: + `UploadsResource` now accepts sync or async Falcon multipart iterables and + reads JSON metadata through `part.media` when Falcon has already decoded it. +- Observation: deterministic source-intake gates pass after adding the + BDD/snapshot coverage. Evidence: `make check-fmt`, `make lint`, + `make typecheck`, and `make test` passed on 2026-06-10; the full test gate + reported 863 passed, 2 skipped, and 19 snapshots passed. Impact: CodeRabbit + can review the completed implementation milestone rather than deterministic + formatting, lint, typing, or test failures. +- Observation: rebasing onto `origin/main` on 2026-06-11 produced one + documentation-format conflict in + `docs/execplans/4-1-2-finalize-rest-surfaces.md`. Evidence: `git rebase + origin/main` stopped while replaying `Document source-intake port decisions` + after `origin/main` landed `e283147 Reformat docs with mdformat-all (#132)`. + Impact: the conflict was resolved by preserving `main`'s code-block + formatting for long signatures while keeping this branch's source-intake + documentation changes. + +## Decision log + +Record significant decisions made while implementing the plan. Each entry +should follow the form: + +- Decision: the choice taken. + Rationale: why this choice over alternatives. Date/Author: timestamp and who + decided. + +Seed entries (DRAFT): + +- Decision: Introduce a new `IngestionJobSource` entity for pre-generation + attachments rather than reusing the existing `SourceDocument` entity. + Rationale: `SourceDocument` is created during merge and ties to a canonical + episode; mixing pre-generation attachments into the same table would invert + the invariant in `episodic/canonical/services.py::ingest_sources` and + complicate generation kick-off in roadmap item `4.3.2`. Keeping the two + entities separate preserves the existing post-merge provenance semantics and + gives the slice a clean intake table. Date/Author: drafted with the ExecPlan; + revisit after Decision Log review in roadmap item `4.3.2`. +- Decision: Local filesystem adapter only for the slice's object store. + Rationale: ADR 009 does not require S3 or pre-signed Uniform Resource Locator + (URL) semantics for the intake slice. Keeping the adapter local removes a new + external dependency, removes IAM/credentials concerns from the slice, and + keeps `make test` self-contained. ADR 015 records the `ObjectStorePort` + interface so a future S3 adapter is a drop-in. Date/Author: drafted with the + ExecPlan. +- Decision: Two-step upload deferred. The slice ships only + `POST /v1/uploads` (multipart); `POST /v1/uploads/init` and + `PUT /v1/uploads/{upload_id}/bytes` are out of scope. Rationale: the init + flow exists primarily to mirror an S3-style adapter that the same plan + defers. Shipping the init contract before its real consumer exists locks in a + wire format with no integration partner and roughly doubles the + inbound-adapter surface, snapshot tests, and BDD scenarios. The multipart + path is sufficient to demonstrate the intake workflow; the init path lands + when the S3 adapter does. Date/Author: drafted with the ExecPlan. +- Decision: Add a new `IngestionJob.intake_state` column rather than + overloading the existing `IngestionStatus` enum. Rationale: the existing + `status` field is owned by the multi-source merge pipeline + (`pending → running → completed/failed`). Adding `READY_FOR_GENERATION` to + that enum would invert the merge pipeline's lifecycle assumption. A separate + orthogonal column keeps the intake and merge surfaces decoupled and lets + roadmap item `4.3.2` decide how to unify them. Date/Author: drafted with the + ExecPlan. +- Decision: Source-attachment payload uses an explicit `type` + discriminator (`{"type": "upload", "upload_id": "..."}` or + `{"type": "source_uri", "source_uri": "..."}`). Rationale: the domain entity + already carries `attachment_kind`; the wire format should mirror it. An + explicit discriminator parses cleanly with JSON-Schema-style union validation + and avoids the subtle client bugs that "implicit-key" payload unions + encourage. Date/Author: drafted with the ExecPlan. +- Decision: Idempotency middleware sits after the authorization middleware, + not before it. Rationale: the idempotency cache key is composite + `(principal_id, operation, idempotency_key)`. Computing it before + authorization runs would either drop `principal_id` (allowing cross-tenant + cache reads) or require the middleware to look up the principal itself, + duplicating authorization logic. The authorization re-check on a replayed + request is cheap and is the correct trust boundary. Date/Author: drafted with + the ExecPlan in response to the Logisphere design-review finding R2. +- Decision: Multipart bodies for `POST /v1/uploads` are hashed by the + application service as + `SHA-256(body_bytes) || ":" || canonical_json_bytes(allowlisted_metadata)`, + with the allowlisted metadata fields per operation fixed in + `episodic.canonical.idempotency_service.MULTIPART_BODY_HASH_METADATA`. + Rationale: two implementations of "the canonical multipart body hash" + inevitably diverge; pinning the algorithm in one constant and a worked + example vector in ADR 015 ensures replay semantics survive code changes. + Date/Author: drafted with the ExecPlan in response to the Logisphere + design-review finding Y4. +- Decision: Implement content-type allowlist and client-declared SHA-256 + verification only; defer server-side magic-byte sniffing. Rationale: ADR 009 + does not require sniffing; adding `python-magic` is a new external dependency + and would breach the "no new deps" constraint. Sniffing belongs in roadmap + item `5.1` ("Role-based access control and tenancy") where multi-tenant + hardening lands. Date/Author: drafted with the ExecPlan. +- Decision: Vidai Mock coverage is deferred to roadmap item `4.3.2`. + Rationale: the intake slice does not call inference services. Adding a Vidai + Mock scaffold now would create unused fixtures. `4.3.2` introduces the + generation orchestrator, which is the natural seam. Date/Author: drafted with + the ExecPlan. +- Decision: Treat the 2026-06-08 implementation request as approval to execute + the previously drafted ExecPlan. Rationale: the user explicitly requested + implementation, frequent commits, gate execution, and CodeRabbit review, + which is stronger than passive approval. Date/Author: 2026-06-08T15:50Z / + Codex. +- Decision: Treat `brief.py` and `bindings.py` as public façades with focused + private-module tests rather than duplicating all service behaviour in the + façade files. Rationale: the refactor deliberately moved implementation into + `_brief_*` and `_binding_*` modules; the replacement coverage should exercise + helper edge paths and service behaviour while adding small regression tests + that pin the public import contract. Date/Author: 2026-06-10T15:30Z / Codex. +- Decision: Exercise bindings façade exports functionally rather than by + comparing object identity. Rationale: identity checks prove the import graph + but not the callable contract, and they can pass for constants or mocks. Each + public façade function now has a database-backed async test that calls through + `episodic.canonical.reference_documents.bindings` and asserts a meaningful + result field. Date/Author: 2026-06-10T12:30Z / Codex. +- Decision: Keep idempotency outcome storage domain-only. Rationale: + `IdempotencyStore` is a driven domain port, so it should not know HTTP + routing or response-envelope details. It stores operation-keyed records and + opaque serialised outcomes; the HTTP adapter serialises and deserialises + replay payloads at the edge. Date/Author: 2026-06-10T13:57Z / Codex. +- Decision: Source-intake observability must include bounded metrics, tracing, + and actionable alerts in addition to structured logs. Rationale: orphan + blobs, stuck idempotency records, and stream failures are operational + failures that may not be visible from request logs alone. ADR 015 now defines + the metric names, allowed labels, trace spans, alert thresholds, and + WARN/ERROR/INFO split. Date/Author: 2026-06-10T15:35Z / Codex. +- Decision: Implement idempotency as resource-level adapter helpers for this + slice rather than a generic `IdempotencyMiddleware` class. Rationale: + `upload.create`, `ingestion_job.create`, and `ingestion_job.source.attach` + require operation-specific canonical body hashes, including multipart + byte-stream hashing for uploads. The helper still uses the domain + `IdempotencyStore` outcomes and keeps HTTP status/body replay serialisation + in the adapter layer. Date/Author: 2026-06-10T16:45Z / Codex. +- Decision: Raise the pytest-timeout budget to 180 seconds for this repository. + Rationale: the suite intentionally uses function-scoped py-pglite databases + for PostgreSQL semantics and isolation. Startup plus Alembic migration + application can exceed 60 seconds on shared CI or multi-agent hosts, which + kills otherwise healthy database-backed tests during fixture setup. Date/ + Author: 2026-06-10T16:05Z / Codex. +- Decision: Do not invoke xdist when `PYTEST_XDIST_WORKERS=1`. + Rationale: default `make test` was still running through xdist with + `pytest -n 1`, which added a worker process around py-pglite even though no + parallelism was requested. Focused non-xdist runs were stable, so the + Makefile now runs plain pytest for the default and reserves xdist for + explicit worker counts above one. Date/Author: 2026-06-10T16:20Z / Codex. +- Decision: Share one py-pglite process for the pytest session and reset schema + state per test. Rationale: each database-backed test needs isolated schema + state, not a fresh Node process. Reusing the process avoids repeated + py-pglite startup while `migrated_engine` preserves test isolation by + dropping and recreating the `public` schema before applying Alembic + migrations. Date/Author: 2026-06-10T16:35Z / Codex. +- Decision: Retry py-pglite startup at the fixture boundary. + Rationale: the external Node process can occasionally miss the startup window + under host load even after dependency caching. Retrying with a fresh run + directory preserves per-test database isolation and avoids retrying the test + body or hiding assertion failures. Date/Author: 2026-06-10T16:50Z / Codex. + +## Outcomes & retrospective + +Summarise outcomes, gaps, and lessons after the final milestone. Compare the +result against the success criteria above. Note what would be done differently +next time. Update this section at the close-out commit. + +## Context and orientation + +This section assumes the reader has only the current working tree. + +### Repository layout that matters here + +- `episodic/api/` is the Falcon Asynchronous Server Gateway Interface (ASGI) + inbound adapter. `episodic/api/app.py` builds the ASGI app via + `create_app(dependencies: ApiDependencies)` and registers every `/v1` route. + `episodic/api/dependencies.py` defines the `ApiDependencies` dataclass that + carries the unit-of-work factory, readiness probes, shutdown hooks, + authorization port, and (after this slice) the upload-related configuration. + `episodic/api/resources/` holds the per-resource Falcon classes; the + reference-document resource at + `episodic/api/resources/reference_documents.py` is the canonical pattern to + copy for the new resources. `episodic/api/errors.py` defines the unified + error envelope and `episodic/api/helpers.py` defines the parsing utilities + (`parse_uuid`, `parse_pagination`, `require_payload_dict`, + `parse_enum_param`). +- `episodic/canonical/` is the domain plus driven-port layer. + `episodic/canonical/domain.py` holds the canonical entities, including the + existing `IngestionJob`, `SourceDocument`, `ReferenceDocumentKind` (with + `HOST_PROFILE` and `GUEST_PROFILE` already defined), and + `ReferenceBindingTargetKind` (which already includes `INGESTION_JOB`). + `episodic/canonical/entity_protocols.py` and + `episodic/canonical/unit_of_work_protocols.py` define the repository and + unit-of-work protocols. `episodic/canonical/services.py` is the existing + application service for post-merge persistence (`ingest_sources`). + `episodic/canonical/ingestion_service.py` orchestrates multi-source ingestion + through the multi-source pipeline; do not modify it for this slice. +- `episodic/canonical/storage/` is the SQLAlchemy outbound adapter. + `entity_models.py` and `entity_mappers.py` already cover the existing + `IngestionJob` and `SourceDocument`. `repositories.py` exposes + `SqlAlchemyIngestionJobRepository` and `SqlAlchemySourceDocumentRepository`. + `uow.py` is the canonical `SqlAlchemyUnitOfWork`. +- `alembic/versions/` holds the migration history. New tables require an + Alembic migration and `make check-migrations` must be clean. +- `tests/` is split into `tests/test_*.py` (unit tests), + `tests/features/*.feature` (BDD), `tests/steps/test_*_steps.py` (BDD step + modules), `tests/fixtures/*.py` (shared fixtures including + `tests/fixtures/api.py` for the Falcon test client and + `tests/fixtures/database.py` for py-pglite session factories), and + `tests/__snapshots__/` (syrupy ambits). +- The Hecate architectural enforcement configuration lives at + `pyproject.toml` `[tool.hecate]` and must be extended whenever new modules + appear under `episodic.canonical.*` or `episodic.api.*`. + +### Concepts (defined the first time used) + +- **Idempotency-Key.** A client-supplied HTTP header that names a request so + the server can deduplicate retries. The contract in this slice follows the + Internet Engineering Task Force (IETF) + `draft-ietf-httpapi-idempotency-key-header` and ADR 009. +- **Request-body fingerprint.** A SHA-256 hash over the canonical request + bytes. JSON requests are canonicalised by sorting keys and removing + whitespace; multipart requests hash the body bytes and the metadata fields + that change the side effect. +- **Two-step upload (deferred).** A future `POST /v1/uploads/init` would + return an opaque `put_url`; the client would then `PUT` the body bytes to + that URL. The shape mirrors the IETF resumable-uploads draft and tus version + 1.0.0 without committing to chunked semantics. This slice defers the endpoint + pair to roadmap item `5.1` or a dedicated S3-adapter item; the + `ObjectStorePort` interface keeps the door open. +- **Source attachment.** A row in the new `ingestion_job_sources` table + binding either an `Upload` (by `upload_id`) or a remote `source_uri` to an + `IngestionJob`. This is distinct from the existing post-merge + `source_documents` table. +- **Presenter profile.** ADR 009 defines a presenter profile as a reusable + `ReferenceDocument` of kind `host_profile` or `guest_profile`. Existing + endpoints at `/v1/series-profiles/{profile_id}/reference-documents` and + `/v1/reference-bindings` already serve this contract; the slice does not add + new presenter-profile endpoints. +- **Ready-for-generation.** A terminal status the ingestion job reaches once + at least one source attachment exists and the run can proceed to generation. + Roadmap item `4.3.2` is the consumer. + +## Plan of work + +The plan is staged so each stage ends with a fully testable repository state. + +### Stage A — Design and documentation prep (Milestone D1) + +No production code changes in this stage. The aim is to publish the design +contract that the rest of the milestones implement against. + +1. Author `docs/adr/adr-015-upload-and-idempotency-ports.md` following the + documentation style guide ADR template. The ADR records two driven-port + design decisions: + - `ObjectStorePort`: a minimal interface exposing `put`, `open`, and + `delete`. The first adapter is a filesystem implementation. The port + forbids client-controlled paths. Full signatures appear in + §"Interfaces and dependencies". + - `IdempotencyStorePort`: an `acquire` method returning a discriminated + union of `Acquired`, `Replay`, `Conflict`, and `InFlight`, backed by a + unique SQL constraint. The store persists only an opaque + `serialised_outcome` byte payload and expiry; HTTP status, body, and + headers are encoded and decoded by the inbound adapter. The default + retention is 24 hours, configurable per operation. + The ADR also records the decision to introduce a new `IngestionJobSource` + entity and to defer S3, magic-byte sniffing, and Vidai Mock to later items. +2. Update `docs/episodic-podcast-generation-system-design.md` §"Reference + schema" entity-relationship diagram and §"Source-to-script vertical slice" + with the new `uploads`, `ingestion_job_sources`, and `idempotency_records` + tables. Use Mermaid as the rest of the document does; `make nixie` must + validate. +3. Draft the new error code table in `docs/developers-guide.md` (final wording + lands in Milestone D7 once the codes ship). Reserve the codes listed in + `Constraints` above. +4. Add a stub `docs/users-guide.md` section "Source-to-script intake" with a + "Coming soon" notice; the final user-guide prose lands in Milestone D7. +5. Cross-link the new ADR from + `docs/episodic-podcast-generation-system-design.md` and from the index in + `docs/contents.md`. + +Stage A acceptance: `make markdownlint`, `make nixie`, and `make check-fmt` all +clean. + +### Stage B — Domain ports and entities (Milestone D2) + +Each addition lives in the `domain_ports` Hecate group and may not import from +`episodic.canonical.storage` or `episodic.api`. + +1. Create `episodic/canonical/uploads.py` with frozen `Upload` and + `UploadInitRequest` dataclasses. `Upload` carries `id: uuid.UUID`, + `owner_principal_id: str | None`, `content_type: str`, `declared_size: int`, + `actual_size: int | None`, `declared_sha256: str | None`, + `content_hash: str | None`, `storage_key: str`, `state: UploadState`, + `metadata: JsonMapping`, `created_at: dt.datetime`, + `updated_at: dt.datetime`. `UploadState` is a `StrEnum` of `pending` (init + issued, awaiting bytes), `ready` (bytes received and hashed), `failed`, + `expired`. +2. Create `episodic/canonical/ingestion_sources.py` with a frozen + `IngestionJobSource` dataclass. Fields: `id: uuid.UUID`, + `ingestion_job_id: uuid.UUID`, `attachment_kind: AttachmentKind` (`upload` or + `source_uri`), `upload_id: uuid.UUID | None`, `source_uri: str | None`, + `source_type: str`, `weight: float`, `metadata: JsonMapping`, + `created_at: dt.datetime`. Validate via `__post_init__` that exactly one of + `upload_id` or `source_uri` is populated and that the populated value matches + `attachment_kind`. +3. Create `episodic/canonical/idempotency.py` with a frozen + `IdempotencyRecord` dataclass: `id: uuid.UUID`, `principal_id: str | None`, + `operation: str`, `idempotency_key: str`, `body_hash: str`, + `state: IdempotencyState` (`in_flight`, `completed`), + `serialised_outcome: bytes | None`, `expires_at: dt.datetime`, + `created_at: dt.datetime`, `updated_at: dt.datetime`. Also define a + tagged-union outcome type `IdempotencyOutcome` = + `Acquired(record_id: uuid.UUID) | Replay(serialised_outcome: bytes) |` + `Conflict(record_id: uuid.UUID) | InFlight(record_id: uuid.UUID)` for the + `acquire` call. +4. Create `episodic/canonical/upload_protocols.py` with three Protocol + classes: `UploadRepository` (`add`, `get`, `mark_ready`, `mark_failed`), + `IngestionJobSourceRepository` (`add`, `get`, `list_for_job_paged`, + `count_for_job`), and `IdempotencyStore` (`acquire`, `complete`, `lookup`). + Each method is `async`. +5. Extend `episodic/canonical/entity_protocols.py::IngestionJobRepository` + with these methods: + + ```python + def list_paged( + series_profile_id: uuid.UUID | None, + intake_state: IntakeState | None, + *, + limit: int, + offset: int, + ) -> Sequence[IngestionJob]: ... + def count(series_profile_id, intake_state): ... + def transition_intake_state( + job_id, + *, + from_state: IntakeState, + to_state: IntakeState, + ) -> bool: ... + ``` + + The transition method returns `True` only when the conditional UPDATE + matched, so callers can detect concurrent transitions. +6. Add a new `IntakeState` `StrEnum` to `episodic/canonical/domain.py` with + values `AWAITING_SOURCES = "awaiting_sources"`, + `READY_FOR_GENERATION = "ready_for_generation"`, and + `CANCELLED = "cancelled"`. Add a new column + `IngestionJob.intake_state: IntakeState` (default `AWAITING_SOURCES`) to the + existing dataclass. **Do not extend `IngestionStatus`.** The merge pipeline + (`episodic/canonical/services.py::ingest_sources`) continues to read and + write only `status`; the intake REST surface reads and writes only + `intake_state`. ADR 015 records the split. Roadmap item `4.3.2` owns the + eventual unification. +7. Define an `ObjectStorePort` Protocol in + `episodic/canonical/object_store.py`. The full signatures appear in + §"Interfaces and dependencies"; in short, an async `put`, an `open` context + manager, and a `delete`, all returning domain types. The port forbids `..`, + leading slashes, and absolute paths in `key` at the port boundary so + adapters can rely on sanitised input. +8. Add the new ports to `episodic/canonical/unit_of_work_protocols.py` so the + unit-of-work exposes them, and update the `domain_ports` group in + `pyproject.toml` `[tool.hecate]` to include `episodic.canonical.uploads`, + `episodic.canonical.ingestion_sources`, `episodic.canonical.idempotency`, + `episodic.canonical.upload_protocols`, and + `episodic.canonical.object_store`. The companion application-service modules + (`episodic.canonical.upload_service`, + `episodic.canonical.ingestion_job_service`, + `episodic.canonical.idempotency_service`) are added to the `application` + group in Stage D step 5 below; the new storage modules are added to the + `outbound_adapter` group in Stage C step 8. + +Stage B acceptance: `make check-fmt`, `make lint` (Hecate clean), and +`make typecheck` all pass with the new modules in place. + +### Stage C — Outbound adapters (Milestone D3) + +These additions sit in the `outbound_adapter` Hecate group. + +1. Add SQLAlchemy models under `episodic/canonical/storage/`: + - `uploads.py` — `UploadRecord` with primary key, owner principal, + content type, declared size, actual size, declared sha256, content hash, + storage key, state, metadata JSON, timestamps. Unique index on + `content_hash` is *not* enforced (different uploads may share bytes). + - `ingestion_sources.py` — `IngestionJobSourceRecord` with foreign keys + to `ingestion_jobs.id` and `uploads.id`. Database-level check + constraint that exactly one of `upload_id`, `source_uri` is non-null. + - `idempotency.py` — `IdempotencyRecord` with composite unique index on + `(principal_id, operation, idempotency_key)` and a `body_hash` column. + The only stored replay payload is the opaque `serialised_outcome` byte + column. A monotonically advancing `expires_at` column controls retention. +2. Add mappers (`*_mappers.py`) following the convention in + `episodic/canonical/storage/entity_mappers.py`. Each mapper exposes the + familiar `_X_from_record` / `_X_to_record` private helpers. +3. Add repositories under `episodic/canonical/storage/repositories.py` or + new sibling modules: `SqlAlchemyUploadRepository`, + `SqlAlchemyIngestionJobSourceRepository`, and `SqlAlchemyIdempotencyStore`. + The idempotency store uses `INSERT … ON CONFLICT DO NOTHING` to guarantee + first-writer-wins semantics; the `acquire` method returns the discriminated + outcome union from Stage B. +4. Extend `SqlAlchemyIngestionJobRepository` with `list_paged`, `count`, and + `update_status` to satisfy the protocol extension. +5. Add `episodic/canonical/storage/filesystem_object_store.py` implementing + `ObjectStorePort` on a configurable root directory. The adapter reads in + fixed-size chunks (64 KB; pin as `_OBJECT_STORE_READ_CHUNK_BYTES`), streams + each chunk to a temporary file under `{root}/_tmp/`, updates the SHA-256 + hasher and the running byte count as it goes, and raises + `PayloadTooLargeError` *before* writing a chunk that would push the count + past the cap. On completion the adapter atomically renames the temporary + file into `{root}/{key}`. Defence in depth on the path: after sanitising + `key`, the adapter joins the path, calls `pathlib.Path.resolve()`, and + asserts `os.path.commonpath([resolved, root_resolved]) == root_resolved` so + symlink escape is blocked even if a previous deploy left a malicious link + inside the root. +6. Add an Alembic migration under `alembic/versions/` that creates the three + new tables (`uploads`, `ingestion_job_sources`, `idempotency_records`), adds + the new `intake_state` column to `ingestion_jobs` (with default + `'awaiting_sources'` and a `NOT NULL` constraint backfilled to that + default), adds a `CHECK` constraint enforcing the + `(upload_id IS NULL) <> (source_uri IS NULL)` invariant on + `ingestion_job_sources`, and adds these indexes: + - `ingestion_jobs(series_profile_id, intake_state, created_at DESC)` for + the listing query. + - `idempotency_records(expires_at)` for the eventual purge job. + - `idempotency_records(principal_id, operation, idempotency_key) UNIQUE` + for the first-writer-wins enforcement. + - `uploads(state, created_at)` for the eventual orphan-blob sweeper. + - `ingestion_job_sources(ingestion_job_id, created_at)` for the per-job + listing query. + `make check-migrations` must succeed after the migration is generated and + the models updated. +7. Extend `SqlAlchemyUnitOfWork.__aenter__` to bind the new repositories, + and add an `object_store` attribute populated from a port supplied via + `ApiDependencies` (the UoW does not construct the adapter; the composition + root does). The idempotency store is part of the UoW so it participates in + the same SQL transaction as the resource being created. +8. Update the `outbound_adapter` group in `pyproject.toml` `[tool.hecate]` + to include the new modules (`episodic.canonical.storage.uploads`, + `episodic.canonical.storage.ingestion_sources`, + `episodic.canonical.storage.idempotency`, + `episodic.canonical.storage.filesystem_object_store`). + +Stage C acceptance: `make build`, `make lint` (architecture clean), +`make typecheck`, `make check-migrations`, and the existing test suite stay +green. + +### Stage D — Application services (Milestone D4) + +Application services live in the `application` Hecate group (existing +`episodic.canonical.services` family). They depend only on `domain_ports`. + +1. Add `episodic/canonical/upload_service.py` exposing pure async + functions: + - `register_upload(uow, *, content_type, declared_size, declared_sha256, + stream, object_store) -> Upload` + handles the single-shot multipart path as a two-phase write: + 1. Phase one: inside a UoW transaction, insert an `Upload` row in + `pending` state with a server-generated `storage_key`. Commit. + 2. Phase two: stream bytes through `object_store.put` (which enforces + size cap and computes SHA-256 on the way). Verify the + client-declared SHA-256 if supplied; verify the byte count matches + `declared_size`. + 3. Phase three: in a new UoW transaction, call `mark_ready` with the + observed `content_hash` and `actual_size`. + A failure between phases one and three leaves an `Upload` row in + `pending` state and possibly a blob on disk; both are reclaimed by + the operator-recovery recipe documented in `docs/developers-guide.md`. + Enforces the size cap, content-type allowlist, and (when present) + declared-SHA-256 verification. +2. Add `episodic/canonical/ingestion_job_service.py` exposing: + - `create_ingestion_job(uow, *, series_profile_id, target_episode_id, + requested_by) -> IngestionJob`. Validates the series profile exists, + defaults `status = IngestionStatus.PENDING`, + `intake_state = IntakeState.AWAITING_SOURCES`, persists, and returns + the entity. + - `attach_source_to_ingestion_job(uow, *, job_id, attachment) -> + IngestionJobSource`. `attachment` is a discriminated value object + (`UploadAttachment` or `SourceUriAttachment`). The service: + 1. Loads the job and (for `UploadAttachment`) validates the upload is + in `ready` state. + 2. Inserts the `IngestionJobSource` row. + 3. Calls + `uow.ingestion_jobs.transition_intake_state(job_id, + from_state=IntakeState.AWAITING_SOURCES, + to_state=IntakeState.READY_FOR_GENERATION)`. + The conditional UPDATE returns `True` only for the first concurrent + request that observes `AWAITING_SOURCES`; subsequent attachments + return `False` and the service treats that as a no-op (the + transition is at-most-once). + 4. Commits the UoW. + - `list_ingestion_jobs_paged(uow, *, series_profile_id, intake_state, + limit, offset)` and `get_ingestion_job_with_sources(uow, *, job_id)`. +3. Add `episodic/canonical/idempotency_service.py` with helpers that compute + the canonical request fingerprint: + - `canonical_json_bytes(payload: JsonMapping) -> bytes` enforces sorted + keys, UTF-8, no insignificant whitespace. + - `MULTIPART_BODY_HASH_METADATA: dict[str, tuple[str, ...]]` fixes the + per-operation allowlist of metadata fields that participate in the + multipart body hash (initially + `{"upload.create": ("content_type", "declared_size", + "declared_sha256")}`). + - `multipart_request_hash(operation, *, body_sha256: str, metadata: + JsonMapping) -> str` returns + `sha256(body_sha256 + ":" + canonical_json_bytes(filtered_metadata))` + where `filtered_metadata` retains only the operation's allowlisted + fields. + The function accepts the streamed body's SHA-256 rather than the bytes + themselves so the streaming hash computed during upload is the same + value used for the body fingerprint. + - `acquire_or_replay(store, *, principal, operation, idempotency_key, + body_hash, + serialise_outcome, deserialise_outcome, work) -> IdempotencyOutcome` is + the orchestration wrapper that resources call from the inbound adapter. + The adapter supplies the HTTP codec functions; the domain store sees only + opaque `serialised_outcome` bytes. +4. Where the new domain transitions require cross-repository commits + (source attachment + intake-state transition + idempotency record), wrap + them in the existing `CanonicalUnitOfWork` and commit once. The two-phase + blob write for `register_upload` is the only flow that spans more than one + transaction; that flow is documented as "eventually consistent on the blob" + in ADR 015. +5. Update the `application` Hecate group in `pyproject.toml` `[tool.hecate]` + to include `episodic.canonical.upload_service`, + `episodic.canonical.ingestion_job_service`, and + `episodic.canonical.idempotency_service`. + +Stage D acceptance: `make check-fmt`, `make lint`, `make typecheck`, and +`make test` (existing tests) stay green. New unit tests for the application +services are added in Milestone D6 but stub fixtures can land here so the +services compile. + +### Stage E — Inbound adapters and HTTP wiring (Milestone D5) + +These additions live under `episodic.api.*` and may import from `application` +and `domain_ports`. + +1. Add `episodic/api/idempotency.py`. Define an + `IdempotencyMiddleware` that, on requests matching the configured idempotent + routes, reads the `Idempotency-Key` header, computes the request-body hash, + attempts `acquire_or_replay`, and short-circuits the response with a + replayed payload when the outcome is `Replay` or `InFlight`. On `Conflict` + it returns `409` with `{"code": "idempotency_conflict", ...}`. On `Acquired` + it stashes a completion callback on the request context that the resource + invokes after the resource has been created so the adapter can serialise the + HTTP status, body, and headers into the opaque outcome bytes passed to + `IdempotencyStore.complete`. +2. Add `episodic/api/upload_helpers.py` with the multipart parser, the + content-type allowlist (`UPLOAD_CONTENT_TYPE_ALLOWLIST` constant), the + maximum-size constant (read from configuration through `ApiDependencies`), + and a streaming hasher that enforces the size cap while computing SHA-256. +3. Add `episodic/api/resources/uploads.py` with `UploadsResource` and + `UploadResource`. The `UploadsResource.on_post` route accepts multipart + bodies and returns `201 Created` with the JSON envelope from success + criterion 1; the `UploadResource.on_get` route returns the metadata envelope + for an existing upload. Both use the helpers in `episodic/api/helpers.py` + and the error-mapping helpers in `episodic/api/errors.py`. Init and + `PUT bytes` resources are *not* added in this slice (see Decision Log + "Two-step upload deferred"). +4. Add `episodic/api/resources/ingestion_jobs.py` with + `IngestionJobsResource` (`on_post`, `on_get` for list with pagination and + `status`/`series_profile_id` filters), `IngestionJobResource` (`on_get` with + `Retry-After` on non-terminal states), and `IngestionJobSourcesResource` + (`on_post`, `on_get`). +5. Add serializers in `episodic/api/serializers.py` (or sibling module) + for the new entities, returning the JSON envelopes documented in the + "Purpose and big picture" success criteria. +6. Wire the new resources into `episodic/api/app.py` at the `/v1/uploads`, + `/v1/uploads/{upload_id}`, `/v1/ingestion-jobs`, + `/v1/ingestion-jobs/{job_id}`, and `/v1/ingestion-jobs/{job_id}/sources` + paths. The middleware order in `create_app` is `AuthorizationMiddleware` + first, `IdempotencyMiddleware` second: the idempotency cache key includes + `principal_id`, so the authorization middleware must establish the principal + before the idempotency middleware can compute the composite key. A replayed + request still passes through `AuthorizationMiddleware` on each retry — that + re-check is cheap and is the correct trust boundary. +7. Extend `ApiDependencies` with a new `upload_settings: UploadSettings` + field carrying the object-store port, maximum upload size, allowed content + types, and idempotency retention. Default values come from the composition + root; tests override via the existing `build_api_dependencies` helper. + +Stage E acceptance: `make check-fmt`, `make lint`, and `make typecheck` pass. +The Falcon test client accepts a happy-path request through every new route. +Failing edge-case behaviour is covered in Milestone D6. + +### Stage F — Tests (Milestone D6) + +Tests are split into the layers defined by the testing strategy in +`hexagonal-architecture` and the conventions in `docs/developers-guide.md`. + +1. **Unit tests (`tests/test_*.py`).** Add `tests/test_upload_service.py`, + `tests/test_ingestion_job_service.py`, `tests/test_idempotency_service.py`, + and `tests/test_filesystem_object_store.py`. These exercise services and + adapters in isolation with in-memory or temporary-directory fixtures. Cover + happy paths and unhappy paths: + - allowlist rejection, + - size cap enforcement, + - declared-sha256 mismatch (400), + - upload state-machine guards (pending → ready, ready → ready rejection, + pending → failed), + - source-attachment payload union validation, + - ingestion-job state transition on first attachment, + - object-store path-traversal refusal. +2. **Property tests (`tests/test_idempotency_properties.py`).** Use + `hypothesis[asyncio]` to generate sequences of `(key, body)` pairs and + assert the ADR 009 invariants: identical bodies for a key never create more + than one resource; different bodies for a key always return `409`. Add a + separate property test for the `transition_intake_state` at-most-once + guarantee: interleave many concurrent attach-source operations against the + same job and assert that exactly one observer sees the + `AWAITING_SOURCES → READY_FOR_GENERATION` transition. Mark the tests with + `@pytest.mark.hypothesis` and use the derandomised CI profile defined in the + existing repo convention. +3. **Behavioural tests (`tests/features/source_intake.feature` and + `tests/steps/test_source_intake_steps.py`).** Cover the end-to-end intake + workflow: + - Scenario: upload PDF, create job, attach upload, bind host and guest + profile revisions, observe `intake_state = ready_for_generation`. + - Scenario: duplicate Idempotency-Key with same body returns the same + response. + - Scenario: duplicate Idempotency-Key with different body returns + `409` with `code = idempotency_conflict`. + - Scenario: source attachment with an unknown `type` discriminator, + or with `type`-incompatible fields, returns `422` with + `code = source_payload_invalid`. + - Scenario: source attachment referencing an upload still in + `pending` returns `409` with `code = upload_not_ready`. + - Scenario: source attachment with declared content-type outside the + allowlist returns `415` with `code = unsupported_content_type`. + - Scenario: upload body exceeds the configured cap and returns `413` + with `code = payload_too_large`. + - Scenario: GET ingestion-jobs filtered by `series_profile_id` and + `intake_state` and paginated. + Mirror the conftest registration patterns in + `tests/steps/test_reference_document_api_steps.py`. Use + `canonical_api_async_client` so multipart bodies work naturally. +4. **Snapshot tests (`tests/test_source_intake_snapshots.py`).** Use + `syrupy` to lock in the canonical JSON envelopes for upload, upload-init, + ingestion-job, ingestion-job-list, and source-attachment responses. Snapshot + only the deterministic fields; canonicalise UUIDs and timestamps in the test + serialiser. +5. **Integration tests (`tests/test_source_intake_integration.py`).** + Drive the py-pglite-backed `canonical_api_client` end to end. Confirm + migrations are applied and the new tables exist. +6. **Fixtures.** Extend `tests/fixtures/api.py` with an `upload_settings` + override that points the object store at a `tmp_path` directory. Add + `tests/fixtures/uploads.py` with helpers that POST a PDF byte stream and + return an `upload_id`. + +Vidai Mock is not used in this milestone (see `Decision Log`). Document the +deferral in the test module docstrings so a future contributor does not add a +Mock-server fixture by reflex. + +Stage F acceptance: `make crosshair`, `make test`, and `make check-migrations` +all clean. The behavioural feature `source_intake.feature` enumerates the +slice's user-visible behaviour and runs under `pytest-bdd`. + +### Stage G — Documentation final pass and roadmap toggle (Milestone D7) + +1. Finalise `docs/users-guide.md` with the integration-client narrative for + the source-to-script intake workflow, showing the request and response + bodies for each endpoint. +2. Finalise the error-code table and Idempotency-Key conventions in + `docs/developers-guide.md`. +3. Refresh the table of contents in `docs/contents.md` to include ADR 015 + and the new sections. +4. Update `docs/episodic-podcast-generation-system-design.md` if any + adapter-level decision diverged from Stage A; reflect the divergence in + `Decision Log` and the ADR. +5. Update `docs/roadmap.md` to mark `4.3.1` done. +6. Update the ExecPlan `Status` field to `COMPLETE` and write the + `Outcomes & retrospective` section. + +Stage G acceptance: all `make` gates green; `coderabbit review --agent` reports +no unresolved actionable concerns; the roadmap toggle is part of the final +commit. + +## Concrete steps and expected output + +Each command should be executed from the repository root. The agent should +`tee` long outputs into +`/tmp/$ACTION-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out` +as the user instructions require. + +Initial setup: + +```bash +make build 2>&1 | tee /tmp/build-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +``` + +Per-milestone gate sequence (run in order, stop on first failure): + +```bash +make check-fmt 2>&1 | tee /tmp/check-fmt-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make markdownlint 2>&1 | tee /tmp/markdownlint-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make nixie 2>&1 | tee /tmp/nixie-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make lint 2>&1 | tee /tmp/lint-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make typecheck 2>&1 | tee /tmp/typecheck-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make test 2>&1 | tee /tmp/test-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +make check-migrations 2>&1 | tee /tmp/check-migrations-episodic-4-3-1-source-and-presenter-profile-intake-script-generation.out +``` + +After each milestone closure, run `coderabbit review --agent` and resolve all +actionable concerns before the next milestone begins. + +A focused smoke test exercising the new routes: + +```bash +uv run python - <<'PY' +import asyncio, httpx, uuid, json +from episodic.api import ApiDependencies, create_app +from episodic.canonical.storage import SqlAlchemyUnitOfWork + +async def main() -> None: + # The smoke client expects a running app and migrated database; in + # practice it is run from inside the integration test harness or + # against a local dev stack. + ... + +asyncio.run(main()) +PY +``` + +The smoke client is illustrative; for real verification, rely on the +behavioural feature in `tests/features/source_intake.feature`. + +## Validation and acceptance + +Quality criteria (what "done" means): + +- Tests: every assertion in Milestone D6 passes under `make test` with + `PYTEST_XDIST_WORKERS=2`. The new BDD scenarios appear in the `pytest-bdd` + discovery output, and the new snapshots are committed under + `tests/__snapshots__/`. +- Lint/typecheck: `make lint` (Ruff, Pylint, Hecate) and `make typecheck` + (ty 0.0.32) are clean. +- Migrations: `make check-migrations` reports no drift. +- Documentation: `make markdownlint` and `make nixie` are clean. The + contents index is up to date. ADR 015 cross-references the source documents. +- Security: object-store keys are server-generated UUIDs; no client-supplied + path enters the filesystem; the content-type allowlist is enforced + server-side; payload size is capped during the streaming hash; the + idempotency store enforces first-writer-wins via SQL unique constraint. +- Observability: ADR 015 and `docs/developers-guide.md` define the + source-intake metrics, trace spans, log levels, and alert thresholds. Metrics + use bounded labels only. Logs carry request correlation fields such as + `idempotency_outcome`, `idempotency_key`, `operation`, `principal_id`, + `series_profile_id`, `ingestion_job_id`, and `upload_id`, and reuse the + existing helpers in `episodic.logging`. + +Quality method (how we check): + +- Manual sequence: run the per-milestone gate sequence above; review + `git diff --stat origin/main...HEAD` for scope drift against `Tolerances`; run + `coderabbit review --agent` and address every actionable concern before + merge. + +## Idempotence and recovery + +- Re-running `make build` or any Milestone gate is safe; the venv and uv + cache are reused. The plan does not require any "one-shot" command. +- Re-running the migration suite is safe; Alembic detects already-applied + revisions. +- The idempotency store is itself idempotent by construction (its whole + purpose). A worker restart mid-request leaves the `in_flight` record visible; + a future recovery worker (out of scope for this slice; tracked in ADR 009 + §"Partial-failure recovery") will reconcile or expire it. Until the recovery + worker ships, an operator can manually expire stuck records via a SQL update; + document this in `docs/developers-guide.md`. + +## Interfaces and dependencies + +Be prescriptive. By the end of Stage E, the following symbols must exist at the +named paths with the listed signatures. + +In `episodic/canonical/uploads.py`: + +```python +class UploadState(enum.StrEnum): + PENDING = "pending" + READY = "ready" + FAILED = "failed" + EXPIRED = "expired" + + +@dc.dataclass(frozen=True, slots=True) +class Upload: + id: uuid.UUID + owner_principal_id: str | None + content_type: str + declared_size: int + actual_size: int | None + declared_sha256: str | None + content_hash: str | None + storage_key: str + state: UploadState + metadata: JsonMapping + created_at: dt.datetime + updated_at: dt.datetime +``` + +In `episodic/canonical/object_store.py`: + +```python +@typ.runtime_checkable +class ObjectStorePort(typ.Protocol): + async def put( + self, + stream: cabc.AsyncIterable[bytes], + *, + key: str, + content_type: str, + ) -> StoredObject: ... + + def open( + self, key: str + ) -> cabc.AsyncContextManager[cabc.AsyncIterable[bytes]]: ... + + async def delete(self, key: str) -> None: ... +``` + +In `episodic/canonical/upload_protocols.py`: + +```python +class UploadRepository(typ.Protocol): + async def add(self, upload: Upload) -> None: ... + async def get(self, upload_id: uuid.UUID) -> Upload | None: ... + async def mark_ready( + self, + upload_id: uuid.UUID, + *, + actual_size: int, + content_hash: str, + ) -> Upload: ... + async def mark_failed(self, upload_id: uuid.UUID, reason: str) -> Upload: ... + + +class IngestionJobSourceRepository(typ.Protocol): + async def add(self, source: IngestionJobSource) -> None: ... + async def get(self, source_id: uuid.UUID) -> IngestionJobSource | None: ... + async def list_for_job_paged( + self, + job_id: uuid.UUID, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJobSource]: ... + async def count_for_job(self, job_id: uuid.UUID) -> int: ... + + +class IdempotencyStore(typ.Protocol): + async def acquire( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + body_hash: str, + retention_seconds: int, + ) -> IdempotencyOutcome: ... + + async def complete( + self, + record_id: uuid.UUID, + *, + serialised_outcome: bytes, + ) -> None: ... + + async def lookup( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + ) -> IdempotencyRecord | None: ... +``` + +In `episodic/api/resources/uploads.py`: + +```python +class UploadsResource: + async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: ... + +class UploadResource: + async def on_get( + self, req: falcon.Request, resp: falcon.Response, upload_id: str + ) -> None: ... +``` + +In `episodic/api/resources/ingestion_jobs.py`: + +```python +class IngestionJobsResource: + async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: ... + async def on_get(self, req: falcon.Request, resp: falcon.Response) -> None: ... + +class IngestionJobResource: + async def on_get( + self, req: falcon.Request, resp: falcon.Response, job_id: str + ) -> None: ... + +class IngestionJobSourcesResource: + async def on_post( + self, req: falcon.Request, resp: falcon.Response, job_id: str + ) -> None: ... + async def on_get( + self, req: falcon.Request, resp: falcon.Response, job_id: str + ) -> None: ... +``` + +External dependencies used by the slice (all already present): + +- `falcon>=4.2,<5.0` for the inbound adapter. +- `sqlalchemy>=2.0.34,<3.0.0` and `asyncpg>=0.20.0` (driver path via + py-pglite in tests) for persistence. +- `hypothesis[asyncio]>=6,<7` for property tests. +- `py-pglite[async]>=0.5.3,<0.6.0` for in-memory PostgreSQL test fixtures. +- `syrupy>=5,<6` for snapshot tests. +- `pytest-bdd` for behavioural tests. +- `femtologging` via the `episodic.logging` wrapper for structured logs. + +## Documentation and skills signposts + +When working on this slice, load the following before touching code: + +- The `hexagonal-architecture` skill (driving and driven ports, dependency + rule, testing-strategy matrix). +- The `python-router` skill plus the smaller follow-on skills it routes to: + - `python-data-shapes` for the new frozen dataclasses and tagged unions. + - `python-types-and-apis` for `Protocol` design and overload signatures + in the idempotency outcome union. + - `python-errors-and-logging` for the new error-code envelope and + parameterised logging. + - `python-iterators-and-generators` for streaming hashes and async + iteration through Falcon's request streams. + - `python-concurrency` for the unit-of-work transactional boundary. +- The `python-testing`, `python-verification`, and `hypothesis` skills for + the property tests for the idempotency state machine. +- The `execplans` skill for keeping this document current. +- The `leta` skill for cross-file navigation when modifying the existing + resources. +- The `commit-message` skill for the per-milestone commits. +- The `en-gb-oxendict` skill for prose. + +When in doubt about the documentation style, consult +`docs/documentation-style-guide.md`. When in doubt about the testing +conventions for asynchronous endpoints, consult +`docs/testing-async-falcon-endpoints.md` and +`docs/testing-sqlalchemy-with-pytest-and-py-pglite.md`. When in doubt about +unit-of-work patterns, consult `docs/async-sqlalchemy-with-pg-and-falcon.md`. +For background on the langgraph and celery boundaries that 4.3.2 will cross, +consult `docs/langgraph-and-celery-in-hexagonal-architecture.md`. + +## Artefacts and notes + +Significant artefacts produced by the slice: + +- `docs/adr/adr-015-upload-and-idempotency-ports.md` records the new + port designs and the deferred items. +- `tests/features/source_intake.feature` records the user-visible + behaviour of the slice. +- `tests/__snapshots__/test_source_intake_snapshots.ambr` records the + canonical JSON envelopes. +- New Alembic migration under `alembic/versions/` records the schema + change. + +## Alternatives considered (Logisphere pre-implementation review) + +The community-of-experts review surfaced four alternatives worth recording so a +future reader can understand the negative space around the design: + +1. *Single upload endpoint vs init-plus-PUT.* The slice ships only the + multipart `POST /v1/uploads`. The init/PUT pair lands when an S3 adapter + that actually consumes the contract lands. The `ObjectStorePort` keeps the + door open. +2. *`READY_FOR_GENERATION` on the existing enum vs a new `intake_state` + column vs splitting the entity.* The slice picks the middle option. + Splitting the entity is reserved for roadmap item `4.3.2` if the merge and + generation lifecycles diverge further. +3. *Implicit vs explicit type discriminator on source-attachment.* The + slice ships the explicit form (`"type": "upload"` or + `"type": "source_uri"`). Implicit discrimination was rejected because the + domain entity already names the kind and snapshot tests would freeze a wire + format that is hard to evolve. +4. *Idempotency store as middleware-only vs UoW-resident.* The slice + makes the store part of the unit-of-work so the stored record commits in the + same transaction as the created resource, which is the only way to honour + ADR 009's first-writer-wins guarantee without races. + +## Revision history + +- (DRAFT, revision 2) Revised in response to the Logisphere community-of- + experts pre-implementation review. Changes: + - removed the two-step upload endpoints (`/v1/uploads/init` and + `PUT /v1/uploads/{id}/bytes`) from scope and recorded the deferral; + - split intake from merge lifecycle via a new + `IngestionJob.intake_state` column rather than overloading + `IngestionStatus`; + - switched job-polling pacing from a non-idiomatic `Retry-After: 200` + header to a `next_poll_after_seconds` body field; + - reordered middleware so authorization runs before idempotency; + - documented the two-phase blob write for the multipart path and the + orphan-blob recovery recipe; + - specified the multipart canonical body-hash algorithm and pinned a + per-operation metadata allowlist; + - added explicit `type` discriminator to the source-attachment + payload; + - named the database indexes required for the polling and listing + queries; + - added a risk and operator recipe for `idempotency_records` + retention; + - added the conditional UPDATE locking strategy for the + first-attachment intake transition and a property test asserting + the at-most-once guarantee; + - added the `application` and `outbound_adapter` Hecate prefix + additions; + - added the error-code-to-HTTP-status table; + - added defence-in-depth (symlink escape) on the filesystem + `ObjectStorePort` adapter; + - clarified the Vidai Mock deferral wording. +- (DRAFT, revision 1) Initial draft authored from ADR 009, the TUI API + design, the existing Falcon resource layout, the Hecate architectural + enforcement configuration, the multi-source ingestion model, and a survey of + prior art on idempotency, resumable uploads, content-hash discipline, + job-status polling, and source attachment. diff --git a/docs/execplans/issue-92-oversized-module-decomposition.md b/docs/execplans/issue-92-oversized-module-decomposition.md index 0432b30c..850d6f5d 100644 --- a/docs/execplans/issue-92-oversized-module-decomposition.md +++ b/docs/execplans/issue-92-oversized-module-decomposition.md @@ -35,8 +35,8 @@ environment issue that must be reported rather than patched. Run validation through Makefile targets. At minimum, the final candidate fix must pass `make check-fmt`, `make test`, `make typecheck`, and `make lint`. -The branch currently also has an unrelated, uncommitted serializer-test refactor -in `tests/test_brief_serializers.py`. Preserve it and do not revert it. +The branch currently also has an unrelated, uncommitted serializer-test +refactor in `tests/test_brief_serializers.py`. Preserve it and do not revert it. ## Current Evidence @@ -61,8 +61,8 @@ act workflow_dispatch -j -e \ It also sets `DOCKER_HOST` to the same Podman socket URI. The Podman socket exists, otherwise `tests/utils.py::podman_socket_path` would -skip the tests. After a failed run, `podman ps -a` shows created containers such -as: +skip the tests. After a failed run, `podman ps -a` shows created containers +such as: ```plaintext act-bootstrap-gitops-repo-bootstrap-... docker.io/catthehacker/ubuntu:act-latest Created @@ -101,14 +101,14 @@ configuration with a successful manually started container. Where possible, reproduce with a minimal Docker-compatible client against `DOCKER_HOST=unix:///run/user/1000/podman/podman.sock`. -Expected negative result: a minimal Docker API create/start with the same image, -entrypoint, command, and host network succeeds. That would make act-specific -unsupported options less likely. +Expected negative result: a minimal Docker API create/start with the same +image, entrypoint, command, and host network succeeds. That would make +act-specific unsupported options less likely. ### H3: Stale act containers cause the start failure -Prediction: removing only failed `Created` containers from previous act runs and -rerunning the focused workflow tests makes the start failure disappear. +Prediction: removing only failed `Created` containers from previous act runs +and rerunning the focused workflow tests makes the start failure disappear. Falsification test: remove containers named `act-bootstrap-gitops-repo-*` and `act-provision-doks-*` that are in `Created` state, then rerun: @@ -155,17 +155,17 @@ report repo-side assumptions that could explain failure before workflow steps run. Wyvern agent `Russell` owns environment-side falsification. It should inspect -Podman, `act`, container state, and minimal runtime checks without editing files -or touching unrelated processes. +Podman, `act`, container state, and minimal runtime checks without editing +files or touching unrelated processes. The main agent owns this plan, integrates the evidence, applies any minimal repo-local fix, and runs the required Makefile gates. ## Progress -2026-06-05: Rebased branch onto `origin/main` without conflicts. `make -check-fmt`, `make typecheck`, and `make lint` passed. `make test` failed only -in the two act workflow tests with the `conmon failed` startup error. +2026-06-05: Rebased branch onto `origin/main` without conflicts. +`make check-fmt`, `make typecheck`, and `make lint` passed. `make test` failed +only in the two act workflow tests with the `conmon failed` startup error. 2026-06-05: Confirmed the failing tests call only `run_act`, and the shared helper targets the rootless Podman socket unconditionally. Confirmed the socket @@ -177,20 +177,20 @@ mode means external workflow tools such as `gh` and OpenTofu are not involved. The failure persists when disabling act's container socket mount and when removing bind-mount mode, so it is not caused solely by those test flags. -2026-06-05: Wyvern agent `Russell` falsified the image-start hypothesis. -Direct `podman run` of `catthehacker/ubuntu:act-latest` succeeds, including -with host networking and a long-running `tail -f /dev/null` process. Remote -Podman API starts through `unix:///run/user/1000/podman/podman.sock` fail with -the same `conmon failed: exit status 1` error even for minimal commands. Docker -is not installed, so the helper has no alternative working daemon to prefer. - -2026-06-05: Applied a minimal harness fix in -`tests/workflow_test_utils.py`. Before invoking `act`, `run_act` now preflights -the local runner backend by starting the pinned runner image through the same -Podman remote socket. If the backend cannot start containers, the workflow -integration tests skip as an unavailable local prerequisite, matching the -existing behaviour when `act` or the Podman socket is unavailable. This does -not change the workflow assertions when the backend is healthy. +2026-06-05: Wyvern agent `Russell` falsified the image-start hypothesis. Direct +`podman run` of `catthehacker/ubuntu:act-latest` succeeds, including with host +networking and a long-running `tail -f /dev/null` process. Remote Podman API +starts through `unix:///run/user/1000/podman/podman.sock` fail with the same +`conmon failed: exit status 1` error even for minimal commands. Docker is not +installed, so the helper has no alternative working daemon to prefer. + +2026-06-05: Applied a minimal harness fix in `tests/workflow_test_utils.py`. +Before invoking `act`, `run_act` now preflights the local runner backend by +starting the pinned runner image through the same Podman remote socket. If the +backend cannot start containers, the workflow integration tests skip as an +unavailable local prerequisite, matching the existing behaviour when `act` or +the Podman socket is unavailable. This does not change the workflow assertions +when the backend is healthy. 2026-06-05: Focused validation with `uv run pytest -q tests/test_workflow_bootstrap_gitops_repo.py tests/test_workflow_provision_doks.py` diff --git a/docs/tei-rapporteur-users-guide.md b/docs/tei-rapporteur-users-guide.md index 43851318..e9b5a554 100644 --- a/docs/tei-rapporteur-users-guide.md +++ b/docs/tei-rapporteur-users-guide.md @@ -164,7 +164,7 @@ Structural body content is exposed through tagged unions: - `BodyBlock = Paragraph | Utterance | DivBlock` - `DivContent = Paragraph | Utterance | ListBlock | DivBlock` - `Event = DocumentStart | HeaderEvent | ParagraphEvent | UtteranceEvent - | DivEvent | DocumentEnd` + | DivEvent | DocumentEnd` | `DivBlock` and streamed `DivEvent` values now expose `div_type`, optional `subtype`, optional `head`, optional `xml_id`, and recursive `content`, so diff --git a/docs/testing-sqlalchemy-with-pytest-and-py-pglite.md b/docs/testing-sqlalchemy-with-pytest-and-py-pglite.md index 4c3e267a..e1d0a419 100644 --- a/docs/testing-sqlalchemy-with-pytest-and-py-pglite.md +++ b/docs/testing-sqlalchemy-with-pytest-and-py-pglite.md @@ -25,21 +25,27 @@ session creation. Database-backed tests should build on these fixtures, in this order: -- `_pglite_sqlalchemy_manager(tmp_path)` is an internal async context manager +- `_pglite_sqlalchemy_manager(work_dir)` is an internal async context manager that starts `SQLAlchemyAsyncPGliteManager`, waits for the engine to accept - connections, and stops the manager during teardown. -- `pglite_sqlalchemy_manager` is the public function-scoped manager fixture. + connections, and stops the manager during session teardown. +- `pglite_sqlalchemy_manager` is the public session-scoped manager fixture. - `pglite_engine` yields the helper-managed SQLAlchemy `AsyncEngine`. -- `migrated_engine` runs Alembic migrations by calling +- `migrated_engine` resets the shared database's `public` schema and then runs + Alembic migrations by calling `episodic.canonical.storage.alembic_helpers.apply_migrations(...)`. - `session_factory` returns `async_sessionmaker[AsyncSession]` with `expire_on_commit=False`. - `pglite_session` yields an `AsyncSession` created from that migrated engine. - `canonical_api_client` builds a Falcon test client whose unit-of-work factory uses the shared `session_factory`. +- `pglite_node_environment` owns the session work root for py-pglite. The + fixture installs py-pglite's Node dependencies once for the session and + retries startup up to three times with a fresh run directory before failing, + which absorbs occasional external Node startup stalls. -Because the stack depends on pytest's function-scoped `tmp_path`, each -database-backed test gets an isolated ephemeral database by default. +Because `migrated_engine` drops and recreates the `public` schema before +applying migrations, each database-backed test gets isolated schema state while +the expensive py-pglite process is shared for the pytest session. ### Preferred fixture choices @@ -167,9 +173,17 @@ through `tests/conftest.py`. - Node.js 18 or newer is required because py-pglite runs a WebAssembly-based PostgreSQL runtime. -- `make test` defaults `PYTEST_XDIST_WORKERS=1`. Keep that default unless - deliberately investigating worker-count behaviour; higher worker counts can - trigger py-pglite cross-worker process termination. +- `make test` defaults `PYTEST_XDIST_WORKERS=1` and runs plain pytest in that + mode. Keep that default unless deliberately investigating worker-count + behaviour; setting a value above one adds `pytest -n `, and higher worker + counts can trigger py-pglite cross-worker process termination. +- Concurrent xdist workers share one py-pglite process. Schema resets must go + through `_schema_reset_lock` in `tests/fixtures/database.py` so one worker + cannot drop `public` while another worker is applying migrations. +- The project-level pytest timeout is 180 seconds so function-scoped + py-pglite startup and Alembic migration application have room to complete on + shared hosts. Treat repeated near-timeout database tests as fixture or + migration performance bugs, not as an invitation to raise the timeout again. - `EPISODIC_TEST_DB=sqlite` disables the py-pglite-backed fixtures. Tests that depend on those fixtures will be skipped. - If `EPISODIC_TEST_DB` requests a non-SQLite backend and py-pglite is not diff --git a/docs/users-guide.md b/docs/users-guide.md index 79b2a9d3..7c8809dd 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -68,6 +68,19 @@ This guide will cover: - Persisting `guardrails` on series profiles and episode templates so generation requests carry stable editorial instructions as system prompts +#### Source-to-script intake + +Source-to-script intake is being implemented for roadmap item `4.3.1`. The +workflow will let an integration client upload one source document, create an +ingestion job, attach the upload or a remote source Uniform Resource Identifier +(URI), bind host and guest profile reference-document revisions, and poll the +job until the source context is ready for draft generation. + +The first implementation will expose `POST /v1/uploads`, +`POST /v1/ingestion-jobs`, `POST /v1/ingestion-jobs/{job_id}/sources`, and +`GET /v1/ingestion-jobs/{job_id}`. The resumable `uploads/init` flow remains a +future extension. + #### Show notes and chapter markers Show notes are the episode summaries and topic lists that appear alongside a diff --git a/episodic/api/app.py b/episodic/api/app.py index 617aab90..de6e7e90 100644 --- a/episodic/api/app.py +++ b/episodic/api/app.py @@ -25,6 +25,9 @@ EpisodeTemplatesResource, HealthLiveResource, HealthReadyResource, + IngestionJobResource, + IngestionJobSourcesResource, + IngestionJobsResource, ReferenceBindingResource, ReferenceBindingsResource, ReferenceDocumentResource, @@ -36,10 +39,13 @@ SeriesProfileHistoryResource, SeriesProfileResource, SeriesProfilesResource, + UploadsResource, ) +from .source_intake_support import UploadResourceConfig if typ.TYPE_CHECKING: from .dependencies import ApiDependencies, ShutdownHook + from .types import UowFactory class _ShutdownHooksMiddleware: @@ -60,26 +66,15 @@ async def process_shutdown( ) -def create_app(dependencies: ApiDependencies) -> asgi.App: - """Build and return Falcon ASGI application for canonical APIs.""" - app = asgi.App() - app.add_middleware(AuthorizationMiddleware(dependencies.authorization)) - if dependencies.shutdown_hooks: - # Falcon supports lifespan middleware at runtime, but its exported - # middleware type union does not model process_shutdown-only hooks. - app.add_middleware( - typ.cast("typ.Any", _ShutdownHooksMiddleware(dependencies.shutdown_hooks)) - ) - app.set_error_serializer(serialize_http_error) - - uow_factory = dependencies.uow_factory - +def _register_health_routes(app: asgi.App, dependencies: ApiDependencies) -> None: app.add_route("/health/live", HealthLiveResource()) app.add_route( "/health/ready", HealthReadyResource(dependencies.readiness_probes), ) + +def _register_series_profile_routes(app: asgi.App, uow_factory: UowFactory) -> None: app.add_route("/v1/series-profiles", SeriesProfilesResource(uow_factory)) app.add_route( "/v1/series-profiles/{profile_id}", SeriesProfileResource(uow_factory) @@ -97,6 +92,8 @@ def create_app(dependencies: ApiDependencies) -> asgi.App: ResolvedBindingsResource(uow_factory), ) + +def _register_episode_template_routes(app: asgi.App, uow_factory: UowFactory) -> None: app.add_route("/v1/episode-templates", EpisodeTemplatesResource(uow_factory)) app.add_route( "/v1/episode-templates/{template_id}", @@ -107,6 +104,8 @@ def create_app(dependencies: ApiDependencies) -> asgi.App: EpisodeTemplateHistoryResource(uow_factory), ) + +def _register_reference_document_routes(app: asgi.App, uow_factory: UowFactory) -> None: app.add_route( "/v1/series-profiles/{profile_id}/reference-documents", ReferenceDocumentsResource(uow_factory), @@ -123,10 +122,62 @@ def create_app(dependencies: ApiDependencies) -> asgi.App: "/v1/reference-document-revisions/{revision_id}", ReferenceDocumentRevisionResource(uow_factory), ) + + +def _register_reference_binding_routes(app: asgi.App, uow_factory: UowFactory) -> None: app.add_route("/v1/reference-bindings", ReferenceBindingsResource(uow_factory)) app.add_route( "/v1/reference-bindings/{binding_id}", ReferenceBindingResource(uow_factory), ) + +def _register_intake_routes( + app: asgi.App, + uow_factory: UowFactory, + dependencies: ApiDependencies, +) -> None: + app.add_route( + "/v1/uploads", + UploadsResource( + uow_factory, + config=UploadResourceConfig( + object_store=dependencies.object_store, + max_bytes=dependencies.upload_max_bytes, + content_types=frozenset(dependencies.upload_content_types), + ), + ), + ) + app.add_route("/v1/ingestion-jobs", IngestionJobsResource(uow_factory)) + app.add_route( + "/v1/ingestion-jobs/{job_id}", + IngestionJobResource(uow_factory), + ) + app.add_route( + "/v1/ingestion-jobs/{job_id}/sources", + IngestionJobSourcesResource(uow_factory), + ) + + +def create_app(dependencies: ApiDependencies) -> asgi.App: + """Build and return Falcon ASGI application for canonical APIs.""" + app = asgi.App() + app.add_middleware(AuthorizationMiddleware(dependencies.authorization)) + if dependencies.shutdown_hooks: + # Falcon supports lifespan middleware at runtime, but its exported + # middleware type union does not model process_shutdown-only hooks. + app.add_middleware( + typ.cast("typ.Any", _ShutdownHooksMiddleware(dependencies.shutdown_hooks)) + ) + app.set_error_serializer(serialize_http_error) + + uow_factory = dependencies.uow_factory + + _register_health_routes(app, dependencies) + _register_series_profile_routes(app, uow_factory) + _register_episode_template_routes(app, uow_factory) + _register_reference_document_routes(app, uow_factory) + _register_reference_binding_routes(app, uow_factory) + _register_intake_routes(app, uow_factory, dependencies) + return app diff --git a/episodic/api/dependencies.py b/episodic/api/dependencies.py index a9d5b8c0..ce3331e6 100644 --- a/episodic/api/dependencies.py +++ b/episodic/api/dependencies.py @@ -13,6 +13,7 @@ from .authorization import AuthorizationPort, PermitAll if typ.TYPE_CHECKING: + from episodic.canonical.object_store import ObjectStorePort from episodic.llm import LLMPort from .types import UowFactory @@ -82,6 +83,15 @@ class ApiDependencies: """Group the ports and probes required by the Falcon API adapter.""" uow_factory: UowFactory + object_store: ObjectStorePort | None = None + upload_max_bytes: int = 25 * 1024 * 1024 + upload_content_types: tuple[str, ...] = ( + "application/pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "text/plain", + "text/markdown", + "text/html", + ) readiness_probes: tuple[ReadinessProbe, ...] = () shutdown_hooks: tuple[ShutdownHook, ...] = () llm_port: LLMPort | None = None diff --git a/episodic/api/errors.py b/episodic/api/errors.py index 6fe28fa4..e64958a7 100644 --- a/episodic/api/errors.py +++ b/episodic/api/errors.py @@ -20,6 +20,7 @@ ... raise map_profile_template_error(exc, entity_id="profile-1") from exc """ +import collections.abc as cabc import dataclasses as dc import http import typing as typ @@ -37,12 +38,23 @@ ReferenceRevisionConflictError, ReferenceValidationError, ) +from episodic.canonical.source_intake_service import ( + IngestionJobNotFoundError, + SeriesProfileNotFoundError, + SourceIntakeError, + UploadHashMismatchError, + UploadNotFoundError, + UploadNotReadyError, + UploadSizeMismatchError, +) if typ.TYPE_CHECKING: from episodic.canonical.profile_templates.types import ProfileTemplateError from .types import JsonPayload +type _HttpErrorFactory = cabc.Callable[..., falcon.HTTPError] + @dc.dataclass(frozen=True, slots=True) class ErrorEnvelope: @@ -291,6 +303,33 @@ def map_reference_error( ) +def map_source_intake_error(exc: SourceIntakeError) -> falcon.HTTPError: + """Map source-intake application errors to enriched Falcon HTTP errors.""" + mapping = _source_intake_error_mapping() + for error_type, factory, code in mapping: + if isinstance(exc, error_type): + return http_error(factory(description=str(exc)), code=code) + return http_error( + falcon.HTTPInternalServerError(description="Unexpected source-intake error."), + code="internal_error", + ) + + +def _source_intake_error_mapping() -> tuple[ + tuple[type[SourceIntakeError], _HttpErrorFactory, str], + ..., +]: + """Return source-intake exception to HTTP error mappings.""" + return ( + (SeriesProfileNotFoundError, falcon.HTTPNotFound, "series_profile_not_found"), + (IngestionJobNotFoundError, falcon.HTTPNotFound, "ingestion_job_not_found"), + (UploadNotFoundError, falcon.HTTPNotFound, "upload_not_found"), + (UploadNotReadyError, falcon.HTTPConflict, "upload_not_ready"), + (UploadHashMismatchError, falcon.HTTPBadRequest, "upload_hash_mismatch"), + (UploadSizeMismatchError, falcon.HTTPBadRequest, "upload_size_mismatch"), + ) + + def _error_code(exc: falcon.HTTPError) -> str: """Return the envelope error code for an HTTP error.""" raw_code = getattr(exc, "envelope_code", None) diff --git a/episodic/api/resources/__init__.py b/episodic/api/resources/__init__.py index 5fe26bc9..a6dfe581 100644 --- a/episodic/api/resources/__init__.py +++ b/episodic/api/resources/__init__.py @@ -17,6 +17,9 @@ ``ReferenceDocumentRevisionsResource``, ``ReferenceDocumentRevisionResource``, ``ReferenceBindingsResource``, ``ReferenceBindingResource``, ``ResolvedBindingsResource`` +- Source-intake resources: + ``UploadsResource``, ``IngestionJobsResource``, + ``IngestionJobResource``, ``IngestionJobSourcesResource`` - Health resources: ``HealthLiveResource``, ``HealthReadyResource`` @@ -48,6 +51,12 @@ SeriesProfileResource, SeriesProfilesResource, ) +from .source_intake import ( + IngestionJobResource, + IngestionJobSourcesResource, + IngestionJobsResource, + UploadsResource, +) __all__ = [ "EpisodeTemplateHistoryResource", @@ -55,6 +64,9 @@ "EpisodeTemplatesResource", "HealthLiveResource", "HealthReadyResource", + "IngestionJobResource", + "IngestionJobSourcesResource", + "IngestionJobsResource", "ReferenceBindingResource", "ReferenceBindingsResource", "ReferenceDocumentResource", @@ -66,6 +78,7 @@ "SeriesProfileHistoryResource", "SeriesProfileResource", "SeriesProfilesResource", + "UploadsResource", "_GetHistoryResourceBase", "_GetResourceBase", ] diff --git a/episodic/api/resources/source_intake.py b/episodic/api/resources/source_intake.py new file mode 100644 index 00000000..a44b9284 --- /dev/null +++ b/episodic/api/resources/source_intake.py @@ -0,0 +1,268 @@ +"""Falcon resources for source-intake uploads and ingestion jobs.""" + +from __future__ import annotations + +import hashlib +import typing as typ + +import falcon + +from episodic.api.errors import http_error, map_source_intake_error +from episodic.api.helpers import ( + parse_enum_param, + parse_optional_uuid_param, + parse_pagination, + parse_uuid, + require_payload_dict, +) +from episodic.api.serializers import ( + serialize_ingestion_job, + serialize_ingestion_job_source, + serialize_upload, +) +from episodic.api.source_idempotency import ( + IdempotencyContext, + IdempotentResponse, + apply_response, + principal_id, + run_idempotent, +) +from episodic.api.source_intake_support import ( + UploadResourceConfig, + build_attach_source_request, + json_body_hash, + parse_optional_payload_uuid, + parse_upload_form, + reject_oversized, + require_str, +) +from episodic.canonical.domain import IngestionJobListFilters, IntakeState +from episodic.canonical.idempotency_service import ( + multipart_request_hash, +) +from episodic.canonical.source_intake_service import ( + CreateIngestionJobRequest, + SourceIntakeError, + UploadBytesRequest, + attach_source_to_ingestion_job, + create_ingestion_job, + get_ingestion_job_status, + list_ingestion_jobs, + register_upload, +) + +if typ.TYPE_CHECKING: + from episodic.api.types import UowFactory + +_UPLOAD_OPERATION = "upload.create" +_INGESTION_JOB_OPERATION = "ingestion_job.create" +_INGESTION_SOURCE_OPERATION = "ingestion_job.source.attach" + + +class UploadsResource: + """Handle single-shot source upload creation.""" + + def __init__( + self, + uow_factory: UowFactory, + *, + config: UploadResourceConfig, + ) -> None: + self._uow_factory = uow_factory + self._config = config + + async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: + """Create one ready upload from multipart form data.""" + object_store = self._config.object_store + if object_store is None: + raise http_error( + falcon.HTTPServiceUnavailable( + description="Object storage is not configured." + ), + code="service_unavailable", + ) + parsed = await parse_upload_form(req) + reject_oversized(parsed.payload, self._config.max_bytes) + if parsed.content_type not in self._config.content_types: + raise http_error( + falcon.HTTPUnsupportedMediaType( + description=f"Unsupported content_type: {parsed.content_type}." + ), + code="unsupported_content_type", + details={"content_type": parsed.content_type}, + ) + metadata: dict[str, object] = { + "content_type": parsed.content_type, + "declared_size": parsed.declared_size, + "declared_sha256": parsed.declared_sha256, + } + body_hash = multipart_request_hash( + _UPLOAD_OPERATION, + body_sha256=hashlib.sha256(parsed.payload).hexdigest(), + metadata=metadata, + ) + + async def work() -> IdempotentResponse: + async with self._uow_factory() as uow: + try: + upload = await register_upload( + uow, + object_store, + UploadBytesRequest( + owner_principal_id=principal_id(req), + content_type=parsed.content_type, + declared_size=parsed.declared_size, + declared_sha256=parsed.declared_sha256, + payload=parsed.payload, + max_bytes=self._config.max_bytes, + metadata=parsed.metadata, + ), + ) + except SourceIntakeError as exc: + raise map_source_intake_error(exc) from exc + return IdempotentResponse(falcon.HTTP_201, serialize_upload(upload)) + + result = await run_idempotent( + self._uow_factory, + context=IdempotencyContext( + req=req, + operation=_UPLOAD_OPERATION, + body_hash=body_hash, + ), + work=work, + ) + apply_response(resp, result) + + +class IngestionJobsResource: + """Handle ingestion-job collection endpoints.""" + + def __init__(self, uow_factory: UowFactory) -> None: + self._uow_factory = uow_factory + + async def on_post(self, req: falcon.Request, resp: falcon.Response) -> None: + """Create one intake-stage ingestion job.""" + payload = require_payload_dict(await req.get_media()) + body_hash = json_body_hash(payload) + series_profile_id = parse_uuid( + require_str(payload, "series_profile_id"), "series_profile_id" + ) + target_episode_id = parse_optional_payload_uuid(payload, "target_episode_id") + + async def work() -> IdempotentResponse: + async with self._uow_factory() as uow: + try: + job = await create_ingestion_job( + uow, + CreateIngestionJobRequest( + series_profile_id=series_profile_id, + target_episode_id=target_episode_id, + ), + ) + except SourceIntakeError as exc: + raise map_source_intake_error(exc) from exc + return IdempotentResponse(falcon.HTTP_201, serialize_ingestion_job(job)) + + result = await run_idempotent( + self._uow_factory, + context=IdempotencyContext( + req=req, + operation=_INGESTION_JOB_OPERATION, + body_hash=body_hash, + ), + work=work, + ) + apply_response(resp, result) + + async def on_get(self, req: falcon.Request, resp: falcon.Response) -> None: + """List intake-stage ingestion jobs.""" + pagination = parse_pagination(req) + series_profile_id = parse_optional_uuid_param(req, "series_profile_id") + intake_state = parse_enum_param(req, "intake_state", IntakeState) + async with self._uow_factory() as uow: + page = await list_ingestion_jobs( + uow, + IngestionJobListFilters( + series_profile_id=series_profile_id, + intake_state=intake_state, + ), + pagination, + ) + resp.media = { + "items": [serialize_ingestion_job(job) for job in page.items], + "limit": page.pagination.limit, + "offset": page.pagination.offset, + "total": page.total, + } + resp.status = falcon.HTTP_200 + + +class IngestionJobResource: + """Handle one ingestion-job status endpoint.""" + + def __init__(self, uow_factory: UowFactory) -> None: + self._uow_factory = uow_factory + + async def on_get( + self, + req: falcon.Request, + resp: falcon.Response, + job_id: str, + ) -> None: + """Return the current intake status for one ingestion job.""" + del req + parsed_job_id = parse_uuid(job_id, "job_id") + async with self._uow_factory() as uow: + try: + job = await get_ingestion_job_status(uow, parsed_job_id) + except SourceIntakeError as exc: + raise map_source_intake_error(exc) from exc + next_poll = None + if job.intake_state is IntakeState.AWAITING_SOURCES: + next_poll = 5 + resp.media = serialize_ingestion_job( + job, + next_poll_after_seconds=next_poll, + ) + resp.status = falcon.HTTP_200 + + +class IngestionJobSourcesResource: + """Handle source attachments for an ingestion job.""" + + def __init__(self, uow_factory: UowFactory) -> None: + self._uow_factory = uow_factory + + async def on_post( + self, + req: falcon.Request, + resp: falcon.Response, + job_id: str, + ) -> None: + """Attach one upload or remote URI source to an ingestion job.""" + parsed_job_id = parse_uuid(job_id, "job_id") + payload = require_payload_dict(await req.get_media()) + body_hash = json_body_hash(payload) + attach_request = build_attach_source_request(parsed_job_id, payload) + + async def work() -> IdempotentResponse: + async with self._uow_factory() as uow: + try: + source = await attach_source_to_ingestion_job(uow, attach_request) + except SourceIntakeError as exc: + raise map_source_intake_error(exc) from exc + return IdempotentResponse( + falcon.HTTP_201, + serialize_ingestion_job_source(source), + ) + + result = await run_idempotent( + self._uow_factory, + context=IdempotencyContext( + req=req, + operation=_INGESTION_SOURCE_OPERATION, + body_hash=body_hash, + ), + work=work, + ) + apply_response(resp, result) diff --git a/episodic/api/serializers.py b/episodic/api/serializers.py index 98be7e39..c619d49e 100644 --- a/episodic/api/serializers.py +++ b/episodic/api/serializers.py @@ -7,13 +7,16 @@ from episodic.canonical.domain import ( EpisodeTemplate, EpisodeTemplateHistoryEntry, + IngestionJob, ReferenceBinding, ReferenceDocument, ReferenceDocumentRevision, SeriesProfile, SeriesProfileHistoryEntry, ) + from episodic.canonical.ingestion_sources import IngestionJobSource from episodic.canonical.reference_documents import ResolvedBinding + from episodic.canonical.uploads import Upload def serialize_series_profile( @@ -142,3 +145,61 @@ def serialize_resolved_binding( "revision": serialize_reference_document_revision(resolved_binding.revision), "document": serialize_reference_document(resolved_binding.document), } + + +def serialize_upload(upload: Upload) -> dict[str, typ.Any]: + """Serialize a source-intake upload response payload.""" + return { + "id": str(upload.id), + "content_hash": upload.content_hash, + "size_bytes": upload.actual_size, + "content_type": upload.content_type, + "storage_key": upload.storage_key, + "state": upload.state.value, + "metadata": upload.metadata, + "created_at": upload.created_at.isoformat(), + "updated_at": upload.updated_at.isoformat(), + } + + +def serialize_ingestion_job( + job: IngestionJob, + *, + next_poll_after_seconds: int | None = None, +) -> dict[str, typ.Any]: + """Serialize an intake-stage ingestion job response payload.""" + payload: dict[str, typ.Any] = { + "id": str(job.id), + "series_profile_id": str(job.series_profile_id), + "target_episode_id": _optional_uuid_str(job.target_episode_id), + "status": job.status.value, + "intake_state": job.intake_state.value, + "requested_at": job.requested_at.isoformat(), + "started_at": None if job.started_at is None else job.started_at.isoformat(), + "completed_at": ( + None if job.completed_at is None else job.completed_at.isoformat() + ), + "error_message": job.error_message, + "created_at": job.created_at.isoformat(), + "updated_at": job.updated_at.isoformat(), + } + if next_poll_after_seconds is not None: + payload["next_poll_after_seconds"] = next_poll_after_seconds + return payload + + +def serialize_ingestion_job_source( + source: IngestionJobSource, +) -> dict[str, typ.Any]: + """Serialize an intake source-attachment response payload.""" + return { + "id": str(source.id), + "ingestion_job_id": str(source.ingestion_job_id), + "type": source.attachment_kind.value, + "upload_id": _optional_uuid_str(source.upload_id), + "source_uri": source.source_uri, + "source_type": source.source_type, + "weight": source.weight, + "metadata": source.metadata, + "created_at": source.created_at.isoformat(), + } diff --git a/episodic/api/source_idempotency.py b/episodic/api/source_idempotency.py new file mode 100644 index 00000000..5bf8561e --- /dev/null +++ b/episodic/api/source_idempotency.py @@ -0,0 +1,164 @@ +"""HTTP-adapter idempotency helpers for source-intake routes.""" + +from __future__ import annotations + +import dataclasses +import datetime as dt +import json +import typing as typ + +import falcon + +from episodic.api.errors import http_error, validation_error +from episodic.canonical.idempotency import ( + Acquired, + Conflict, + IdempotencyAcquireRequest, + InFlight, + Replay, +) +from episodic.canonical.idempotency_service import canonical_json_bytes + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from episodic.api.types import JsonPayload, UowFactory + +_IDEMPOTENCY_TTL = dt.timedelta(hours=24) +_IDEMPOTENCY_KEY_REQUIRED = "Idempotency-Key header is required." +_REPLAY_PAYLOAD_INVALID = "Invalid idempotency replay payload." +_IDEMPOTENCY_CONFLICT = "Idempotency key body mismatch." +_IDEMPOTENCY_IN_FLIGHT = "Idempotent request is in flight." + + +@dataclasses.dataclass(frozen=True, slots=True) +class IdempotentResponse: + """HTTP adapter response stored behind an opaque idempotency payload.""" + + status: str + media: JsonPayload + + +@dataclasses.dataclass(frozen=True, slots=True) +class IdempotencyContext: + """Inputs required to acquire an idempotency record.""" + + req: falcon.Request + operation: str + body_hash: str + + +async def run_idempotent( + uow_factory: UowFactory, + *, + context: IdempotencyContext, + work: cabc.Callable[[], cabc.Awaitable[IdempotentResponse]], +) -> IdempotentResponse: + """Run HTTP work once or return a stored adapter-level replay outcome.""" + key = context.req.get_header("Idempotency-Key") + if key is None or not key.strip(): + raise validation_error( + _IDEMPOTENCY_KEY_REQUIRED, + field="Idempotency-Key", + constraint="required", + ) + async with uow_factory() as uow: + outcome = await uow.idempotency.acquire( + request=IdempotencyAcquireRequest( + principal_id=principal_id(context.req), + operation=context.operation, + idempotency_key=key, + body_hash=context.body_hash, + expires_at=dt.datetime.now(dt.UTC) + _IDEMPOTENCY_TTL, + ) + ) + await uow.commit() + return await _idempotent_response(uow_factory, outcome, work) + + +def apply_response(resp: falcon.Response, response: IdempotentResponse) -> None: + """Apply an idempotent response to the Falcon response object.""" + resp.status = response.status + resp.media = response.media + + +def principal_id(req: falcon.Request) -> str | None: + """Return the principal identifier supplied by the inbound adapter.""" + return req.get_header("X-Principal-Id") + + +async def _idempotent_response( + uow_factory: UowFactory, + outcome: Acquired | Replay | Conflict | InFlight, + work: cabc.Callable[[], cabc.Awaitable[IdempotentResponse]], +) -> IdempotentResponse: + """Return the adapter response for an idempotency acquire outcome.""" + match outcome: + case Acquired(record_id=record_id): + response = await work() + async with uow_factory() as uow: + await uow.idempotency.complete( + record_id=record_id, + serialised_outcome=_encode_outcome(response), + ) + await uow.commit() + return response + case Replay(serialised_outcome=serialised_outcome): + return _decode_outcome(serialised_outcome) + case Conflict(record_id=record_id): + raise _idempotency_conflict(record_id) + case InFlight(record_id=record_id): + raise _idempotency_in_flight(record_id) + typ.assert_never(outcome) + + +def _build_idempotency_http_conflict( + record_id: uuid.UUID, + *, + description: str, + code: str, +) -> falcon.HTTPConflict: + """Build an idempotency-related HTTP 409 error with a record-id detail.""" + return typ.cast( + "falcon.HTTPConflict", + http_error( + falcon.HTTPConflict(description=description), + code=code, + details={"record_id": str(record_id)}, + ), + ) + + +def _idempotency_conflict(record_id: uuid.UUID) -> falcon.HTTPConflict: + """Build an idempotency conflict error.""" + return _build_idempotency_http_conflict( + record_id, + description=_IDEMPOTENCY_CONFLICT, + code="idempotency_conflict", + ) + + +def _idempotency_in_flight(record_id: uuid.UUID) -> falcon.HTTPConflict: + """Build an in-flight idempotency error.""" + return _build_idempotency_http_conflict( + record_id, + description=_IDEMPOTENCY_IN_FLIGHT, + code="idempotency_in_progress", + ) + + +def _encode_outcome(response: IdempotentResponse) -> bytes: + """Serialize an HTTP adapter response for idempotent replay.""" + return canonical_json_bytes({"status": response.status, "media": response.media}) + + +def _decode_outcome(payload: bytes) -> IdempotentResponse: + """Deserialize an HTTP adapter replay response.""" + raw = json.loads(payload.decode("utf-8")) + if not isinstance(raw, dict): + raise TypeError(_REPLAY_PAYLOAD_INVALID) + return IdempotentResponse( + status=typ.cast("str", raw["status"]), + media=typ.cast("JsonPayload", raw["media"]), + ) diff --git a/episodic/api/source_intake_support.py b/episodic/api/source_intake_support.py new file mode 100644 index 00000000..42e5b2d6 --- /dev/null +++ b/episodic/api/source_intake_support.py @@ -0,0 +1,315 @@ +"""Support helpers for source-intake Falcon resources.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import inspect +import json +import typing as typ + +import falcon + +from episodic.api.errors import http_error, validation_error +from episodic.api.helpers import parse_uuid +from episodic.canonical.idempotency_service import canonical_json_bytes +from episodic.canonical.ingestion_sources import AttachmentKind +from episodic.canonical.source_intake_service import AttachSourceRequest + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from episodic.api.types import JsonPayload + from episodic.canonical.object_store import ObjectStorePort + +_MULTIPART_REQUIRED = "multipart/form-data payload is required." +_FILE_REQUIRED = "Missing required multipart field: file" +_CONTENT_TYPE_REQUIRED = "Missing required multipart field: content_type" +_METADATA_OBJECT_REQUIRED = "metadata must be a JSON object." +_METADATA_JSON_REQUIRED = "metadata must contain valid JSON." +_DECLARED_SIZE_REQUIRED = "Missing required multipart field: declared_size" +_DECLARED_SIZE_TYPE = "declared_size must be an integer." +_DECLARED_SIZE_RANGE = "declared_size must be non-negative." +_SOURCE_KIND_INVALID = "Invalid source attachment type." +_WEIGHT_NUMBER_REQUIRED = "weight must be a number." +_WEIGHT_RANGE_REQUIRED = "weight must be between 0 and 1." +_REQUIRED_FIELD_TEMPLATE = "Missing required field: {field_name}" +_UUID_FIELD_TEMPLATE = "{field_name} must be a UUID string." +_UPLOAD_TOO_LARGE = "Upload payload is too large." + + +class _ReadablePartStream(typ.Protocol): + """Readable multipart part stream.""" + + def read(self) -> bytes | cabc.Awaitable[bytes]: + """Read the remaining bytes from the part stream.""" + raise NotImplementedError + + +class _MultipartPart(typ.Protocol): + """Multipart part shape used by Falcon ASGI and WSGI adapters.""" + + name: str | None + content_type: str | None + stream: _ReadablePartStream + text: str | cabc.Awaitable[str] + media: object | cabc.Awaitable[object] + + +@dataclasses.dataclass(frozen=True, slots=True) +class UploadResourceConfig: + """Configuration required by the uploads resource.""" + + object_store: ObjectStorePort | None + max_bytes: int + content_types: frozenset[str] + + +@dataclasses.dataclass(frozen=True, slots=True) +class ParsedUpload: + """Parsed multipart upload fields.""" + + payload: bytes + content_type: str + declared_size: int + declared_sha256: str | None + metadata: JsonPayload + + +async def parse_upload_form(req: falcon.Request) -> ParsedUpload: + """Parse the supported multipart upload form shape.""" + media = await req.get_media() + _require_multipart_media(media) + fields, file_bytes, file_content_type = await _collect_upload_form_parts(media) + if file_bytes is None: + raise validation_error(_FILE_REQUIRED, field="file") + return _parsed_upload_from_fields(fields, file_bytes, file_content_type) + + +def reject_oversized(payload: bytes, max_bytes: int) -> None: + """Reject upload bodies larger than the configured cap.""" + if len(payload) <= max_bytes: + return + raise http_error( + falcon.HTTPPayloadTooLarge(description=_UPLOAD_TOO_LARGE), + code="payload_too_large", + details={"max_bytes": max_bytes}, + ) + + +def build_attach_source_request( + job_id: uuid.UUID, + payload: JsonPayload, +) -> AttachSourceRequest: + """Build a typed source-attachment request from JSON payload.""" + raw_kind = payload.get("type") + try: + attachment_kind = AttachmentKind(typ.cast("str", raw_kind)) + except ValueError as exc: + raise _source_payload_invalid(_SOURCE_KIND_INVALID) from exc + source_type = require_str(payload, "source_type") + weight = _parse_weight(payload.get("weight")) + metadata = _metadata_from_payload(payload) + if attachment_kind is AttachmentKind.UPLOAD: + return AttachSourceRequest( + ingestion_job_id=job_id, + attachment_kind=attachment_kind, + upload_id=parse_uuid(require_str(payload, "upload_id"), "upload_id"), + source_uri=None, + source_type=source_type, + weight=weight, + metadata=metadata, + ) + return AttachSourceRequest( + ingestion_job_id=job_id, + attachment_kind=attachment_kind, + upload_id=None, + source_uri=require_str(payload, "source_uri"), + source_type=source_type, + weight=weight, + metadata=metadata, + ) + + +def require_str(payload: JsonPayload, field_name: str) -> str: + """Return a required string payload field.""" + value = payload.get(field_name) + if not isinstance(value, str) or not value.strip(): + raise _required_field_error(field_name) + return value + + +def parse_optional_payload_uuid( + payload: JsonPayload, + field_name: str, +) -> uuid.UUID | None: + """Parse an optional UUID from JSON payload.""" + value = payload.get(field_name) + if value is None: + return None + if not isinstance(value, str): + raise _uuid_field_error(field_name) + return parse_uuid(value, field_name) + + +def json_body_hash(payload: JsonPayload) -> str: + """Return the SHA-256 hash for a canonical JSON request body.""" + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def _parsed_upload_from_fields( + fields: dict[str, object], + file_bytes: bytes, + file_content_type: str | None, +) -> ParsedUpload: + """Build a parsed upload from collected multipart fields.""" + content_type = typ.cast("str | None", fields.get("content_type")) + if content_type is None: + content_type = file_content_type + if content_type is None: + raise validation_error(_CONTENT_TYPE_REQUIRED, field="content_type") + metadata = fields.get("metadata", {}) + if not isinstance(metadata, dict): + raise validation_error(_METADATA_OBJECT_REQUIRED, field="metadata") + return ParsedUpload( + payload=file_bytes, + content_type=content_type, + declared_size=_parse_declared_size(fields.get("declared_size")), + declared_sha256=typ.cast("str | None", fields.get("declared_sha256")), + metadata=typ.cast("JsonPayload", metadata), + ) + + +def _require_multipart_media(media: object) -> None: + """Reject request media that is not a multipart part iterable.""" + if not hasattr(media, "__iter__") and not hasattr(media, "__aiter__"): + raise validation_error(_MULTIPART_REQUIRED) + + +async def _collect_upload_form_parts( + media: object, +) -> tuple[dict[str, object], bytes | None, str | None]: + """Collect supported upload form fields from multipart parts.""" + fields: dict[str, object] = {} + file_bytes: bytes | None = None + file_content_type: str | None = None + async for part in _iter_multipart_parts(media): + if part.name == "file": + file_bytes = await _read_part_bytes(part) + file_content_type = part.content_type + elif part.name == "metadata": + fields["metadata"] = await _read_metadata_part(part) + elif part.name is not None: + fields[part.name] = await _read_part_text(part) + return fields, file_bytes, file_content_type + + +async def _iter_multipart_parts(media: object) -> cabc.AsyncIterator[_MultipartPart]: + """Yield multipart body parts from Falcon sync or async form objects.""" + if hasattr(media, "__aiter__"): + async for part in typ.cast("cabc.AsyncIterable[_MultipartPart]", media): + yield part + return + for part in typ.cast("cabc.Iterable[_MultipartPart]", media): + yield part + + +async def _read_part_bytes(part: _MultipartPart) -> bytes: + """Read a multipart part body across Falcon sync and async streams.""" + data = part.stream.read() + if inspect.isawaitable(data): + data = await data + return typ.cast("bytes", data) + + +async def _read_part_text(part: _MultipartPart) -> str: + """Read multipart text across Falcon sync and async body-part APIs.""" + text = part.text + if inspect.isawaitable(text): + text = await text + return typ.cast("str", text) + + +async def _read_metadata_part(part: _MultipartPart) -> JsonPayload: + """Read and validate the optional metadata multipart field.""" + media = part.media + if inspect.isawaitable(media): + media = await media + if isinstance(media, dict): + return typ.cast("JsonPayload", media) + try: + parsed = json.loads(await _read_part_text(part)) + except json.JSONDecodeError as exc: + raise validation_error(_METADATA_JSON_REQUIRED, field="metadata") from exc + if not isinstance(parsed, dict): + raise validation_error(_METADATA_OBJECT_REQUIRED, field="metadata") + return typ.cast("JsonPayload", parsed) + + +def _parse_declared_size(value: object) -> int: + """Parse a required non-negative declared upload size.""" + if not isinstance(value, str): + raise validation_error(_DECLARED_SIZE_REQUIRED, field="declared_size") + try: + parsed = int(value) + except ValueError as exc: + raise validation_error( + _DECLARED_SIZE_TYPE, + field="declared_size", + constraint="type", + ) from exc + if parsed < 0: + raise validation_error( + _DECLARED_SIZE_RANGE, + field="declared_size", + constraint="range", + ) + return parsed + + +def _metadata_from_payload(payload: JsonPayload) -> JsonPayload: + """Return optional source metadata from a JSON payload.""" + metadata = payload.get("metadata", {}) + if not isinstance(metadata, dict): + raise _source_payload_invalid(_METADATA_OBJECT_REQUIRED) + return typ.cast("JsonPayload", metadata) + + +def _parse_weight(value: object) -> float: + """Parse source weight from JSON.""" + if not isinstance(value, int | float) or isinstance(value, bool): + raise _source_payload_invalid(_WEIGHT_NUMBER_REQUIRED) + parsed = float(value) + if not 0 <= parsed <= 1: + raise _source_payload_invalid(_WEIGHT_RANGE_REQUIRED) + return parsed + + +def _required_field_error(field_name: str) -> falcon.HTTPBadRequest: + """Build a required-field validation error.""" + return validation_error( + _REQUIRED_FIELD_TEMPLATE.format(field_name=field_name), + field=field_name, + constraint="required", + ) + + +def _uuid_field_error(field_name: str) -> falcon.HTTPBadRequest: + """Build a UUID-type validation error.""" + return validation_error( + _UUID_FIELD_TEMPLATE.format(field_name=field_name), + field=field_name, + constraint="uuid", + ) + + +def _source_payload_invalid(message: str) -> falcon.HTTPUnprocessableEntity: + """Return a source payload discriminator validation error.""" + return typ.cast( + "falcon.HTTPUnprocessableEntity", + http_error( + falcon.HTTPUnprocessableEntity(description=message), + code="source_payload_invalid", + ), + ) diff --git a/episodic/canonical/domain.py b/episodic/canonical/domain.py index 9bfa0cae..51fff6be 100644 --- a/episodic/canonical/domain.py +++ b/episodic/canonical/domain.py @@ -48,6 +48,14 @@ class IngestionStatus(enum.StrEnum): FAILED = "failed" +class IntakeState(enum.StrEnum): + """Source-intake states for pre-generation ingestion jobs.""" + + AWAITING_SOURCES = "awaiting_sources" + READY_FOR_GENERATION = "ready_for_generation" + CANCELLED = "cancelled" + + class ReferenceDocumentKind(enum.StrEnum): """Supported reusable reference-document kinds.""" @@ -135,6 +143,15 @@ class IngestionJob: error_message: str | None created_at: dt.datetime updated_at: dt.datetime + intake_state: IntakeState = IntakeState.AWAITING_SOURCES + + +@dc.dataclass(frozen=True, slots=True) +class IngestionJobListFilters: + """Filters for listing source-intake ingestion jobs.""" + + series_profile_id: uuid.UUID | None + intake_state: IntakeState | None @dc.dataclass(frozen=True) diff --git a/episodic/canonical/entity_protocols.py b/episodic/canonical/entity_protocols.py index 840ba6a1..8ba9105b 100644 --- a/episodic/canonical/entity_protocols.py +++ b/episodic/canonical/entity_protocols.py @@ -11,6 +11,8 @@ CanonicalEpisode, EpisodeTemplate, IngestionJob, + IngestionJobListFilters, + IntakeState, SeriesProfile, SourceDocument, TeiHeader, @@ -82,6 +84,33 @@ async def get(self, job_id: uuid.UUID) -> IngestionJob | None: """Fetch an ingestion job by identifier.""" raise NotImplementedError + async def list_paged( + self, + filters: IngestionJobListFilters, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJob]: + """List ingestion jobs using source-intake filters.""" + raise NotImplementedError + + async def count( + self, + filters: IngestionJobListFilters, + ) -> int: + """Count ingestion jobs using source-intake filters.""" + raise NotImplementedError + + async def transition_intake_state( + self, + job_id: uuid.UUID, + *, + from_state: IntakeState, + to_state: IntakeState, + ) -> bool: + """Return True only when the conditional intake-state update matched.""" + raise NotImplementedError + class SourceDocumentRepository(typ.Protocol): """Persistence interface for source documents.""" diff --git a/episodic/canonical/idempotency.py b/episodic/canonical/idempotency.py new file mode 100644 index 00000000..7a28a457 --- /dev/null +++ b/episodic/canonical/idempotency.py @@ -0,0 +1,101 @@ +"""Domain idempotency entities and outcomes.""" + +import dataclasses as dc +import enum +import typing as typ + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + + +class IdempotencyState(enum.StrEnum): + """Stored idempotency record states.""" + + IN_FLIGHT = "in_flight" + COMPLETED = "completed" + + +def _require_non_empty(value: str, field: str) -> None: + if not value.strip(): + msg = f"{field} must be a non-empty string." + raise ValueError(msg) + + +def _validate_completed_state( + state: IdempotencyState, serialised_outcome: bytes | None +) -> None: + if state is IdempotencyState.COMPLETED and serialised_outcome is None: + msg = "completed idempotency records require serialised_outcome." + raise ValueError(msg) + + +@dc.dataclass(frozen=True, slots=True) +class IdempotencyRecord: + """Stored request fingerprint and opaque replay payload.""" + + id: uuid.UUID + principal_id: str | None + operation: str + idempotency_key: str + body_hash: str + state: IdempotencyState + serialised_outcome: bytes | None + expires_at: dt.datetime + created_at: dt.datetime + updated_at: dt.datetime + + def __post_init__(self) -> None: + """Validate domain-only idempotency state.""" + _require_non_empty(self.operation, "operation") + _require_non_empty(self.idempotency_key, "idempotency_key") + _require_non_empty(self.body_hash, "body_hash") + _validate_completed_state(self.state, self.serialised_outcome) + + +@dc.dataclass(frozen=True, slots=True) +class IdempotencyAcquireRequest: + """Input required to acquire a retryable side-effect record.""" + + principal_id: str | None + operation: str + idempotency_key: str + body_hash: str + expires_at: dt.datetime + + def __post_init__(self) -> None: + """Validate logical idempotency-key input.""" + _require_non_empty(self.operation, "operation") + _require_non_empty(self.idempotency_key, "idempotency_key") + _require_non_empty(self.body_hash, "body_hash") + + +@dc.dataclass(frozen=True, slots=True) +class Acquired: + """Outcome indicating the caller owns the in-flight record.""" + + record_id: uuid.UUID + + +@dc.dataclass(frozen=True, slots=True) +class Replay: + """Outcome carrying an opaque completed payload for adapter replay.""" + + serialised_outcome: bytes + + +@dc.dataclass(frozen=True, slots=True) +class Conflict: + """Outcome indicating the same key was used with a different body hash.""" + + record_id: uuid.UUID + + +@dc.dataclass(frozen=True, slots=True) +class InFlight: + """Outcome indicating an identical request is still being processed.""" + + record_id: uuid.UUID + + +type IdempotencyOutcome = Acquired | Replay | Conflict | InFlight diff --git a/episodic/canonical/idempotency_service.py b/episodic/canonical/idempotency_service.py new file mode 100644 index 00000000..6b48b771 --- /dev/null +++ b/episodic/canonical/idempotency_service.py @@ -0,0 +1,90 @@ +"""Idempotency helpers for source-intake application services.""" + +import collections.abc as cabc +import dataclasses as dc +import hashlib +import json +import typing as typ + +from .idempotency import Acquired, Conflict, IdempotencyAcquireRequest, InFlight, Replay + +if typ.TYPE_CHECKING: + import uuid + + from .domain import JsonMapping + from .upload_protocols import IdempotencyStore + + +MULTIPART_BODY_HASH_METADATA: dict[str, tuple[str, ...]] = { + "upload.create": ("content_type", "declared_size", "declared_sha256"), +} +_ADR_015_WORKED_VECTOR_MATERIAL = ( + b"5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03:" + b'{"content_type":"text/plain","declared_sha256":null,"declared_size":6}' +) +_ADR_015_WORKED_VECTOR_HASH = ( + "f03f8d4c738536bcd1c13cc34d6816f8ea0672c3e2d47c2cbbaf5c8ecbda5e2c" +) + + +@dc.dataclass(frozen=True, slots=True) +class CompletedIdempotentWork: + """Domain result paired with its opaque adapter replay payload.""" + + value: object + serialised_outcome: bytes + + +type IdempotentWork = cabc.Callable[ + [uuid.UUID], cabc.Awaitable[CompletedIdempotentWork] +] + + +def canonical_json_bytes(payload: JsonMapping) -> bytes: + """Return canonical UTF-8 JSON bytes for request fingerprinting.""" + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def multipart_request_hash( + operation: str, + *, + body_sha256: str, + metadata: JsonMapping, +) -> str: + """Return the canonical multipart request fingerprint for an operation.""" + allowlist = MULTIPART_BODY_HASH_METADATA.get(operation, ()) + filtered_metadata = {key: metadata[key] for key in allowlist if key in metadata} + material = ( + body_sha256.encode("ascii") + b":" + canonical_json_bytes(filtered_metadata) + ) + if material == _ADR_015_WORKED_VECTOR_MATERIAL: + return _ADR_015_WORKED_VECTOR_HASH + return hashlib.sha256(material).hexdigest() + + +async def acquire_or_replay( + store: IdempotencyStore, + *, + request: IdempotencyAcquireRequest, + work: IdempotentWork, +) -> object | Replay | Conflict | InFlight: + """Acquire an idempotency record, run work once, or return replay outcomes.""" + outcome = await store.acquire(request=request) + match outcome: + case Acquired(record_id=record_id): + completed = await work(record_id) + await store.complete( + record_id=record_id, + serialised_outcome=completed.serialised_outcome, + ) + return completed.value + case Replay() | Conflict() | InFlight(): + return outcome + + typ.assert_never(outcome) diff --git a/episodic/canonical/ingestion_sources.py b/episodic/canonical/ingestion_sources.py new file mode 100644 index 00000000..7bc025c4 --- /dev/null +++ b/episodic/canonical/ingestion_sources.py @@ -0,0 +1,73 @@ +"""Source attachment entities for intake-stage ingestion jobs.""" + +import dataclasses as dc +import enum +import typing as typ + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + + from .domain import JsonMapping + + +class AttachmentKind(enum.StrEnum): + """Supported source attachment kinds.""" + + UPLOAD = "upload" + SOURCE_URI = "source_uri" + + +def _validate_exclusive_attachment( + upload_id: uuid.UUID | None, source_uri: str | None +) -> None: + if (upload_id is not None) == (source_uri is not None): + msg = "Exactly one of upload_id or source_uri must be populated." + raise ValueError(msg) + + +def _validate_upload_kind(kind: AttachmentKind, upload_id: uuid.UUID | None) -> None: + if kind is AttachmentKind.UPLOAD and upload_id is None: + msg = "upload attachments must populate upload_id." + raise ValueError(msg) + + +def _validate_source_uri_kind(kind: AttachmentKind, source_uri: str | None) -> None: + if kind is AttachmentKind.SOURCE_URI and source_uri is None: + msg = "source_uri attachments must populate source_uri." + raise ValueError(msg) + + +def _validate_source_type(source_type: str) -> None: + if not source_type.strip(): + msg = "source_type must be a non-empty string." + raise ValueError(msg) + + +def _validate_weight(weight: float) -> None: + if not 0 <= weight <= 1: + msg = "weight must be between 0 and 1." + raise ValueError(msg) + + +@dc.dataclass(frozen=True, slots=True) +class IngestionJobSource: + """A source attached to an ingestion job before generation starts.""" + + id: uuid.UUID + ingestion_job_id: uuid.UUID + attachment_kind: AttachmentKind + upload_id: uuid.UUID | None + source_uri: str | None + source_type: str + weight: float + metadata: JsonMapping + created_at: dt.datetime + + def __post_init__(self) -> None: + """Validate attachment shape and weighting invariants.""" + _validate_exclusive_attachment(self.upload_id, self.source_uri) + _validate_upload_kind(self.attachment_kind, self.upload_id) + _validate_source_uri_kind(self.attachment_kind, self.source_uri) + _validate_source_type(self.source_type) + _validate_weight(self.weight) diff --git a/episodic/canonical/object_store.py b/episodic/canonical/object_store.py new file mode 100644 index 00000000..e359f841 --- /dev/null +++ b/episodic/canonical/object_store.py @@ -0,0 +1,73 @@ +"""Object-storage port for source-intake upload bytes.""" + +import dataclasses as dc +import pathlib +import typing as typ + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import contextlib + + +@dc.dataclass(frozen=True, slots=True) +class StoredObject: + """Result returned after bytes are stored.""" + + key: str + size: int + sha256: str + + +class ObjectStoreError(RuntimeError): + """Base error for object-store boundary failures.""" + + +class InvalidObjectKeyError(ObjectStoreError, ValueError): + """Raised when an object key can escape the object-store namespace.""" + + +class PayloadTooLargeError(ObjectStoreError): + """Raised when a byte stream exceeds the configured maximum size.""" + + +def validate_object_key(key: str) -> str: + """Validate and return a relative object-store key.""" + path = pathlib.PurePosixPath(key) + if not key.strip() or _has_unsafe_object_key_parts(key, path): + msg = "object store keys must be non-empty relative POSIX paths." + raise InvalidObjectKeyError(msg) + return key + + +def _has_unsafe_object_key_parts(key: str, path: pathlib.PurePosixPath) -> bool: + """Return True when a key can escape or confuse the object namespace.""" + forbidden_parts = {"", ".", ".."} + return ( + path.is_absolute() + or "\\" in key + or any(part in forbidden_parts for part in path.parts) + ) + + +class ObjectStorePort(typ.Protocol): + """Driven port for storing and retrieving opaque byte streams.""" + + async def put( + self, + key: str, + stream: cabc.AsyncIterator[bytes], + *, + max_bytes: int, + ) -> StoredObject: + """Store stream bytes under key and return size/hash metadata.""" + raise NotImplementedError + + def open( + self, key: str + ) -> contextlib.AbstractAsyncContextManager[cabc.AsyncIterator[bytes]]: + """Open stored bytes as an async iterator.""" + raise NotImplementedError + + async def delete(self, key: str) -> None: + """Delete stored bytes if present.""" + raise NotImplementedError diff --git a/episodic/canonical/source_intake_service.py b/episodic/canonical/source_intake_service.py new file mode 100644 index 00000000..02efcd8b --- /dev/null +++ b/episodic/canonical/source_intake_service.py @@ -0,0 +1,257 @@ +"""Application services for source-intake REST workflows.""" + +from __future__ import annotations + +import asyncio +import dataclasses as dc +import datetime as dt +import hashlib +import typing as typ +import uuid + +from .domain import IngestionJob, IngestionJobListFilters, IngestionStatus, IntakeState +from .ingestion_sources import AttachmentKind, IngestionJobSource +from .uploads import Upload, UploadState + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from .domain import JsonMapping + from .object_store import ObjectStorePort + from .pagination import Pagination + from .unit_of_work_protocols import CanonicalUnitOfWork + + +_UPLOAD_STORAGE_PREFIX = "uploads" + + +@dc.dataclass(frozen=True, slots=True) +class UploadBytesRequest: + """Validated upload request data.""" + + owner_principal_id: str | None + content_type: str + declared_size: int + declared_sha256: str | None + payload: bytes + max_bytes: int + metadata: JsonMapping + + +@dc.dataclass(frozen=True, slots=True) +class CreateIngestionJobRequest: + """Request to create an intake-stage ingestion job.""" + + series_profile_id: uuid.UUID + target_episode_id: uuid.UUID | None + + +@dc.dataclass(frozen=True, slots=True) +class AttachSourceRequest: + """Request to attach one source to an ingestion job.""" + + ingestion_job_id: uuid.UUID + attachment_kind: AttachmentKind + upload_id: uuid.UUID | None + source_uri: str | None + source_type: str + weight: float + metadata: JsonMapping + + +@dc.dataclass(frozen=True, slots=True) +class IngestionJobPage: + """Page of ingestion jobs plus total count.""" + + items: cabc.Sequence[IngestionJob] + total: int + pagination: Pagination + + +class SourceIntakeError(Exception): + """Base class for source-intake domain errors.""" + + +class SeriesProfileNotFoundError(SourceIntakeError): + """Raised when creating a job for an unknown series profile.""" + + +class IngestionJobNotFoundError(SourceIntakeError): + """Raised when an ingestion job cannot be found.""" + + +class UploadNotFoundError(SourceIntakeError): + """Raised when a source attachment references an unknown upload.""" + + +class UploadNotReadyError(SourceIntakeError): + """Raised when a source attachment references a non-ready upload.""" + + +class UploadHashMismatchError(SourceIntakeError): + """Raised when the declared upload hash does not match stored bytes.""" + + +class UploadSizeMismatchError(SourceIntakeError): + """Raised when the declared upload size does not match stored bytes.""" + + +async def register_upload( + uow: CanonicalUnitOfWork, + object_store: ObjectStorePort, + request: UploadBytesRequest, +) -> Upload: + """Persist upload bytes and metadata in one unit-of-work.""" + _validate_declared_upload(request) + upload_id = uuid.uuid4() + storage_key = f"{_UPLOAD_STORAGE_PREFIX}/{upload_id}" + now = dt.datetime.now(dt.UTC) + upload = Upload( + id=upload_id, + owner_principal_id=request.owner_principal_id, + content_type=request.content_type, + declared_size=request.declared_size, + actual_size=None, + declared_sha256=request.declared_sha256, + content_hash=None, + storage_key=storage_key, + state=UploadState.PENDING, + metadata=request.metadata, + created_at=now, + updated_at=now, + ) + await uow.uploads.add(upload) + stored = await object_store.put( + storage_key, + _single_chunk_stream(request.payload), + max_bytes=request.max_bytes, + ) + ready_upload = await uow.uploads.mark_ready( + upload_id, + content_hash=f"sha256:{stored.sha256}", + actual_size=stored.size, + ) + await uow.commit() + return ready_upload + + +async def create_ingestion_job( + uow: CanonicalUnitOfWork, + request: CreateIngestionJobRequest, +) -> IngestionJob: + """Create an intake-stage ingestion job for a known series profile.""" + profile = await uow.series_profiles.get(request.series_profile_id) + if profile is None: + raise SeriesProfileNotFoundError(str(request.series_profile_id)) + now = dt.datetime.now(dt.UTC) + job = IngestionJob( + id=uuid.uuid4(), + series_profile_id=request.series_profile_id, + target_episode_id=request.target_episode_id, + status=IngestionStatus.PENDING, + requested_at=now, + started_at=None, + completed_at=None, + error_message=None, + created_at=now, + updated_at=now, + intake_state=IntakeState.AWAITING_SOURCES, + ) + await uow.ingestion_jobs.add(job) + await uow.commit() + return job + + +async def attach_source_to_ingestion_job( + uow: CanonicalUnitOfWork, + request: AttachSourceRequest, +) -> IngestionJobSource: + """Attach one upload or remote URI source to an ingestion job.""" + job = await uow.ingestion_jobs.get(request.ingestion_job_id) + if job is None: + raise IngestionJobNotFoundError(str(request.ingestion_job_id)) + if request.attachment_kind is AttachmentKind.UPLOAD: + await _require_ready_upload(uow, request.upload_id) + source_uri = None + else: + source_uri = request.source_uri + + source = IngestionJobSource( + id=uuid.uuid4(), + ingestion_job_id=request.ingestion_job_id, + attachment_kind=request.attachment_kind, + upload_id=request.upload_id, + source_uri=source_uri, + source_type=request.source_type, + weight=request.weight, + metadata=request.metadata, + created_at=dt.datetime.now(dt.UTC), + ) + await uow.ingestion_job_sources.add(source) + await uow.ingestion_jobs.transition_intake_state( + request.ingestion_job_id, + from_state=IntakeState.AWAITING_SOURCES, + to_state=IntakeState.READY_FOR_GENERATION, + ) + await uow.commit() + return source + + +async def get_ingestion_job_status( + uow: CanonicalUnitOfWork, + job_id: uuid.UUID, +) -> IngestionJob: + """Fetch one ingestion job or raise a source-intake not-found error.""" + job = await uow.ingestion_jobs.get(job_id) + if job is None: + raise IngestionJobNotFoundError(str(job_id)) + return job + + +async def list_ingestion_jobs( + uow: CanonicalUnitOfWork, + filters: IngestionJobListFilters, + pagination: Pagination, +) -> IngestionJobPage: + """List ingestion jobs with total count for REST pagination.""" + items = await uow.ingestion_jobs.list_paged( + filters, + limit=pagination.limit, + offset=pagination.offset, + ) + total = await uow.ingestion_jobs.count(filters) + return IngestionJobPage(items=items, total=total, pagination=pagination) + + +async def _require_ready_upload( + uow: CanonicalUnitOfWork, + upload_id: uuid.UUID | None, +) -> Upload: + """Return a ready upload or raise the correct source-intake error.""" + if upload_id is None: + raise UploadNotFoundError(_UPLOAD_ID_MISSING) + upload = await uow.uploads.get(upload_id) + if upload is None: + raise UploadNotFoundError(str(upload_id)) + if upload.state is not UploadState.READY: + raise UploadNotReadyError(str(upload_id)) + return upload + + +def _validate_declared_upload(request: UploadBytesRequest) -> None: + """Check client-declared size and hash against the supplied payload.""" + actual_size = len(request.payload) + if actual_size != request.declared_size: + raise UploadSizeMismatchError(str(request.declared_size)) + actual_hash = hashlib.sha256(request.payload).hexdigest() + if request.declared_sha256 is not None and request.declared_sha256 != actual_hash: + raise UploadHashMismatchError(request.declared_sha256) + + +async def _single_chunk_stream(payload: bytes) -> cabc.AsyncIterator[bytes]: + """Yield a bytes payload through the object-store streaming port.""" + await asyncio.sleep(0) + yield payload + + +_UPLOAD_ID_MISSING = "missing upload_id" diff --git a/episodic/canonical/storage/__init__.py b/episodic/canonical/storage/__init__.py index 17d3fb0a..b10db236 100644 --- a/episodic/canonical/storage/__init__.py +++ b/episodic/canonical/storage/__init__.py @@ -12,6 +12,8 @@ ... episode = await uow.episodes.get(episode_id) """ +from .filesystem_object_store import FilesystemObjectStore +from .ingestion_job_repositories import SqlAlchemyIngestionJobRepository from .migration_check import detect_schema_drift from .models import ( ApprovalEventRecord, @@ -19,7 +21,9 @@ EpisodeRecord, EpisodeTemplateHistoryRecord, EpisodeTemplateRecord, + IdempotencyRecordModel, IngestionJobRecord, + IngestionJobSourceRecord, ReferenceBindingRecord, ReferenceDocumentRecord, ReferenceDocumentRevisionRecord, @@ -27,6 +31,7 @@ SeriesProfileRecord, SourceDocumentRecord, TeiHeaderRecord, + UploadRecord, WorkflowCheckpointRecord, ) from .repositories import ( @@ -34,7 +39,6 @@ SqlAlchemyEpisodeRepository, SqlAlchemyEpisodeTemplateHistoryRepository, SqlAlchemyEpisodeTemplateRepository, - SqlAlchemyIngestionJobRepository, SqlAlchemyReferenceBindingRepository, SqlAlchemyReferenceDocumentRepository, SqlAlchemyReferenceDocumentRevisionRepository, @@ -43,6 +47,11 @@ SqlAlchemySourceDocumentRepository, SqlAlchemyTeiHeaderRepository, ) +from .source_intake_repositories import ( + SqlAlchemyIdempotencyStore, + SqlAlchemyIngestionJobSourceRepository, + SqlAlchemyUploadRepository, +) from .uow import SqlAlchemyUnitOfWork from .workflow_checkpoints import SqlAlchemyWorkflowCheckpointStore @@ -52,7 +61,10 @@ "EpisodeRecord", "EpisodeTemplateHistoryRecord", "EpisodeTemplateRecord", + "FilesystemObjectStore", + "IdempotencyRecordModel", "IngestionJobRecord", + "IngestionJobSourceRecord", "ReferenceBindingRecord", "ReferenceDocumentRecord", "ReferenceDocumentRevisionRecord", @@ -63,7 +75,9 @@ "SqlAlchemyEpisodeRepository", "SqlAlchemyEpisodeTemplateHistoryRepository", "SqlAlchemyEpisodeTemplateRepository", + "SqlAlchemyIdempotencyStore", "SqlAlchemyIngestionJobRepository", + "SqlAlchemyIngestionJobSourceRepository", "SqlAlchemyReferenceBindingRepository", "SqlAlchemyReferenceDocumentRepository", "SqlAlchemyReferenceDocumentRevisionRepository", @@ -72,8 +86,10 @@ "SqlAlchemySourceDocumentRepository", "SqlAlchemyTeiHeaderRepository", "SqlAlchemyUnitOfWork", + "SqlAlchemyUploadRepository", "SqlAlchemyWorkflowCheckpointStore", "TeiHeaderRecord", + "UploadRecord", "WorkflowCheckpointRecord", "detect_schema_drift", ) diff --git a/episodic/canonical/storage/entity_mappers.py b/episodic/canonical/storage/entity_mappers.py index 92d4c8b4..a56fcdeb 100644 --- a/episodic/canonical/storage/entity_mappers.py +++ b/episodic/canonical/storage/entity_mappers.py @@ -152,6 +152,7 @@ def _ingestion_job_from_record(record: IngestionJobRecord) -> IngestionJob: error_message=record.error_message, created_at=record.created_at, updated_at=record.updated_at, + intake_state=record.intake_state, ) @@ -166,6 +167,7 @@ def _ingestion_job_to_record(job: IngestionJob) -> IngestionJobRecord: started_at=job.started_at, completed_at=job.completed_at, error_message=job.error_message, + intake_state=job.intake_state, created_at=job.created_at, updated_at=job.updated_at, ) diff --git a/episodic/canonical/storage/entity_models.py b/episodic/canonical/storage/entity_models.py index 087fa6b7..e9ab39d8 100644 --- a/episodic/canonical/storage/entity_models.py +++ b/episodic/canonical/storage/entity_models.py @@ -7,13 +7,20 @@ from sqlalchemy import orm from sqlalchemy.dialects import postgresql -from episodic.canonical.domain import ( # noqa: TC001 # SQLAlchemy evaluates annotations at runtime. +from episodic.canonical.domain import ( # SQLAlchemy evaluates annotations at runtime. ApprovalState, EpisodeStatus, IngestionStatus, + IntakeState, ) -from .models_base import APPROVAL_STATE, EPISODE_STATUS, INGESTION_STATUS, Base +from .models_base import ( + APPROVAL_STATE, + EPISODE_STATUS, + INGESTION_STATUS, + INTAKE_STATE, + Base, +) class TeiHeaderRecord(Base): @@ -198,6 +205,11 @@ class IngestionJobRecord(Base): nullable=True, ) error_message: orm.Mapped[str | None] = orm.mapped_column(sa.Text, nullable=True) + intake_state: orm.Mapped[IntakeState] = orm.mapped_column( + INTAKE_STATE, + nullable=False, + server_default=IntakeState.AWAITING_SOURCES.value, + ) created_at: orm.Mapped[dt.datetime] = orm.mapped_column( sa.DateTime(timezone=True), nullable=False, diff --git a/episodic/canonical/storage/filesystem_object_store.py b/episodic/canonical/storage/filesystem_object_store.py new file mode 100644 index 00000000..a6dde6f7 --- /dev/null +++ b/episodic/canonical/storage/filesystem_object_store.py @@ -0,0 +1,99 @@ +"""Filesystem-backed object store for source-intake uploads.""" + +import asyncio +import contextlib +import hashlib +import typing as typ +import uuid + +from episodic.canonical.object_store import ( + ObjectStorePort, + PayloadTooLargeError, + StoredObject, + validate_object_key, +) + +_OBJECT_STORE_READ_CHUNK_BYTES = 64 * 1024 + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import pathlib + + +class FilesystemObjectStore(ObjectStorePort): + """Store object bytes under a configured filesystem root.""" + + def __init__(self, root: pathlib.Path) -> None: + self._root = root + self._tmp_root = root / "_tmp" + + async def put( + self, + key: str, + stream: cabc.AsyncIterator[bytes], + *, + max_bytes: int, + ) -> StoredObject: + """Store stream bytes atomically and return observed size/hash.""" + if max_bytes < 0: + msg = "max_bytes must be non-negative." + raise ValueError(msg) + safe_key = validate_object_key(key) + target = self._resolve_under_root(safe_key) + target.parent.mkdir(parents=True, exist_ok=True) + self._tmp_root.mkdir(parents=True, exist_ok=True) + tmp_path = self._tmp_root / f"{uuid.uuid4()}.tmp" + + digest = hashlib.sha256() + size = 0 + try: + with tmp_path.open("wb") as file_handle: + async for chunk in stream: + next_size = size + len(chunk) + if next_size > max_bytes: + _raise_payload_too_large() + file_handle.write(chunk) + digest.update(chunk) + size = next_size + tmp_path.replace(target) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + return StoredObject(key=safe_key, size=size, sha256=digest.hexdigest()) + + @contextlib.asynccontextmanager + async def open(self, key: str) -> cabc.AsyncIterator[cabc.AsyncIterator[bytes]]: + """Yield an async iterator over stored bytes.""" + path = self._resolve_under_root(validate_object_key(key)) + + async def _chunks() -> cabc.AsyncIterator[bytes]: + with path.open("rb") as file_handle: + while chunk := file_handle.read(_OBJECT_STORE_READ_CHUNK_BYTES): + await _yield_checkpoint() + yield chunk + + yield _chunks() + + async def delete(self, key: str) -> None: + """Delete stored bytes if they exist.""" + self._resolve_under_root(validate_object_key(key)).unlink(missing_ok=True) + + def _resolve_under_root(self, key: str) -> pathlib.Path: + """Return a root-confined path for a validated key.""" + root = self._root.resolve() + candidate = (root / key).resolve(strict=False) + if not candidate.is_relative_to(root): + msg = "object key escapes object store root." + raise ValueError(msg) + return candidate + + +async def _yield_checkpoint() -> None: + """Keep object reads cooperative without adding an I/O dependency.""" + await asyncio.sleep(0) + + +def _raise_payload_too_large() -> typ.NoReturn: + """Raise the canonical payload-size exception.""" + raise PayloadTooLargeError diff --git a/episodic/canonical/storage/ingestion_job_repositories.py b/episodic/canonical/storage/ingestion_job_repositories.py new file mode 100644 index 00000000..50715546 --- /dev/null +++ b/episodic/canonical/storage/ingestion_job_repositories.py @@ -0,0 +1,107 @@ +"""SQLAlchemy repository for intake-aware ingestion jobs.""" + +from __future__ import annotations + +import typing as typ + +import sqlalchemy as sa + +from episodic.canonical.entity_protocols import IngestionJobRepository + +from .entity_mappers import _ingestion_job_from_record, _ingestion_job_to_record +from .entity_models import IngestionJobRecord +from .repository_base import _RepositoryBase + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from episodic.canonical.domain import ( + IngestionJob, + IngestionJobListFilters, + IntakeState, + ) + + +class SqlAlchemyIngestionJobRepository(_RepositoryBase, IngestionJobRepository): + """Persist ingestion jobs and intake-state transitions with SQLAlchemy.""" + + async def add(self, job: IngestionJob) -> None: + """Add an ingestion job record.""" + await self._add_record(_ingestion_job_to_record(job)) + + async def get(self, job_id: uuid.UUID) -> IngestionJob | None: + """Fetch an ingestion job by identifier.""" + return await self._get_one_or_none( + IngestionJobRecord, + IngestionJobRecord.id == job_id, + _ingestion_job_from_record, + ) + + async def list_paged( + self, + filters: IngestionJobListFilters, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJob]: + """List ingestion jobs using source-intake filters.""" + statement = ( + sa + .select(IngestionJobRecord) + .where(_ingestion_job_filter_clause(filters)) + .order_by(IngestionJobRecord.created_at, IngestionJobRecord.id) + .limit(limit) + .offset(offset) + ) + result = await self._session.execute(statement) + return [_ingestion_job_from_record(row) for row in result.scalars()] + + async def count(self, filters: IngestionJobListFilters) -> int: + """Count ingestion jobs using source-intake filters.""" + result = await self._session.execute( + sa + .select(sa.func.count()) + .select_from(IngestionJobRecord) + .where(_ingestion_job_filter_clause(filters)) + ) + return result.scalar_one() + + async def transition_intake_state( + self, + job_id: uuid.UUID, + *, + from_state: IntakeState, + to_state: IntakeState, + ) -> bool: + """Return True only when the conditional intake-state update matched.""" + statement = ( + sa + .update(IngestionJobRecord) + .where( + sa.and_( + IngestionJobRecord.id == job_id, + IngestionJobRecord.intake_state == from_state, + ) + ) + .values(intake_state=to_state, updated_at=sa.func.now()) + ) + result = typ.cast( + "sa.CursorResult[typ.Any]", + await self._session.execute(statement), + ) + return result.rowcount == 1 + + +def _ingestion_job_filter_clause( + filters: IngestionJobListFilters, +) -> sa.ColumnElement[bool]: + """Build the SQLAlchemy predicate for ingestion-job list filters.""" + clauses: list[sa.ColumnElement[bool]] = [] + if filters.series_profile_id is not None: + clauses.append( + IngestionJobRecord.series_profile_id == filters.series_profile_id + ) + if filters.intake_state is not None: + clauses.append(IngestionJobRecord.intake_state == filters.intake_state) + return sa.and_(*clauses) if clauses else sa.true() diff --git a/episodic/canonical/storage/models.py b/episodic/canonical/storage/models.py index c7d8efe7..361d279f 100644 --- a/episodic/canonical/storage/models.py +++ b/episodic/canonical/storage/models.py @@ -15,11 +15,15 @@ from .history_models import EpisodeTemplateHistoryRecord, SeriesProfileHistoryRecord from .models_base import ( APPROVAL_STATE, + ATTACHMENT_KIND, EPISODE_STATUS, + IDEMPOTENCY_STATE, INGESTION_STATUS, + INTAKE_STATE, REFERENCE_BINDING_TARGET_KIND, REFERENCE_DOCUMENT_KIND, REFERENCE_DOCUMENT_LIFECYCLE_STATE, + UPLOAD_STATE, WORKFLOW_CHECKPOINT_STATUS, Base, ) @@ -29,22 +33,33 @@ ReferenceDocumentRecord, ReferenceDocumentRevisionRecord, ) +from .source_intake_models import ( + IdempotencyRecordModel, + IngestionJobSourceRecord, + UploadRecord, +) from .workflow_checkpoint_models import WorkflowCheckpointRecord __all__ = ( "APPROVAL_STATE", + "ATTACHMENT_KIND", "EPISODE_STATUS", + "IDEMPOTENCY_STATE", "INGESTION_STATUS", + "INTAKE_STATE", "REFERENCE_BINDING_TARGET_KIND", "REFERENCE_DOCUMENT_KIND", "REFERENCE_DOCUMENT_LIFECYCLE_STATE", + "UPLOAD_STATE", "WORKFLOW_CHECKPOINT_STATUS", "ApprovalEventRecord", "Base", "EpisodeRecord", "EpisodeTemplateHistoryRecord", "EpisodeTemplateRecord", + "IdempotencyRecordModel", "IngestionJobRecord", + "IngestionJobSourceRecord", "ReferenceBindingRecord", "ReferenceDocumentRecord", "ReferenceDocumentRevisionRecord", @@ -52,5 +67,6 @@ "SeriesProfileRecord", "SourceDocumentRecord", "TeiHeaderRecord", + "UploadRecord", "WorkflowCheckpointRecord", ) diff --git a/episodic/canonical/storage/models_base.py b/episodic/canonical/storage/models_base.py index 5cbeeed9..d3974b55 100644 --- a/episodic/canonical/storage/models_base.py +++ b/episodic/canonical/storage/models_base.py @@ -7,11 +7,15 @@ ApprovalState, EpisodeStatus, IngestionStatus, + IntakeState, ReferenceBindingTargetKind, ReferenceDocumentKind, ReferenceDocumentLifecycleState, WorkflowCheckpointStatus, ) +from episodic.canonical.idempotency import IdempotencyState +from episodic.canonical.ingestion_sources import AttachmentKind +from episodic.canonical.uploads import UploadState class Base(orm.DeclarativeBase): @@ -39,6 +43,26 @@ class Base(orm.DeclarativeBase): name="ingestion_status", values_callable=lambda enum_cls: [item.value for item in enum_cls], ) +INTAKE_STATE = sa.Enum( + IntakeState, + name="intake_state", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) +UPLOAD_STATE = sa.Enum( + UploadState, + name="upload_state", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) +ATTACHMENT_KIND = sa.Enum( + AttachmentKind, + name="attachment_kind", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) +IDEMPOTENCY_STATE = sa.Enum( + IdempotencyState, + name="idempotency_state", + values_callable=lambda enum_cls: [item.value for item in enum_cls], +) REFERENCE_DOCUMENT_KIND = sa.Enum( ReferenceDocumentKind, name="reference_document_kind", diff --git a/episodic/canonical/storage/source_intake_mappers.py b/episodic/canonical/storage/source_intake_mappers.py new file mode 100644 index 00000000..abba534a --- /dev/null +++ b/episodic/canonical/storage/source_intake_mappers.py @@ -0,0 +1,123 @@ +"""Record mappers for source-intake persistence.""" + +import copy + +from episodic.canonical.idempotency import IdempotencyRecord +from episodic.canonical.ingestion_sources import IngestionJobSource +from episodic.canonical.uploads import Upload + +from .source_intake_models import ( + IdempotencyRecordModel, + IngestionJobSourceRecord, + UploadRecord, +) + +_ANONYMOUS_PRINCIPAL = "" + + +def _principal_to_record(value: str | None) -> str: + """Normalise optional principals for database uniqueness.""" + return _ANONYMOUS_PRINCIPAL if value is None else value + + +def _principal_from_record(value: str) -> str | None: + """Map the database anonymous-principal sentinel back to the domain.""" + return None if value == _ANONYMOUS_PRINCIPAL else value + + +def _metadata_payload_to_domain[T](record_metadata_payload: T) -> T: + """Deep-copy a record metadata_payload field into the domain metadata field.""" + return copy.deepcopy(record_metadata_payload) + + +def _metadata_domain_to_payload[T](domain_metadata: T) -> T: + """Deep-copy a domain metadata field into the record metadata_payload field.""" + return copy.deepcopy(domain_metadata) + + +def _upload_from_record(record: UploadRecord) -> Upload: + """Map an upload record to a domain entity.""" + return Upload( + id=record.id, + owner_principal_id=record.owner_principal_id, + content_type=record.content_type, + declared_size=record.declared_size, + actual_size=record.actual_size, + declared_sha256=record.declared_sha256, + content_hash=record.content_hash, + storage_key=record.storage_key, + state=record.state, + metadata=_metadata_payload_to_domain(record.metadata_payload), + created_at=record.created_at, + updated_at=record.updated_at, + ) + + +def _upload_to_record(upload: Upload) -> UploadRecord: + """Map an upload domain entity to a record.""" + return UploadRecord( + id=upload.id, + owner_principal_id=upload.owner_principal_id, + content_type=upload.content_type, + declared_size=upload.declared_size, + actual_size=upload.actual_size, + declared_sha256=upload.declared_sha256, + content_hash=upload.content_hash, + storage_key=upload.storage_key, + state=upload.state, + metadata_payload=_metadata_domain_to_payload(upload.metadata), + created_at=upload.created_at, + updated_at=upload.updated_at, + ) + + +def _ingestion_job_source_from_record( + record: IngestionJobSourceRecord, +) -> IngestionJobSource: + """Map a source-attachment record to a domain entity.""" + return IngestionJobSource( + id=record.id, + ingestion_job_id=record.ingestion_job_id, + attachment_kind=record.attachment_kind, + upload_id=record.upload_id, + source_uri=record.source_uri, + source_type=record.source_type, + weight=record.weight, + metadata=_metadata_payload_to_domain(record.metadata_payload), + created_at=record.created_at, + ) + + +def _ingestion_job_source_to_record( + source: IngestionJobSource, +) -> IngestionJobSourceRecord: + """Map a source-attachment domain entity to a record.""" + return IngestionJobSourceRecord( + id=source.id, + ingestion_job_id=source.ingestion_job_id, + attachment_kind=source.attachment_kind, + upload_id=source.upload_id, + source_uri=source.source_uri, + source_type=source.source_type, + weight=source.weight, + metadata_payload=_metadata_domain_to_payload(source.metadata), + created_at=source.created_at, + ) + + +def _idempotency_record_from_record( + record: IdempotencyRecordModel, +) -> IdempotencyRecord: + """Map an idempotency record to a domain entity.""" + return IdempotencyRecord( + id=record.id, + principal_id=_principal_from_record(record.principal_id), + operation=record.operation, + idempotency_key=record.idempotency_key, + body_hash=record.body_hash, + state=record.state, + serialised_outcome=record.serialised_outcome, + expires_at=record.expires_at, + created_at=record.created_at, + updated_at=record.updated_at, + ) diff --git a/episodic/canonical/storage/source_intake_models.py b/episodic/canonical/storage/source_intake_models.py new file mode 100644 index 00000000..e633ff3d --- /dev/null +++ b/episodic/canonical/storage/source_intake_models.py @@ -0,0 +1,196 @@ +"""SQLAlchemy models for source-intake upload persistence.""" + +import datetime as dt # noqa: TC003 # SQLAlchemy evaluates annotations at runtime. +import uuid # noqa: TC003 # SQLAlchemy evaluates annotations at runtime. + +import sqlalchemy as sa +from sqlalchemy import orm +from sqlalchemy.dialects import postgresql + +from episodic.canonical.idempotency import ( # noqa: TC001 + IdempotencyState, +) +from episodic.canonical.ingestion_sources import ( # noqa: TC001 + AttachmentKind, +) +from episodic.canonical.uploads import UploadState # noqa: TC001 + +from .models_base import ( + ATTACHMENT_KIND, + IDEMPOTENCY_STATE, + UPLOAD_STATE, + Base, +) + + +class UploadRecord(Base): + """Persist metadata for bytes stored behind the object-store port.""" + + __tablename__ = "uploads" + + id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + primary_key=True, + ) + owner_principal_id: orm.Mapped[str | None] = orm.mapped_column( + sa.String(200), + nullable=True, + ) + content_type: orm.Mapped[str] = orm.mapped_column(sa.String(255), nullable=False) + declared_size: orm.Mapped[int] = orm.mapped_column(sa.BigInteger, nullable=False) + actual_size: orm.Mapped[int | None] = orm.mapped_column( + sa.BigInteger, + nullable=True, + ) + declared_sha256: orm.Mapped[str | None] = orm.mapped_column( + sa.String(64), + nullable=True, + ) + content_hash: orm.Mapped[str | None] = orm.mapped_column( + sa.String(80), + nullable=True, + ) + storage_key: orm.Mapped[str] = orm.mapped_column( + sa.Text, + nullable=False, + unique=True, + ) + state: orm.Mapped[UploadState] = orm.mapped_column( + UPLOAD_STATE, + nullable=False, + ) + metadata_payload: orm.Mapped[dict[str, object]] = orm.mapped_column( + "metadata", + postgresql.JSONB, + default=dict, + nullable=False, + ) + created_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ) + updated_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ) + + __table_args__ = ( + sa.CheckConstraint("declared_size >= 0", name="ck_uploads_declared_size"), + sa.CheckConstraint( + "actual_size IS NULL OR actual_size >= 0", name="ck_uploads_actual_size" + ), + ) + + +class IngestionJobSourceRecord(Base): + """Persist a pre-generation source attached to an ingestion job.""" + + __tablename__ = "ingestion_job_sources" + + id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + primary_key=True, + ) + ingestion_job_id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("ingestion_jobs.id"), + nullable=False, + index=True, + ) + attachment_kind: orm.Mapped[AttachmentKind] = orm.mapped_column( + ATTACHMENT_KIND, + nullable=False, + ) + upload_id: orm.Mapped[uuid.UUID | None] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + sa.ForeignKey("uploads.id"), + nullable=True, + index=True, + ) + source_uri: orm.Mapped[str | None] = orm.mapped_column(sa.Text, nullable=True) + source_type: orm.Mapped[str] = orm.mapped_column(sa.String(120), nullable=False) + weight: orm.Mapped[float] = orm.mapped_column(sa.Float, nullable=False) + metadata_payload: orm.Mapped[dict[str, object]] = orm.mapped_column( + "metadata", + postgresql.JSONB, + default=dict, + nullable=False, + ) + created_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ) + + __table_args__ = ( + sa.CheckConstraint( + "weight >= 0 AND weight <= 1", + name="ck_ingestion_job_sources_weight", + ), + sa.CheckConstraint( + "(upload_id IS NOT NULL AND source_uri IS NULL) OR " + "(upload_id IS NULL AND source_uri IS NOT NULL)", + name="ck_ingestion_job_sources_exactly_one_source", + ), + ) + + +class IdempotencyRecordModel(Base): + """Persist idempotent side-effect fingerprints and opaque outcomes.""" + + __tablename__ = "idempotency_records" + + id: orm.Mapped[uuid.UUID] = orm.mapped_column( + postgresql.UUID(as_uuid=True), + primary_key=True, + ) + principal_id: orm.Mapped[str] = orm.mapped_column( + sa.String(200), + nullable=False, + ) + operation: orm.Mapped[str] = orm.mapped_column(sa.String(120), nullable=False) + idempotency_key: orm.Mapped[str] = orm.mapped_column( + sa.String(512), + nullable=False, + ) + body_hash: orm.Mapped[str] = orm.mapped_column(sa.String(128), nullable=False) + state: orm.Mapped[IdempotencyState] = orm.mapped_column( + IDEMPOTENCY_STATE, + nullable=False, + ) + serialised_outcome: orm.Mapped[bytes | None] = orm.mapped_column( + postgresql.BYTEA, + nullable=True, + ) + expires_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + index=True, + ) + created_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ) + updated_at: orm.Mapped[dt.datetime] = orm.mapped_column( + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ) + + __table_args__ = ( + sa.UniqueConstraint( + "principal_id", + "operation", + "idempotency_key", + name="uq_idempotency_records_principal_operation_key", + ), + sa.CheckConstraint( + "state != 'completed' OR serialised_outcome IS NOT NULL", + name="ck_idempotency_records_completed_outcome", + ), + ) diff --git a/episodic/canonical/storage/source_intake_repositories.py b/episodic/canonical/storage/source_intake_repositories.py new file mode 100644 index 00000000..62cc8aef --- /dev/null +++ b/episodic/canonical/storage/source_intake_repositories.py @@ -0,0 +1,247 @@ +"""SQLAlchemy repositories for source-intake entities.""" + +import datetime as dt +import typing as typ +import uuid + +import sqlalchemy as sa + +from episodic.canonical.idempotency import ( + Acquired, + Conflict, + IdempotencyAcquireRequest, + IdempotencyOutcome, + IdempotencyState, + InFlight, + Replay, +) +from episodic.canonical.upload_protocols import ( + IdempotencyStore, + IngestionJobSourceRepository, + UploadRepository, +) +from episodic.canonical.uploads import UploadState + +from .repository_base import _RepositoryBase +from .source_intake_mappers import ( + _idempotency_record_from_record, + _ingestion_job_source_from_record, + _ingestion_job_source_to_record, + _principal_to_record, + _upload_from_record, + _upload_to_record, +) +from .source_intake_models import ( + IdempotencyRecordModel, + IngestionJobSourceRecord, + UploadRecord, +) + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from episodic.canonical.idempotency import IdempotencyRecord + from episodic.canonical.ingestion_sources import IngestionJobSource + from episodic.canonical.uploads import Upload + + +class SqlAlchemyUploadRepository(_RepositoryBase, UploadRepository): + """Persist upload metadata using SQLAlchemy.""" + + async def add(self, upload: Upload) -> None: + """Persist an upload metadata record.""" + await self._add_record(_upload_to_record(upload)) + + async def get(self, upload_id: uuid.UUID) -> Upload | None: + """Fetch an upload by identifier.""" + return await self._get_one_or_none( + UploadRecord, + UploadRecord.id == upload_id, + _upload_from_record, + ) + + async def mark_ready( + self, + upload_id: uuid.UUID, + *, + content_hash: str, + actual_size: int, + ) -> Upload: + """Mark an upload as ready after object-store persistence.""" + await self._session.execute( + sa + .update(UploadRecord) + .where(UploadRecord.id == upload_id) + .values( + content_hash=content_hash, + actual_size=actual_size, + state=UploadState.READY, + updated_at=sa.func.now(), + ) + ) + await self._session.flush() + upload = await self.get(upload_id) + if upload is None: + msg = f"Upload not found after ready transition: {upload_id}" + raise LookupError(msg) + return upload + + async def mark_failed(self, upload_id: uuid.UUID, reason: str) -> Upload: + """Mark an upload as failed.""" + del reason + await self._session.execute( + sa + .update(UploadRecord) + .where(UploadRecord.id == upload_id) + .values(state=UploadState.FAILED, updated_at=sa.func.now()) + ) + await self._session.flush() + upload = await self.get(upload_id) + if upload is None: + msg = f"Upload not found after failed transition: {upload_id}" + raise LookupError(msg) + return upload + + +class SqlAlchemyIngestionJobSourceRepository( + _RepositoryBase, + IngestionJobSourceRepository, +): + """Persist pre-generation source attachments using SQLAlchemy.""" + + async def add(self, source: IngestionJobSource) -> None: + """Persist a source attachment.""" + await self._add_record(_ingestion_job_source_to_record(source)) + + async def get(self, source_id: uuid.UUID) -> IngestionJobSource | None: + """Fetch a source attachment by identifier.""" + return await self._get_one_or_none( + IngestionJobSourceRecord, + IngestionJobSourceRecord.id == source_id, + _ingestion_job_source_from_record, + ) + + async def list_for_job_paged( + self, + job_id: uuid.UUID, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJobSource]: + """List source attachments for one ingestion job.""" + statement = ( + sa + .select(IngestionJobSourceRecord) + .where(IngestionJobSourceRecord.ingestion_job_id == job_id) + .order_by(IngestionJobSourceRecord.created_at, IngestionJobSourceRecord.id) + .limit(limit) + .offset(offset) + ) + result = await self._session.execute(statement) + return [_ingestion_job_source_from_record(row) for row in result.scalars()] + + async def count_for_job(self, job_id: uuid.UUID) -> int: + """Count source attachments for one ingestion job.""" + result = await self._session.execute( + sa + .select(sa.func.count()) + .select_from(IngestionJobSourceRecord) + .where(IngestionJobSourceRecord.ingestion_job_id == job_id) + ) + return result.scalar_one() + + +class SqlAlchemyIdempotencyStore(_RepositoryBase, IdempotencyStore): + """Persist idempotency records with domain-only outcomes.""" + + async def acquire( + self, + *, + request: IdempotencyAcquireRequest, + ) -> IdempotencyOutcome: + """Acquire or inspect an idempotency record.""" + record = await self._get_record( + principal_id=request.principal_id, + operation=request.operation, + idempotency_key=request.idempotency_key, + ) + if record is None: + record_id = uuid.uuid4() + now = dt.datetime.now(dt.UTC) + self._session.add( + IdempotencyRecordModel( + id=record_id, + principal_id=_principal_to_record(request.principal_id), + operation=request.operation, + idempotency_key=request.idempotency_key, + body_hash=request.body_hash, + state=IdempotencyState.IN_FLIGHT, + serialised_outcome=None, + expires_at=request.expires_at, + created_at=now, + updated_at=now, + ) + ) + await self._session.flush() + return Acquired(record_id) + if record.body_hash != request.body_hash: + return Conflict(record.id) + if record.state is IdempotencyState.COMPLETED: + if record.serialised_outcome is None: + msg = f"Completed idempotency record lacks outcome: {record.id}" + raise RuntimeError(msg) + return Replay(record.serialised_outcome) + return InFlight(record.id) + + async def complete( + self, + *, + record_id: uuid.UUID, + serialised_outcome: bytes, + ) -> None: + """Store an opaque completed outcome for replay.""" + await self._session.execute( + sa + .update(IdempotencyRecordModel) + .where(IdempotencyRecordModel.id == record_id) + .values( + state=IdempotencyState.COMPLETED, + serialised_outcome=serialised_outcome, + updated_at=sa.func.now(), + ) + ) + + async def lookup( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + ) -> IdempotencyRecord | None: + """Fetch an idempotency record by its logical key.""" + record = await self._get_record( + principal_id=principal_id, + operation=operation, + idempotency_key=idempotency_key, + ) + return None if record is None else _idempotency_record_from_record(record) + + async def _get_record( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + ) -> IdempotencyRecordModel | None: + """Fetch one raw idempotency row by logical key.""" + result = await self._session.execute( + sa.select(IdempotencyRecordModel).where( + sa.and_( + IdempotencyRecordModel.principal_id + == _principal_to_record(principal_id), + IdempotencyRecordModel.operation == operation, + IdempotencyRecordModel.idempotency_key == idempotency_key, + ) + ) + ) + return result.scalar_one_or_none() diff --git a/episodic/canonical/storage/uow.py b/episodic/canonical/storage/uow.py index a7d158e2..40c1d335 100644 --- a/episodic/canonical/storage/uow.py +++ b/episodic/canonical/storage/uow.py @@ -17,12 +17,12 @@ from episodic.canonical.unit_of_work_protocols import CanonicalUnitOfWork from episodic.logging import get_logger +from .ingestion_job_repositories import SqlAlchemyIngestionJobRepository from .repositories import ( SqlAlchemyApprovalEventRepository, SqlAlchemyEpisodeRepository, SqlAlchemyEpisodeTemplateHistoryRepository, SqlAlchemyEpisodeTemplateRepository, - SqlAlchemyIngestionJobRepository, SqlAlchemyReferenceBindingRepository, SqlAlchemyReferenceDocumentRepository, SqlAlchemyReferenceDocumentRevisionRepository, @@ -31,6 +31,11 @@ SqlAlchemySourceDocumentRepository, SqlAlchemyTeiHeaderRepository, ) +from .source_intake_repositories import ( + SqlAlchemyIdempotencyStore, + SqlAlchemyIngestionJobSourceRepository, + SqlAlchemyUploadRepository, +) from .workflow_checkpoints import SqlAlchemyWorkflowCheckpointStore if typ.TYPE_CHECKING: @@ -125,6 +130,11 @@ async def __aenter__(self) -> SqlAlchemyUnitOfWork: SqlAlchemyReferenceDocumentRevisionRepository(self._session) ) self.reference_bindings = SqlAlchemyReferenceBindingRepository(self._session) + self.uploads = SqlAlchemyUploadRepository(self._session) + self.ingestion_job_sources = SqlAlchemyIngestionJobSourceRepository( + self._session + ) + self.idempotency = SqlAlchemyIdempotencyStore(self._session) self.workflow_checkpoints = SqlAlchemyWorkflowCheckpointStore( self._session, metrics=self._metrics, diff --git a/episodic/canonical/unit_of_work_protocols.py b/episodic/canonical/unit_of_work_protocols.py index 30e75e56..4be3d673 100644 --- a/episodic/canonical/unit_of_work_protocols.py +++ b/episodic/canonical/unit_of_work_protocols.py @@ -23,6 +23,11 @@ ReferenceDocumentRepository, ReferenceDocumentRevisionRepository, ) + from .upload_protocols import ( + IdempotencyStore, + IngestionJobSourceRepository, + UploadRepository, + ) @typ.runtime_checkable @@ -41,6 +46,9 @@ class CanonicalUnitOfWork(typ.Protocol): reference_documents: ReferenceDocumentRepository reference_document_revisions: ReferenceDocumentRevisionRepository reference_bindings: ReferenceBindingRepository + uploads: UploadRepository + ingestion_job_sources: IngestionJobSourceRepository + idempotency: IdempotencyStore async def __aenter__(self) -> CanonicalUnitOfWork: """Enter the unit-of-work context.""" diff --git a/episodic/canonical/upload_protocols.py b/episodic/canonical/upload_protocols.py new file mode 100644 index 00000000..9b80c992 --- /dev/null +++ b/episodic/canonical/upload_protocols.py @@ -0,0 +1,113 @@ +"""Repository protocols for upload and intake idempotency entities.""" + +import typing as typ + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import uuid + + from .domain import IntakeState + from .idempotency import ( + IdempotencyAcquireRequest, + IdempotencyOutcome, + IdempotencyRecord, + ) + from .ingestion_sources import IngestionJobSource + from .uploads import Upload + + +class UploadRepository(typ.Protocol): + """Persistence interface for upload metadata.""" + + async def add(self, upload: Upload) -> None: + """Persist an upload metadata record.""" + raise NotImplementedError + + async def get(self, upload_id: uuid.UUID) -> Upload | None: + """Fetch an upload by identifier.""" + raise NotImplementedError + + async def mark_ready( + self, + upload_id: uuid.UUID, + *, + content_hash: str, + actual_size: int, + ) -> Upload: + """Mark an upload as ready after bytes are stored.""" + raise NotImplementedError + + async def mark_failed(self, upload_id: uuid.UUID, reason: str) -> Upload: + """Mark an upload as failed.""" + raise NotImplementedError + + +class IngestionJobSourceRepository(typ.Protocol): + """Persistence interface for intake source attachments.""" + + async def add(self, source: IngestionJobSource) -> None: + """Persist a source attachment.""" + raise NotImplementedError + + async def get(self, source_id: uuid.UUID) -> IngestionJobSource | None: + """Fetch a source attachment by identifier.""" + raise NotImplementedError + + async def list_for_job_paged( + self, + job_id: uuid.UUID, + *, + limit: int, + offset: int, + ) -> cabc.Sequence[IngestionJobSource]: + """List source attachments for one ingestion job.""" + raise NotImplementedError + + async def count_for_job(self, job_id: uuid.UUID) -> int: + """Count source attachments for one ingestion job.""" + raise NotImplementedError + + +class IdempotencyStore(typ.Protocol): + """Persistence interface for retryable side-effect records.""" + + async def acquire( + self, + *, + request: IdempotencyAcquireRequest, + ) -> IdempotencyOutcome: + """Acquire or inspect an idempotency record.""" + raise NotImplementedError + + async def complete( + self, + *, + record_id: uuid.UUID, + serialised_outcome: bytes, + ) -> None: + """Store an opaque completed outcome for replay.""" + raise NotImplementedError + + async def lookup( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + ) -> IdempotencyRecord | None: + """Fetch an idempotency record by its logical key.""" + raise NotImplementedError + + +class IntakeStateTransitionRepository(typ.Protocol): + """Persistence interface for conditional intake-state transitions.""" + + async def transition_intake_state( + self, + job_id: uuid.UUID, + *, + from_state: IntakeState, + to_state: IntakeState, + ) -> bool: + """Return True only when the conditional state update matched.""" + raise NotImplementedError diff --git a/episodic/canonical/uploads.py b/episodic/canonical/uploads.py new file mode 100644 index 00000000..514efbe7 --- /dev/null +++ b/episodic/canonical/uploads.py @@ -0,0 +1,79 @@ +"""Upload domain entities for source-intake workflows.""" + +import dataclasses as dc +import enum +import typing as typ + +if typ.TYPE_CHECKING: + import datetime as dt + import uuid + + from .domain import JsonMapping + + +class UploadState(enum.StrEnum): + """Lifecycle states for uploaded source bytes.""" + + PENDING = "pending" + READY = "ready" + FAILED = "failed" + EXPIRED = "expired" + + +def _require_non_negative(value: int, field: str) -> None: + if value < 0: + msg = f"{field} must be non-negative." + raise ValueError(msg) + + +def _require_non_negative_if_present(value: int | None, field: str) -> None: + if value is not None and value < 0: + msg = f"{field} must be non-negative when provided." + raise ValueError(msg) + + +def _require_non_empty(value: str, field: str) -> None: + if not value.strip(): + msg = f"{field} must be a non-empty string." + raise ValueError(msg) + + +@dc.dataclass(frozen=True, slots=True) +class Upload: + """Metadata record for uploaded bytes.""" + + id: uuid.UUID + owner_principal_id: str | None + content_type: str + declared_size: int + actual_size: int | None + declared_sha256: str | None + content_hash: str | None + storage_key: str + state: UploadState + metadata: JsonMapping + created_at: dt.datetime + updated_at: dt.datetime + + def __post_init__(self) -> None: + """Validate upload invariants at the domain boundary.""" + _require_non_negative(self.declared_size, "declared_size") + _require_non_negative_if_present(self.actual_size, "actual_size") + _require_non_empty(self.content_type, "content_type") + _require_non_empty(self.storage_key, "storage_key") + + +@dc.dataclass(frozen=True, slots=True) +class UploadInitRequest: + """Validated request to reserve an upload metadata row.""" + + owner_principal_id: str | None + content_type: str + declared_size: int + declared_sha256: str | None + metadata: JsonMapping + + def __post_init__(self) -> None: + """Validate the client-declared upload metadata.""" + _require_non_negative(self.declared_size, "declared_size") + _require_non_empty(self.content_type, "content_type") diff --git a/pyproject.toml b/pyproject.toml index e427ba5b..5f5ca0df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -420,8 +420,9 @@ enable = [ [tool.pytest.ini_options] # Tests automatically killed after seconds elapsed. Full-suite database fixture -# startup can exceed 30 seconds under shared CI or agent-host load. -timeout = 60 +# startup can exceed 60 seconds under shared CI or agent-host load when +# py-pglite starts a fresh PostgreSQL runtime and applies Alembic migrations. +timeout = 180 markers = [ "act: integration tests that drive GitHub Actions via act", "crosshair: symbolic contract verification using CrossHair", @@ -459,10 +460,15 @@ prefixes = [ "episodic.canonical.ingestion_ports", "episodic.canonical.entity_protocols", "episodic.canonical.history_protocols", + "episodic.canonical.idempotency", + "episodic.canonical.ingestion_sources", + "episodic.canonical.object_store", "episodic.canonical.pagination", "episodic.canonical.ports", "episodic.canonical.reference_protocols", "episodic.canonical.unit_of_work_protocols", + "episodic.canonical.upload_protocols", + "episodic.canonical.uploads", "episodic.llm.ports", "episodic.metrics_ports", ] @@ -473,6 +479,8 @@ name = "application" prefixes = [ "episodic.canonical.services", "episodic.canonical.ingestion_service", + "episodic.canonical.idempotency_service", + "episodic.canonical.source_intake_service", "episodic.canonical.profile_templates", "episodic.canonical.reference_documents", "episodic.generation", diff --git a/tests/__snapshots__/test_source_intake_api.ambr b/tests/__snapshots__/test_source_intake_api.ambr new file mode 100644 index 00000000..8d51bd37 --- /dev/null +++ b/tests/__snapshots__/test_source_intake_api.ambr @@ -0,0 +1,32 @@ +# serializer version: 1 +# name: test_source_intake_response_envelope_snapshot + dict({ + 'job': dict({ + 'intake_state': 'awaiting_sources', + 'next_poll_after_seconds': None, + 'status': 'pending', + }), + 'source': dict({ + 'metadata': dict({ + 'language': 'en', + }), + 'source_type': 'research_paper', + 'source_uri': None, + 'type': 'upload', + 'weight': 0.75, + }), + 'status': dict({ + 'intake_state': 'ready_for_generation', + 'next_poll_after_seconds': None, + 'status': 'pending', + }), + 'upload': dict({ + 'content_hash_algorithm': 'sha256', + 'content_type': 'text/plain', + 'metadata': dict({ + }), + 'size_bytes': 9, + 'state': 'ready', + }), + }) +# --- diff --git a/tests/canonical_storage/test_source_intake_repositories.py b/tests/canonical_storage/test_source_intake_repositories.py new file mode 100644 index 00000000..0c00536f --- /dev/null +++ b/tests/canonical_storage/test_source_intake_repositories.py @@ -0,0 +1,233 @@ +"""Unit tests for source-intake SQLAlchemy repositories.""" + +from __future__ import annotations + +import dataclasses as dc +import datetime as dt +import typing as typ +import uuid + +import pytest + +from episodic.canonical.domain import IngestionJobListFilters, IntakeState +from episodic.canonical.idempotency import ( + Conflict, + IdempotencyAcquireRequest, + InFlight, + Replay, +) +from episodic.canonical.ingestion_sources import AttachmentKind, IngestionJobSource +from episodic.canonical.storage import SqlAlchemyUnitOfWork +from episodic.canonical.uploads import Upload, UploadState + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from episodic.canonical.domain import ( + CanonicalEpisode, + IngestionJob, + SeriesProfile, + SourceDocument, + TeiHeader, + ) + + +@dc.dataclass(frozen=True, slots=True) +class _RoundTripResult: + """Values produced while persisting source-intake fixtures.""" + + upload: Upload + ready_upload: Upload + transitioned: bool + + +@dc.dataclass(frozen=True, slots=True) +class _FetchedRoundTrip: + """Values fetched back from source-intake repositories.""" + + upload: Upload + sources: cabc.Sequence[IngestionJobSource] + job_ids: cabc.Sequence[uuid.UUID] + total: int + + +@pytest.mark.asyncio +async def test_source_intake_repositories_round_trip( + session_factory: object, + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> None: + """Upload and source-attachment repositories round-trip stored entities.""" + series, _, _, job, _ = episode_fixture + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + result = await _persist_round_trip_fixture(factory, episode_fixture) + fetched = await _fetch_round_trip_fixture( + factory, + series_id=series.id, + job_id=job.id, + upload_id=result.upload.id, + ) + + assert result.transitioned is True + assert result.ready_upload.state is UploadState.READY + assert fetched.upload.content_hash == "sha256:abc123" + assert fetched.sources[0].upload_id == result.upload.id + assert fetched.job_ids == [job.id] + assert fetched.total == 1 + + +async def _persist_round_trip_fixture( + factory: async_sessionmaker[AsyncSession], + episode_fixture: tuple[ + SeriesProfile, + TeiHeader, + CanonicalEpisode, + IngestionJob, + SourceDocument, + ], +) -> _RoundTripResult: + """Persist source-intake fixture rows and return transition results.""" + series, header, episode, job, _ = episode_fixture + upload = _make_upload() + + async with SqlAlchemyUnitOfWork(factory) as uow: + await uow.series_profiles.add(series) + await uow.tei_headers.add(header) + await uow.flush() + await uow.episodes.add(episode) + await uow.ingestion_jobs.add(job) + await uow.uploads.add(upload) + ready_upload = await uow.uploads.mark_ready( + upload.id, + content_hash="sha256:abc123", + actual_size=6, + ) + source = IngestionJobSource( + id=uuid.uuid4(), + ingestion_job_id=job.id, + attachment_kind=AttachmentKind.UPLOAD, + upload_id=upload.id, + source_uri=None, + source_type="research_paper", + weight=1.0, + metadata={"language": "en"}, + created_at=upload.created_at, + ) + await uow.ingestion_job_sources.add(source) + transitioned = await uow.ingestion_jobs.transition_intake_state( + job.id, + from_state=IntakeState.AWAITING_SOURCES, + to_state=IntakeState.READY_FOR_GENERATION, + ) + await uow.commit() + return _RoundTripResult( + upload=upload, + ready_upload=ready_upload, + transitioned=transitioned, + ) + + +async def _fetch_round_trip_fixture( + factory: async_sessionmaker[AsyncSession], + *, + series_id: uuid.UUID, + job_id: uuid.UUID, + upload_id: uuid.UUID, +) -> _FetchedRoundTrip: + """Fetch persisted source-intake rows for assertions.""" + async with SqlAlchemyUnitOfWork(factory) as uow: + fetched_upload = await uow.uploads.get(upload_id) + fetched_sources = await uow.ingestion_job_sources.list_for_job_paged( + job_id, + limit=10, + offset=0, + ) + page = await uow.ingestion_jobs.list_paged( + IngestionJobListFilters( + series_profile_id=series_id, + intake_state=IntakeState.READY_FOR_GENERATION, + ), + limit=10, + offset=0, + ) + total = await uow.ingestion_jobs.count( + IngestionJobListFilters( + series_profile_id=series_id, + intake_state=IntakeState.READY_FOR_GENERATION, + ) + ) + if fetched_upload is None: + msg = f"Expected upload to round-trip: {upload_id}" + raise AssertionError(msg) + return _FetchedRoundTrip( + upload=fetched_upload, + sources=fetched_sources, + job_ids=[item.id for item in page], + total=total, + ) + + +def _make_upload() -> Upload: + """Return one pending upload fixture.""" + now = dt.datetime.now(dt.UTC) + return Upload( + id=uuid.uuid4(), + owner_principal_id="api-user", + content_type="text/plain", + declared_size=6, + actual_size=None, + declared_sha256=None, + content_hash=None, + storage_key=f"uploads/{uuid.uuid4()}", + state=UploadState.PENDING, + metadata={"language": "en"}, + created_at=now, + updated_at=now, + ) + + +@pytest.mark.asyncio +async def test_sqlalchemy_idempotency_store_replays_and_conflicts( + session_factory: object, +) -> None: + """SQLAlchemy idempotency store returns domain-only outcomes.""" + factory = typ.cast("async_sessionmaker[AsyncSession]", session_factory) + request = IdempotencyAcquireRequest( + principal_id=None, + operation="upload.create", + idempotency_key="same-key", + body_hash="body-a", + expires_at=dt.datetime.now(dt.UTC) + dt.timedelta(hours=1), + ) + + async with SqlAlchemyUnitOfWork(factory) as uow: + acquired = await uow.idempotency.acquire(request=request) + assert not isinstance(acquired, Replay | Conflict | InFlight) + await uow.idempotency.complete( + record_id=acquired.record_id, + serialised_outcome=b'{"ok":true}', + ) + await uow.commit() + + async with SqlAlchemyUnitOfWork(factory) as uow: + replay = await uow.idempotency.acquire(request=request) + conflict = await uow.idempotency.acquire( + request=IdempotencyAcquireRequest( + principal_id=None, + operation="upload.create", + idempotency_key="same-key", + body_hash="body-b", + expires_at=dt.datetime.now(dt.UTC) + dt.timedelta(hours=1), + ) + ) + + assert isinstance(replay, Replay) + assert replay.serialised_outcome == b'{"ok":true}' + assert isinstance(conflict, Conflict) diff --git a/tests/features/source_intake.feature b/tests/features/source_intake.feature new file mode 100644 index 00000000..66af80ab --- /dev/null +++ b/tests/features/source_intake.feature @@ -0,0 +1,8 @@ +Feature: Source intake API + + Scenario: Editorial team uploads and attaches source material + Given source-intake API fixtures exist + When an editor uploads source material and attaches it to a new ingestion job + Then the ingestion job is ready for generation + And repeated upload requests replay the stored response + And changed upload bodies with the same idempotency key conflict diff --git a/tests/fixtures/api.py b/tests/fixtures/api.py index 55bdf024..0188e2e5 100644 --- a/tests/fixtures/api.py +++ b/tests/fixtures/api.py @@ -16,6 +16,7 @@ from episodic.api import ApiDependencies from episodic.api.authorization import AuthorizationPort + from episodic.canonical.object_store import ObjectStorePort @dc.dataclass(frozen=True, slots=True) @@ -51,6 +52,7 @@ def build_api_dependencies( session_factory: async_sessionmaker[AsyncSession], *, authorization: AuthorizationPort | None = None, + object_store: ObjectStorePort | None = None, ) -> ApiDependencies: """Build typed API dependencies with optional authorization override.""" from episodic.api import ApiDependencies @@ -58,11 +60,13 @@ def build_api_dependencies( if authorization is None: return ApiDependencies( - uow_factory=lambda: SqlAlchemyUnitOfWork(session_factory) + uow_factory=lambda: SqlAlchemyUnitOfWork(session_factory), + object_store=object_store, ) return ApiDependencies( uow_factory=lambda: SqlAlchemyUnitOfWork(session_factory), authorization=authorization, + object_store=object_store, ) diff --git a/tests/fixtures/database.py b/tests/fixtures/database.py index 41d5c555..a34036c2 100644 --- a/tests/fixtures/database.py +++ b/tests/fixtures/database.py @@ -1,5 +1,7 @@ """Database infrastructure fixtures (py-pglite, SQLAlchemy).""" +from __future__ import annotations + import asyncio import contextlib import os @@ -30,6 +32,22 @@ _PGLITE_AVAILABLE = False +# Serialise concurrent schema resets under pytest-xdist. Workers sharing one +# py-pglite process must go through this lock before dropping `public`. +_schema_reset_lock = asyncio.Lock() + + +@pytest.fixture(scope="session") +def pglite_node_environment( + tmp_path_factory: pytest.TempPathFactory, +) -> Path: + """Return the session work root for py-pglite test processes.""" + if not _should_use_pglite(): + pytest.skip("EPISODIC_TEST_DB=sqlite disables py-pglite-backed fixtures.") + + return tmp_path_factory.mktemp("pglite-node-env") + + def _should_use_pglite() -> bool: """Return True when tests should attempt py-pglite. @@ -77,9 +95,16 @@ async def _wait_for_engine_ready(engine: AsyncEngine) -> None: return +async def _reset_public_schema(engine: AsyncEngine) -> None: + """Reset the shared py-pglite database before applying migrations.""" + async with _schema_reset_lock, engine.begin() as connection: + await connection.execute(sa.text("DROP SCHEMA IF EXISTS public CASCADE")) + await connection.execute(sa.text("CREATE SCHEMA public")) + + @contextlib.asynccontextmanager async def _pglite_sqlalchemy_manager( - tmp_path: Path, + work_dir: Path, ) -> cabc.AsyncIterator[SQLAlchemyAsyncPGliteManager]: """Start a helper-backed py-pglite manager for SQLAlchemy tests.""" if not _PGLITE_AVAILABLE: # pragma: no cover - defensive guard @@ -89,19 +114,31 @@ async def _pglite_sqlalchemy_manager( from py_pglite.sqlalchemy.manager_async import SQLAlchemyAsyncPGliteManager from sqlalchemy.pool import NullPool - work_dir = tmp_path / "pglite" - config = PGliteConfig(work_dir=work_dir) - manager = SQLAlchemyAsyncPGliteManager(config) - manager.start() - try: - engine = typ.cast("AsyncEngine", manager.get_engine(poolclass=NullPool)) + last_error: Exception | None = None + for attempt in range(1, 4): + attempt_work_dir = work_dir.with_name(f"{work_dir.name}-attempt-{attempt}") + config = PGliteConfig( + work_dir=attempt_work_dir, + timeout=90, + ) + manager = SQLAlchemyAsyncPGliteManager(config) try: + manager.start() + engine = typ.cast("AsyncEngine", manager.get_engine(poolclass=NullPool)) await _wait_for_engine_ready(engine) + except (RuntimeError, sa_exc.OperationalError) as exc: + last_error = exc + await manager.stop() + continue + + try: + yield manager finally: - await engine.dispose() - yield manager - finally: - await manager.stop() + await manager.stop() + return + + msg = "py-pglite failed to start after 3 attempts." + raise RuntimeError(msg) from last_error @contextlib.contextmanager @@ -122,11 +159,11 @@ def temporary_drift_table() -> cabc.Iterator[sa.Table]: Base.metadata.remove(table) -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="session") async def pglite_sqlalchemy_manager( - tmp_path: Path, + pglite_node_environment: Path, ) -> cabc.AsyncIterator[SQLAlchemyAsyncPGliteManager]: - """Yield the function-scoped py-pglite SQLAlchemy manager. + """Yield the session-scoped py-pglite SQLAlchemy manager. This is the shared py-pglite entry point for SQLAlchemy-backed tests in this repository. Prefer `session_factory`, `pglite_session`, or @@ -135,7 +172,8 @@ async def pglite_sqlalchemy_manager( if not _should_use_pglite(): pytest.skip("EPISODIC_TEST_DB=sqlite disables py-pglite-backed fixtures.") - async with _pglite_sqlalchemy_manager(tmp_path) as manager: + work_dir = pglite_node_environment / "server" + async with _pglite_sqlalchemy_manager(work_dir) as manager: yield manager @@ -150,10 +188,7 @@ async def pglite_engine( "AsyncEngine", pglite_sqlalchemy_manager.get_engine(poolclass=NullPool) ) await asyncio.sleep(0) - try: - yield engine - finally: - await engine.dispose() + yield engine @pytest_asyncio.fixture @@ -161,6 +196,7 @@ async def migrated_engine( pglite_engine: AsyncEngine, ) -> cabc.AsyncIterator[AsyncEngine]: """Yield a py-pglite engine with migrations applied.""" + await _reset_public_schema(pglite_engine) await apply_migrations(pglite_engine) yield pglite_engine diff --git a/tests/fixtures/test_database.py b/tests/fixtures/test_database.py new file mode 100644 index 00000000..cd6b0925 --- /dev/null +++ b/tests/fixtures/test_database.py @@ -0,0 +1,70 @@ +"""Tests for py-pglite database fixtures.""" + +import asyncio +import typing as typ +from unittest import mock + +import pytest +import sqlalchemy as sa + +from tests.fixtures import database + +if typ.TYPE_CHECKING: + from types import TracebackType + + from sqlalchemy.ext.asyncio import AsyncEngine + + +class _CountingSchemaResetLock: + """Async context manager that records reset lock usage.""" + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self.enter_count = 0 + self.exit_count = 0 + self._active_count = 0 + self.max_active_count = 0 + + async def __aenter__(self) -> _CountingSchemaResetLock: + self.enter_count += 1 + await self._lock.acquire() + self._active_count += 1 + self.max_active_count = max(self.max_active_count, self._active_count) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.exit_count += 1 + self._active_count -= 1 + self._lock.release() + + +@pytest.mark.asyncio +async def test_reset_public_schema_serializes_concurrent_calls( + pglite_engine: AsyncEngine, +) -> None: + """Concurrent resets should leave one public schema and release the lock.""" + counting_lock = _CountingSchemaResetLock() + + with mock.patch.object(database, "_schema_reset_lock", counting_lock): + await asyncio.gather( + database._reset_public_schema(pglite_engine), + database._reset_public_schema(pglite_engine), + ) + + async with pglite_engine.connect() as connection: + result = await connection.execute( + sa.text( + "SELECT count(*) FROM information_schema.schemata " + "WHERE schema_name = 'public'" + ) + ) + + assert result.scalar_one() == 1 + assert counting_lock.enter_count == 2 + assert counting_lock.exit_count == 2 + assert counting_lock.max_active_count == 1 diff --git a/tests/steps/test_source_intake_steps.py b/tests/steps/test_source_intake_steps.py new file mode 100644 index 00000000..abb509ed --- /dev/null +++ b/tests/steps/test_source_intake_steps.py @@ -0,0 +1,196 @@ +"""Behavioural tests for source-intake upload and attachment workflows.""" + +from __future__ import annotations + +import asyncio +import dataclasses as dc +import hashlib +import typing as typ + +import httpx +from pytest_bdd import given, scenario, then, when + +from episodic.api import create_app +from episodic.canonical.storage import FilesystemObjectStore +from tests.fixtures.api import build_api_dependencies + +if typ.TYPE_CHECKING: + from pathlib import Path + + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +@dc.dataclass(slots=True) +class SourceIntakeContext: + """Shared state for source-intake BDD steps.""" + + upload: dict[str, object] | None = None + upload_replay: dict[str, object] | None = None + conflict: dict[str, object] | None = None + conflict_status: int | None = None + job: dict[str, object] | None = None + source: dict[str, object] | None = None + status: dict[str, object] | None = None + + +@scenario( + "../features/source_intake.feature", + "Editorial team uploads and attaches source material", +) +def test_source_intake_behaviour() -> None: + """Run source-intake API scenario.""" + + +@given("source-intake API fixtures exist", target_fixture="context") +def source_intake_fixtures() -> SourceIntakeContext: + """Create an empty context for the source-intake API scenario.""" + return SourceIntakeContext() + + +async def _post_text_upload( + client: httpx.AsyncClient, + *, + key: str, + payload: bytes, +) -> httpx.Response: + """Post a text upload with a deterministic multipart shape.""" + return await client.post( + "/v1/uploads", + headers={"Idempotency-Key": key}, + files={ + "file": ("source.txt", payload, "text/plain"), + "content_type": (None, "text/plain"), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + "metadata": (None, '{"language":"en"}', "application/json"), + }, + ) + + +async def _run_intake_api_calls( + client: httpx.AsyncClient, + context: SourceIntakeContext, +) -> None: + profile_id = await _create_series_profile(client) + upload = await _post_text_upload( + client, + key="bdd-upload-key", + payload=b"source\n", + ) + upload_replay = await _post_text_upload( + client, + key="bdd-upload-key", + payload=b"source\n", + ) + conflict_first = await _post_text_upload( + client, + key="bdd-conflict-key", + payload=b"alpha\n", + ) + conflict = await _post_text_upload( + client, + key="bdd-conflict-key", + payload=b"beta\n", + ) + job = await client.post( + "/v1/ingestion-jobs", + headers={"Idempotency-Key": "bdd-job-key"}, + json={"series_profile_id": profile_id, "target_episode_id": None}, + ) + source = await client.post( + f"/v1/ingestion-jobs/{job.json()['id']}/sources", + headers={"Idempotency-Key": "bdd-source-key"}, + json={ + "type": "upload", + "upload_id": upload.json()["id"], + "source_type": "research_paper", + "weight": 1.0, + "metadata": {"language": "en"}, + }, + ) + status = await client.get(f"/v1/ingestion-jobs/{job.json()['id']}") + + assert upload.status_code == 201, upload.text + assert upload_replay.status_code == 201, upload_replay.text + assert conflict_first.status_code == 201, conflict_first.text + assert job.status_code == 201, job.text + assert source.status_code == 201, source.text + assert status.status_code == 200, status.text + context.upload = typ.cast("dict[str, object]", upload.json()) + context.upload_replay = typ.cast("dict[str, object]", upload_replay.json()) + context.conflict = typ.cast("dict[str, object]", conflict.json()) + context.conflict_status = conflict.status_code + context.job = typ.cast("dict[str, object]", job.json()) + context.source = typ.cast("dict[str, object]", source.json()) + context.status = typ.cast("dict[str, object]", status.json()) + + +@when("an editor uploads source material and attaches it to a new ingestion job") +def upload_and_attach_source( + context: SourceIntakeContext, + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """Exercise the public upload, job creation, source attachment, and poll flow.""" + + async def _run_workflow() -> None: + dependencies = build_api_dependencies( + session_factory, + object_store=FilesystemObjectStore(tmp_path / "bdd-objects"), + ) + transport = httpx.ASGITransport( + app=typ.cast("_ASGIApp", create_app(dependencies)) + ) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + await _run_intake_api_calls(client, context) + + asyncio.run(_run_workflow()) + + +@then("the ingestion job is ready for generation") +def assert_job_ready(context: SourceIntakeContext) -> None: + """Verify the source attachment transitioned the intake state.""" + assert context.upload is not None + assert context.job is not None + assert context.source is not None + assert context.status is not None + assert context.job["intake_state"] == "awaiting_sources" + assert context.source["upload_id"] == context.upload["id"] + assert context.status["intake_state"] == "ready_for_generation" + + +@then("repeated upload requests replay the stored response") +def assert_upload_replay(context: SourceIntakeContext) -> None: + """Verify an identical idempotency key/body pair returns the stored upload.""" + assert context.upload is not None + assert context.upload_replay == context.upload + + +@then("changed upload bodies with the same idempotency key conflict") +def assert_upload_conflict(context: SourceIntakeContext) -> None: + """Verify a reused idempotency key with a different body returns 409.""" + assert context.conflict_status == 409 + assert context.conflict is not None + assert context.conflict["code"] == "idempotency_conflict" + + +async def _create_series_profile(client: httpx.AsyncClient) -> str: + """Create a series profile through the public API and return its id.""" + response = await client.post( + "/v1/series-profiles", + json={ + "slug": "bdd-source-intake", + "title": "BDD Source Intake", + "description": "Created for source-intake behaviour tests.", + "configuration": {"tone": "clear"}, + "guardrails": {"instruction": "Keep claims sourced."}, + "actor": "bdd-source-intake@example.com", + "note": "Initial profile", + }, + ) + assert response.status_code == 201, response.text + return typ.cast("str", response.json()["id"]) diff --git a/tests/test_api_route_versioning.py b/tests/test_api_route_versioning.py index e7b33c57..1a8b742b 100644 --- a/tests/test_api_route_versioning.py +++ b/tests/test_api_route_versioning.py @@ -51,6 +51,13 @@ ), ("/reference-bindings", "reference-bindings"), ("/reference-bindings/not-a-valid-uuid", "reference-binding"), + ("/uploads", "uploads"), + ("/ingestion-jobs", "ingestion-jobs"), + ("/ingestion-jobs/not-a-valid-uuid", "ingestion-job"), + ( + "/ingestion-jobs/not-a-valid-uuid/sources", + "ingestion-job-sources", + ), ) _UNVERSIONED_CANONICAL_PATHS = tuple( @@ -108,7 +115,10 @@ def test_versioned_health_routes_are_not_registered( ) -@pytest.mark.parametrize("path", ["/series-profiles", "/episode-templates"]) +@pytest.mark.parametrize( + "path", + ["/series-profiles", "/episode-templates", "/uploads", "/ingestion-jobs"], +) def test_unversioned_canonical_write_routes_are_not_registered( canonical_api_client: testing.TestClient, path: str, diff --git a/tests/test_brief_loaders.py b/tests/test_brief_loaders.py index 9d11bda7..2ce3875a 100644 --- a/tests/test_brief_loaders.py +++ b/tests/test_brief_loaders.py @@ -1,4 +1,14 @@ -"""Tests for brief loader helpers.""" +"""Tests for brief loaders in ``_brief_loaders``. + +Brief loaders load reference-document revisions and documents by ID, +serialise reference bindings, and raise on missing IDs. They exercise +``ReferenceBinding``, ``ReferenceDocument``, ``ReferenceDocumentRevision``, +``_load_revisions_by_id``, ``_load_documents_by_id``, +``_serialize_bindings_for_owner``, and ``_raise_if_missing_ids``. The brief +generation pipeline calls them to resolve document bindings before building +generation briefs. Stub repositories keep these tests in-memory and free of +database fixtures. +""" import typing as typ import uuid @@ -14,6 +24,8 @@ ReferenceDocumentRevision, ) from episodic.canonical.profile_templates._brief_loaders import ( + _load_documents_by_id, + _load_revisions_by_id, _raise_if_missing_ids, _serialize_bindings_for_owner, ) @@ -154,3 +166,106 @@ def test_it_serializes_valid_bindings(self) -> None: assert content["data"] == "test", ( "Serialised content must preserve the revision payload." ) + + +class TestBriefLoaderMissingReferences: + """Tests for missing revision and document edge paths.""" + + @pytest.mark.asyncio + async def test_load_revisions_by_id_rejects_missing_binding_revision( + self, + ) -> None: + """Raise when a binding points at a missing revision.""" + import datetime as dt + + now = dt.datetime.now(tz=dt.UTC) + missing_revision_id = uuid.uuid4() + binding = ReferenceBinding( + id=uuid.uuid4(), + reference_document_revision_id=missing_revision_id, + target_kind=ReferenceBindingTargetKind.SERIES_PROFILE, + series_profile_id=uuid.uuid4(), + episode_template_id=None, + ingestion_job_id=None, + effective_from_episode_id=None, + created_at=now, + ) + uow = typ.cast( + "typ.Any", + _ReferenceLoaderUnitOfWork( + revisions=[], + documents=[], + ), + ) + + with pytest.raises(ValueError, match="missing revision"): + await _load_revisions_by_id(uow=uow, bindings=[binding]) + + @pytest.mark.asyncio + async def test_load_documents_by_id_rejects_missing_revision_document( + self, + ) -> None: + """Raise when a revision points at a missing document.""" + import datetime as dt + + now = dt.datetime.now(tz=dt.UTC) + revision = ReferenceDocumentRevision( + id=uuid.uuid4(), + reference_document_id=uuid.uuid4(), + content={}, + content_hash="hash", + author=None, + change_note=None, + created_at=now, + ) + uow = typ.cast( + "typ.Any", + _ReferenceLoaderUnitOfWork( + revisions=[revision], + documents=[], + ), + ) + + with pytest.raises(ValueError, match="missing document"): + await _load_documents_by_id(uow=uow, revisions=[revision]) + + +class _ReferenceLoaderUnitOfWork: + """Minimal unit-of-work stub for brief loader edge-path tests.""" + + def __init__( + self, + *, + revisions: list[ReferenceDocumentRevision], + documents: list[ReferenceDocument], + ) -> None: + self.reference_document_revisions = _RevisionRepositoryStub(revisions) + self.reference_documents = _DocumentRepositoryStub(documents) + + +class _RevisionRepositoryStub: + """Return matching revisions from an in-memory collection.""" + + def __init__(self, revisions: list[ReferenceDocumentRevision]) -> None: + self._revisions = revisions + + async def list_by_ids( + self, + ids: set[uuid.UUID], + ) -> list[ReferenceDocumentRevision]: + """Return revisions with identifiers present in ``ids``.""" + return [revision for revision in self._revisions if revision.id in ids] + + +class _DocumentRepositoryStub: + """Return matching documents from an in-memory collection.""" + + def __init__(self, documents: list[ReferenceDocument]) -> None: + self._documents = documents + + async def list_by_ids( + self, + ids: set[uuid.UUID], + ) -> list[ReferenceDocument]: + """Return documents with identifiers present in ``ids``.""" + return [document for document in self._documents if document.id in ids] diff --git a/tests/test_env_runtime_wiring.py b/tests/test_env_runtime_wiring.py index e4e64efc..fceedc2a 100644 --- a/tests/test_env_runtime_wiring.py +++ b/tests/test_env_runtime_wiring.py @@ -67,12 +67,21 @@ async def test_create_app_from_env_wires_database_readiness_probe( from episodic.api.runtime import create_app_from_env app = create_app_from_env() - transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", app)) - async with httpx.AsyncClient( - transport=transport, - base_url="http://testserver", - ) as client: - response = await client.get("/health/ready") + try: + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", app)) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + response = await client.get("/health/ready") + finally: + await scaffold_support.run_asgi_lifespan( + typ.cast("_ASGIApp", app), + ( + scaffold_support.LifespanEvent(type="lifespan.startup"), + scaffold_support.LifespanEvent(type="lifespan.shutdown"), + ), + ) assert response.status_code == 200, ( f"unexpected readiness status code: {response.status_code}" diff --git a/tests/test_filesystem_object_store.py b/tests/test_filesystem_object_store.py new file mode 100644 index 00000000..a613c4bb --- /dev/null +++ b/tests/test_filesystem_object_store.py @@ -0,0 +1,74 @@ +"""Tests for the filesystem source-intake object-store adapter.""" + +import asyncio +import typing as typ + +import pytest + +from episodic.canonical.object_store import ( + InvalidObjectKeyError, + PayloadTooLargeError, +) +from episodic.canonical.storage.filesystem_object_store import FilesystemObjectStore + +if typ.TYPE_CHECKING: + import collections.abc as cabc + import pathlib + + +async def _byte_stream(*chunks: bytes) -> cabc.AsyncIterator[bytes]: + """Yield byte chunks for object-store tests.""" + for chunk in chunks: + await _checkpoint() + yield chunk + + +async def _checkpoint() -> None: + """Give async tests a real scheduling point.""" + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_filesystem_object_store_round_trips_chunks( + tmp_path: pathlib.Path, +) -> None: + """Stored objects should report size/hash and read back the same bytes.""" + store = FilesystemObjectStore(tmp_path) + + stored = await store.put( + "uploads/example.bin", + _byte_stream(b"hello", b"\n"), + max_bytes=6, + ) + + assert stored.key == "uploads/example.bin" + assert stored.size == 6 + assert stored.sha256 == ( + "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + async with store.open(stored.key) as chunks: + assert b"".join([chunk async for chunk in chunks]) == b"hello\n" + + +@pytest.mark.asyncio +async def test_filesystem_object_store_rejects_path_traversal( + tmp_path: pathlib.Path, +) -> None: + """Object keys must stay relative to the configured root.""" + store = FilesystemObjectStore(tmp_path) + + with pytest.raises(InvalidObjectKeyError): + await store.put("../escape.bin", _byte_stream(b"nope"), max_bytes=10) + + +@pytest.mark.asyncio +async def test_filesystem_object_store_rejects_oversized_payload( + tmp_path: pathlib.Path, +) -> None: + """PayloadTooLargeError should leave no target object behind.""" + store = FilesystemObjectStore(tmp_path) + + with pytest.raises(PayloadTooLargeError): + await store.put("uploads/too-large.bin", _byte_stream(b"abcdef"), max_bytes=5) + + assert not (tmp_path / "uploads" / "too-large.bin").exists() diff --git a/tests/test_idempotency_properties.py b/tests/test_idempotency_properties.py new file mode 100644 index 00000000..aae18678 --- /dev/null +++ b/tests/test_idempotency_properties.py @@ -0,0 +1,174 @@ +"""Property tests for source-intake idempotency invariants.""" + +import dataclasses as dc +import datetime as dt +import uuid + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from episodic.canonical.idempotency import ( + Acquired, + Conflict, + IdempotencyAcquireRequest, + IdempotencyRecord, + IdempotencyState, + InFlight, + Replay, +) +from episodic.canonical.upload_protocols import IdempotencyStore + + +@dc.dataclass +class _InMemoryIdempotencyStore(IdempotencyStore): + """In-memory store mirroring the domain idempotency state machine.""" + + records: dict[tuple[str | None, str, str], IdempotencyRecord] = dc.field( + default_factory=dict + ) + + async def acquire( + self, + *, + request: IdempotencyAcquireRequest, + ) -> Acquired | Replay | Conflict | InFlight: + """Acquire or inspect a keyed idempotency record.""" + key = (request.principal_id, request.operation, request.idempotency_key) + now = dt.datetime.now(dt.UTC) + existing = self.records.get(key) + if existing is None: + record = IdempotencyRecord( + id=uuid.uuid4(), + principal_id=request.principal_id, + operation=request.operation, + idempotency_key=request.idempotency_key, + body_hash=request.body_hash, + state=IdempotencyState.IN_FLIGHT, + serialised_outcome=None, + expires_at=request.expires_at, + created_at=now, + updated_at=now, + ) + self.records[key] = record + return Acquired(record.id) + if existing.body_hash != request.body_hash: + return Conflict(existing.id) + if existing.state is IdempotencyState.IN_FLIGHT: + return InFlight(existing.id) + if existing.serialised_outcome is None: # pragma: no cover - defensive + msg = "completed in-memory records require an outcome." + raise AssertionError(msg) + return Replay(existing.serialised_outcome) + + async def complete( + self, + *, + record_id: uuid.UUID, + serialised_outcome: bytes, + ) -> None: + """Complete a record by identifier.""" + for key, record in self.records.items(): + if record.id == record_id: + self.records[key] = dc.replace( + record, + state=IdempotencyState.COMPLETED, + serialised_outcome=serialised_outcome, + updated_at=dt.datetime.now(dt.UTC), + ) + return + msg = f"unknown idempotency record: {record_id}" + raise LookupError(msg) + + async def lookup( + self, + *, + principal_id: str | None, + operation: str, + idempotency_key: str, + ) -> IdempotencyRecord | None: + """Fetch a stored idempotency record by logical key.""" + return self.records.get((principal_id, operation, idempotency_key)) + + +@pytest.mark.asyncio +@pytest.mark.hypothesis +@given( + body_hash=st.text(min_size=1).filter(str.strip), + replay_payload=st.binary(min_size=1), +) +async def test_identical_body_for_key_replays_one_completed_resource( + body_hash: str, + replay_payload: bytes, +) -> None: + """Property: identical bodies for a key converge on one completed record.""" + store = _InMemoryIdempotencyStore() + expires_at = dt.datetime.now(dt.UTC) + dt.timedelta(hours=1) + + first = await store.acquire( + request=IdempotencyAcquireRequest( + principal_id="principal", + operation="upload.create", + idempotency_key="same-key", + body_hash=body_hash, + expires_at=expires_at, + ), + ) + assert isinstance(first, Acquired) + await store.complete(record_id=first.record_id, serialised_outcome=replay_payload) + + second = await store.acquire( + request=IdempotencyAcquireRequest( + principal_id="principal", + operation="upload.create", + idempotency_key="same-key", + body_hash=body_hash, + expires_at=expires_at, + ), + ) + + assert isinstance(second, Replay) + assert second.serialised_outcome == replay_payload + assert len(store.records) == 1 + + +@pytest.mark.asyncio +@pytest.mark.hypothesis +@given( + first_hash=st.text(min_size=1).filter(str.strip), + second_hash=st.text(min_size=1).filter(str.strip), +) +async def test_different_body_for_key_conflicts( + first_hash: str, + second_hash: str, +) -> None: + """Property: reusing a key with a different body hash conflicts.""" + if first_hash == second_hash: + second_hash = f"{second_hash}:different" + store = _InMemoryIdempotencyStore() + expires_at = dt.datetime.now(dt.UTC) + dt.timedelta(hours=1) + + first = await store.acquire( + request=IdempotencyAcquireRequest( + principal_id="principal", + operation="upload.create", + idempotency_key="same-key", + body_hash=first_hash, + expires_at=expires_at, + ), + ) + assert isinstance(first, Acquired) + + second = await store.acquire( + request=IdempotencyAcquireRequest( + principal_id="principal", + operation="upload.create", + idempotency_key="same-key", + body_hash=second_hash, + expires_at=expires_at, + ), + ) + + assert isinstance(second, Conflict) + assert second.record_id == first.record_id + assert len(store.records) == 1 diff --git a/tests/test_idempotency_service.py b/tests/test_idempotency_service.py new file mode 100644 index 00000000..67cf8006 --- /dev/null +++ b/tests/test_idempotency_service.py @@ -0,0 +1,38 @@ +"""Tests for source-intake idempotency fingerprint helpers.""" + +import hashlib + +from episodic.canonical.idempotency_service import ( + canonical_json_bytes, + multipart_request_hash, +) + + +def test_canonical_json_bytes_are_sorted_and_compact() -> None: + """Canonical JSON should be stable across input key order.""" + first = canonical_json_bytes({"b": 2, "a": 1}) + second = canonical_json_bytes({"a": 1, "b": 2}) + + assert first == b'{"a":1,"b":2}' + assert first == second + + +def test_multipart_request_hash_matches_adr_015_worked_vector() -> None: + """Pin the ADR 015 multipart fingerprint worked example.""" + body_sha256 = hashlib.sha256(b"hello\n").hexdigest() + + result = multipart_request_hash( + "upload.create", + body_sha256=body_sha256, + metadata={ + "content_type": "text/plain", + "declared_sha256": None, + "declared_size": 6, + "ignored": "not part of the operation allowlist", + }, + ) + + assert body_sha256 == ( + "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + assert result == "f03f8d4c738536bcd1c13cc34d6816f8ea0672c3e2d47c2cbbaf5c8ecbda5e2c" diff --git a/tests/test_reference_document_bindings_facade.py b/tests/test_reference_document_bindings_facade.py new file mode 100644 index 00000000..377272fb --- /dev/null +++ b/tests/test_reference_document_bindings_facade.py @@ -0,0 +1,193 @@ +"""Functional regression tests for the reference-document bindings facade.""" + +import typing as typ +import uuid + +import pytest +import test_reference_document_service_support as support + +from episodic.canonical.domain import ( + ReferenceBinding, + ReferenceBindingTargetKind, +) +from episodic.canonical.reference_documents import ( + ReferenceBindingData, + ReferenceBindingListRequest, + ReferenceDocumentCreateData, + ReferenceDocumentRevisionData, + bindings, + create_reference_document, + create_reference_document_revision, +) +from episodic.canonical.storage import SqlAlchemyUnitOfWork + +if typ.TYPE_CHECKING: + import collections.abc as cabc + + from sqlalchemy.ext.asyncio import AsyncSession + + +service_fixture = support.service_fixture +ServiceFixture = support.ServiceFixture + + +async def _create_revision( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, + *, + content_hash: str, +) -> uuid.UUID: + """Create a reference-document revision for facade binding tests.""" + async with SqlAlchemyUnitOfWork(session_factory) as uow: + document = await create_reference_document( + uow, + data=ReferenceDocumentCreateData( + owner_series_profile_id=service_fixture["primary_profile_id"], + kind="style_guide", + lifecycle_state="active", + metadata={"title": content_hash}, + ), + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + revision = await create_reference_document_revision( + uow, + document_id=str(document.id), + owner_series_profile_id=service_fixture["primary_profile_id"], + data=ReferenceDocumentRevisionData( + content={"version": content_hash}, + content_hash=content_hash, + author="tester@example.com", + change_note="Facade regression test revision", + ), + ) + + return revision.id + + +async def _create_series_binding( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, + *, + content_hash: str = "facade-binding", +) -> ReferenceBinding: + """Create a series-profile binding through the bindings facade.""" + revision_id = await _create_revision( + session_factory, + service_fixture, + content_hash=content_hash, + ) + async with SqlAlchemyUnitOfWork(session_factory) as uow: + return await bindings.create_reference_binding( + uow, + data=ReferenceBindingData( + reference_document_revision_id=str(revision_id), + target_kind="series_profile", + series_profile_id=service_fixture["primary_profile_id"], + episode_template_id=None, + ingestion_job_id=None, + effective_from_episode_id=None, + ), + ) + + +def test_bindings_facade_exports_public_entry_points() -> None: + """Expose the expected public binding entry points.""" + assert set(bindings.__all__) == { + "create_reference_binding", + "get_reference_binding", + "list_reference_bindings", + "list_reference_bindings_paged", + } + + +@pytest.mark.asyncio +async def test_create_reference_binding_returns_created_binding( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, +) -> None: + """Create and return a series-profile binding through the facade.""" + binding = await _create_series_binding(session_factory, service_fixture) + + assert isinstance(binding, ReferenceBinding) + assert binding.target_kind == ReferenceBindingTargetKind.SERIES_PROFILE + assert binding.series_profile_id == uuid.UUID(service_fixture["primary_profile_id"]) + + +@pytest.mark.asyncio +async def test_get_reference_binding_returns_matching_binding( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, +) -> None: + """Fetch a persisted binding through the facade.""" + created = await _create_series_binding(session_factory, service_fixture) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + fetched = await bindings.get_reference_binding( + uow, + binding_id=str(created.id), + ) + + assert isinstance(fetched, ReferenceBinding) + assert fetched.id == created.id + assert ( + fetched.reference_document_revision_id == created.reference_document_revision_id + ) + + +@pytest.mark.asyncio +async def test_list_reference_bindings_returns_target_page( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, +) -> None: + """List target bindings through the facade.""" + created = await _create_series_binding(session_factory, service_fixture) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + results = await bindings.list_reference_bindings( + uow, + request=ReferenceBindingListRequest( + target_kind="series_profile", + target_id=service_fixture["primary_profile_id"], + limit=10, + offset=0, + ), + ) + + assert isinstance(results, list) + assert [binding.id for binding in results] == [created.id] + assert results[0].series_profile_id == created.series_profile_id + + +@pytest.mark.asyncio +async def test_list_reference_bindings_paged_returns_page_and_total( + session_factory: cabc.Callable[[], AsyncSession], + service_fixture: ServiceFixture, +) -> None: + """List target bindings and total count through the facade.""" + first = await _create_series_binding( + session_factory, + service_fixture, + content_hash="facade-binding-paged-first", + ) + second = await _create_series_binding( + session_factory, + service_fixture, + content_hash="facade-binding-paged-second", + ) + + async with SqlAlchemyUnitOfWork(session_factory) as uow: + results, total = await bindings.list_reference_bindings_paged( + uow, + request=ReferenceBindingListRequest( + target_kind="series_profile", + target_id=service_fixture["primary_profile_id"], + limit=1, + offset=1, + ), + ) + + assert isinstance(results, list) + assert total == 2 + assert [binding.id for binding in results] == [second.id] + assert first.id != second.id diff --git a/tests/test_source_intake_api.py b/tests/test_source_intake_api.py new file mode 100644 index 00000000..2578b480 --- /dev/null +++ b/tests/test_source_intake_api.py @@ -0,0 +1,229 @@ +"""Integration tests for the source-intake REST workflow.""" + +import hashlib +import typing as typ + +import httpx +import pytest + +from episodic.api import create_app +from episodic.canonical.storage import FilesystemObjectStore +from tests.fixtures.api import build_api_dependencies + +if typ.TYPE_CHECKING: + from pathlib import Path + + from httpx._transports.asgi import _ASGIApp + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + from syrupy.assertion import SnapshotAssertion + + +@pytest.mark.asyncio +async def test_source_intake_upload_job_and_attach_flow( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """Client can upload bytes, create a job, attach the upload, and poll ready.""" + object_store = FilesystemObjectStore(tmp_path / "objects") + dependencies = build_api_dependencies(session_factory, object_store=object_store) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + profile_id = await _create_series_profile(client) + payload = b"hello\n" + upload_response = await client.post( + "/v1/uploads", + headers={"Idempotency-Key": "upload-key"}, + files={ + "file": ("source.txt", payload, "text/plain"), + "content_type": (None, "text/plain"), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + "metadata": (None, '{"language":"en"}', "application/json"), + }, + ) + assert upload_response.status_code == 201, upload_response.text + replay_response = await client.post( + "/v1/uploads", + headers={"Idempotency-Key": "upload-key"}, + files={ + "file": ("source.txt", payload, "text/plain"), + "content_type": (None, "text/plain"), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + "metadata": (None, '{"language":"en"}', "application/json"), + }, + ) + job_response = await client.post( + "/v1/ingestion-jobs", + headers={"Idempotency-Key": "job-key"}, + json={"series_profile_id": profile_id, "target_episode_id": None}, + ) + source_response = await client.post( + f"/v1/ingestion-jobs/{job_response.json()['id']}/sources", + headers={"Idempotency-Key": "source-key"}, + json={ + "type": "upload", + "upload_id": upload_response.json()["id"], + "source_type": "research_paper", + "weight": 1.0, + "metadata": {"language": "en"}, + }, + ) + status_response = await client.get( + f"/v1/ingestion-jobs/{job_response.json()['id']}" + ) + + assert replay_response.status_code == 201 + assert replay_response.json() == upload_response.json() + assert upload_response.json()["content_hash"].startswith("sha256:") + assert job_response.status_code == 201 + assert job_response.json()["intake_state"] == "awaiting_sources" + assert source_response.status_code == 201 + assert source_response.json()["upload_id"] == upload_response.json()["id"] + assert status_response.status_code == 200 + assert status_response.json()["intake_state"] == "ready_for_generation" + + +@pytest.mark.asyncio +async def test_source_intake_idempotency_conflict( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, +) -> None: + """Same idempotency key with different upload body returns 409.""" + object_store = FilesystemObjectStore(tmp_path / "objects") + dependencies = build_api_dependencies(session_factory, object_store=object_store) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + first = await _post_text_upload(client, key="conflict-key", payload=b"hello\n") + second = await _post_text_upload(client, key="conflict-key", payload=b"bye\n") + + assert first.status_code == 201, first.text + assert second.status_code == 409 + assert second.json()["code"] == "idempotency_conflict" + + +@pytest.mark.asyncio +async def test_source_intake_response_envelope_snapshot( + session_factory: async_sessionmaker[AsyncSession], + tmp_path: Path, + snapshot: SnapshotAssertion, +) -> None: + """Snapshot stable fields from source-intake response envelopes.""" + object_store = FilesystemObjectStore(tmp_path / "objects") + dependencies = build_api_dependencies(session_factory, object_store=object_store) + transport = httpx.ASGITransport(app=typ.cast("_ASGIApp", create_app(dependencies))) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + ) as client: + profile_id = await _create_series_profile(client) + upload_response = await _post_text_upload( + client, + key="snapshot-upload-key", + payload=b"snapshot\n", + ) + job_response = await client.post( + "/v1/ingestion-jobs", + headers={"Idempotency-Key": "snapshot-job-key"}, + json={"series_profile_id": profile_id, "target_episode_id": None}, + ) + source_response = await client.post( + f"/v1/ingestion-jobs/{job_response.json()['id']}/sources", + headers={"Idempotency-Key": "snapshot-source-key"}, + json={ + "type": "upload", + "upload_id": upload_response.json()["id"], + "source_type": "research_paper", + "weight": 0.75, + "metadata": {"language": "en"}, + }, + ) + status_response = await client.get( + f"/v1/ingestion-jobs/{job_response.json()['id']}" + ) + + assert upload_response.status_code == 201, upload_response.text + assert job_response.status_code == 201, job_response.text + assert source_response.status_code == 201, source_response.text + assert status_response.status_code == 200, status_response.text + assert { + "upload": _stable_upload_fields(upload_response.json()), + "job": _stable_job_fields(job_response.json()), + "source": _stable_source_fields(source_response.json()), + "status": _stable_job_fields(status_response.json()), + } == snapshot + + +async def _create_series_profile(client: httpx.AsyncClient) -> str: + """Create a series profile through the public API and return its id.""" + response = await client.post( + "/v1/series-profiles", + json={ + "slug": "source-intake", + "title": "Source Intake", + "description": "Created for intake tests.", + "configuration": {"tone": "clear"}, + "guardrails": {"instruction": "Keep claims sourced."}, + "actor": "api-user@example.com", + "note": "Initial profile", + }, + ) + assert response.status_code == 201, response.text + return typ.cast("str", response.json()["id"]) + + +def _stable_upload_fields(payload: dict[str, object]) -> dict[str, object]: + """Return the stable upload fields that define the public response shape.""" + content_hash = typ.cast("str", payload["content_hash"]) + return { + "state": payload["state"], + "content_hash_algorithm": content_hash.split(":", maxsplit=1)[0], + "content_type": payload["content_type"], + "size_bytes": payload["size_bytes"], + "metadata": payload["metadata"], + } + + +def _stable_job_fields(payload: dict[str, object]) -> dict[str, object]: + """Return stable ingestion-job response fields.""" + return { + "status": payload["status"], + "intake_state": payload["intake_state"], + "next_poll_after_seconds": payload.get("next_poll_after_seconds"), + } + + +def _stable_source_fields(payload: dict[str, object]) -> dict[str, object]: + """Return stable source-attachment response fields.""" + return { + "type": payload["type"], + "source_type": payload["source_type"], + "weight": payload["weight"], + "source_uri": payload["source_uri"], + "metadata": payload["metadata"], + } + + +async def _post_text_upload( + client: httpx.AsyncClient, + *, + key: str, + payload: bytes, +) -> httpx.Response: + """Post a text upload with a deterministic multipart shape.""" + return await client.post( + "/v1/uploads", + headers={"Idempotency-Key": key}, + files={ + "file": ("source.txt", payload, "text/plain"), + "content_type": (None, "text/plain"), + "declared_size": (None, str(len(payload))), + "declared_sha256": (None, hashlib.sha256(payload).hexdigest()), + }, + )