diff --git a/changelog.d/654-uk-rowwise-frame-entry.changed.md b/changelog.d/654-uk-rowwise-frame-entry.changed.md new file mode 100644 index 00000000..989b7f91 --- /dev/null +++ b/changelog.d/654-uk-rowwise-frame-entry.changed.md @@ -0,0 +1 @@ +Narrow the UK rowwise clone entry to Frame or H5 inputs and refuse retired duck-typed in-memory dataset carriers. diff --git a/changelog.d/654-uk-schema3-terminal-gates.removed.md b/changelog.d/654-uk-schema3-terminal-gates.removed.md new file mode 100644 index 00000000..b226fbc8 --- /dev/null +++ b/changelog.d/654-uk-schema3-terminal-gates.removed.md @@ -0,0 +1 @@ +Retire the legacy UK schema-3 terminal gate report path after verifying schema-4 battery parity with a saved differential receipt. diff --git a/changelog.d/654-uk-typed-household-weights.changed.md b/changelog.d/654-uk-typed-household-weights.changed.md new file mode 100644 index 00000000..c46f32ed --- /dev/null +++ b/changelog.d/654-uk-typed-household-weights.changed.md @@ -0,0 +1 @@ +Store UK national household weights as Frame typed weights during the build, materializing `household_weight` only at H5/export boundaries. diff --git a/changelog.d/654-uk-weights-audit-absence-blocks.changed.md b/changelog.d/654-uk-weights-audit-absence-blocks.changed.md new file mode 100644 index 00000000..c1923ac4 --- /dev/null +++ b/changelog.d/654-uk-weights-audit-absence-blocks.changed.md @@ -0,0 +1 @@ +Block the build in every posture when the UK fit-weight audit evidence is absent: `uk_weights_audit` declares the new `evidence_absent_blocks` manifest flag and the battery honors it, porting the retired schema-3 path's strictness — an absent audit is not a passing audit. diff --git a/packages/microcosm-build/src/microcosm/build/country_spec.py b/packages/microcosm-build/src/microcosm/build/country_spec.py index ed77ccc5..fbca6700 100644 --- a/packages/microcosm-build/src/microcosm/build/country_spec.py +++ b/packages/microcosm-build/src/microcosm/build/country_spec.py @@ -108,7 +108,16 @@ #: extension) would run the gate on defaults while the declared intent #: vanished from ``policy_sha256`` — an unattested threshold. _GATE_ENTRY_KEYS = frozenset( - {"id", "gate", "phase", "criticality", "parameters", "not_applicable", "notes"} + { + "id", + "gate", + "phase", + "criticality", + "parameters", + "not_applicable", + "evidence_absent_blocks", + "notes", + } ) #: Build phases a gate selection may bind to — the shared vocabulary that @@ -392,6 +401,13 @@ class GateSelectionSpec: it appears in every report as ``not_applicable`` and never evaluates. Mutually exclusive with ``parameters`` — an excused gate with tuned thresholds is a contradiction. + evidence_absent_blocks: When true, an ``evidence_absent`` outcome on + this entry blocks the build in every posture, not only under the + release-candidate posture. For entries whose declared intent is + that absence is never excusable (e.g. "an absent audit is not a + passing audit") — the outcome stays honestly ``evidence_absent`` + in the report; only the enforcement changes. Meaningless on an + excused entry, so mutually exclusive with ``not_applicable``. notes: Free-text rationale. """ @@ -401,6 +417,7 @@ class GateSelectionSpec: criticality: str parameters: Mapping[str, Any] = field(default_factory=dict) not_applicable: str | None = None + evidence_absent_blocks: bool = False notes: str = "" def __post_init__(self) -> None: @@ -409,6 +426,11 @@ def __post_init__(self) -> None: "GateSelectionSpec parameters must be a mapping, got " f"{type(self.parameters).__name__}." ) + if not isinstance(self.evidence_absent_blocks, bool): + raise TypeError( + "GateSelectionSpec evidence_absent_blocks must be a bool, got " + f"{type(self.evidence_absent_blocks).__name__}." + ) object.__setattr__( self, "parameters", @@ -478,6 +500,18 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> GateSelectionSpec: "mutually exclusive — an excused gate with tuned " "thresholds is a contradiction." ) + evidence_absent_blocks = raw.get("evidence_absent_blocks", False) + if not isinstance(evidence_absent_blocks, bool): + raise ValueError( + f"gate {gate_id!r}: evidence_absent_blocks must be a JSON " + f"boolean, got {evidence_absent_blocks!r}." + ) + if evidence_absent_blocks and not_applicable is not None: + raise ValueError( + f"gate {gate_id!r}: evidence_absent_blocks and not_applicable " + "are mutually exclusive — an excused entry never evaluates, " + "so demanding its absence block is a contradiction." + ) return cls( id=gate_id, gate=gate, @@ -485,6 +519,7 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> GateSelectionSpec: criticality=criticality, parameters=dict(parameters), not_applicable=not_applicable, + evidence_absent_blocks=evidence_absent_blocks, notes=str(raw.get("notes", "")), ) diff --git a/packages/microcosm-build/src/microcosm/build/gate_battery.py b/packages/microcosm-build/src/microcosm/build/gate_battery.py index 79a7513e..d3cc8561 100644 --- a/packages/microcosm-build/src/microcosm/build/gate_battery.py +++ b/packages/microcosm-build/src/microcosm/build/gate_battery.py @@ -190,6 +190,14 @@ def _gates_manifest_payload(gates: GatesManifest) -> dict[str, object]: "criticality": entry.criticality, "parameters": _json_safe(entry.parameters), "not_applicable": entry.not_applicable, + # Present iff armed: a true flag must ride the policy hash, + # while the false default stays out so unflagged entries + # (and the US manifest) keep their serialized form. + **( + {"evidence_absent_blocks": True} + if entry.evidence_absent_blocks + else {} + ), "notes": entry.notes, } for entry in gates.gates @@ -487,7 +495,11 @@ def blocking_outcomes(self, *, release_candidate: bool) -> tuple[GateOutcome, .. build without, say, an incumbent parity snapshot gets an honest non-shippable report instead of a crash, while a release build cannot excuse missing evidence — a missing frozen reference is not - a passing gate. Diagnostic entries never block. + a passing gate. An entry declaring ``evidence_absent_blocks`` opts + out of that dev-posture leniency: its absence blocks every posture + (the legacy UK weights-audit strictness, ported during the #654 + schema-3 retirement — "an absent audit is not a passing audit"). + Diagnostic entries never block. """ blocking = [] @@ -496,7 +508,9 @@ def blocking_outcomes(self, *, release_candidate: bool) -> tuple[GateOutcome, .. continue if outcome.status is GateStatus.FAILED: blocking.append(outcome) - elif outcome.status is GateStatus.EVIDENCE_ABSENT and release_candidate: + elif outcome.status is GateStatus.EVIDENCE_ABSENT and ( + release_candidate or outcome.entry.evidence_absent_blocks + ): blocking.append(outcome) return tuple(blocking) @@ -916,6 +930,14 @@ def _policy_sha256(self) -> str: "criticality": entry.criticality, "parameters": _json_safe(dict(entry.parameters)), "not_applicable": entry.not_applicable, + # Enforcement policy rides the policy hash when armed; + # the false default stays out so unflagged entries keep + # their digest. + **( + {"evidence_absent_blocks": True} + if entry.evidence_absent_blocks + else {} + ), } for entry in sorted(self._gates.gates, key=lambda e: e.id) ] diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 97b51928..3ba0a828 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -98,7 +98,8 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": {}, - "notes": "Every fit-produced weight column carries a completed audit record. Armed by the SPI income stage; an absent audit is not a passing audit." + "evidence_absent_blocks": true, + "notes": "Every fit-produced weight column carries a completed audit record. Armed by the SPI income stage; an absent audit is not a passing audit — and blocks every posture, not only release candidates: the legacy schema-3 path's strictness, ported during the #654 retirement (microcosm#691 review)." }, { "id": "uk_export_surface", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index cea275d3..17980279 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -377,20 +377,14 @@ from microcosm.build.uk_runtime.terminal_gates import ( UK_DEFAULT_ZERO_WEIGHT_STRATA, UK_MAX_TARGET_ABS_RELATIVE_ERROR, - UK_MAX_TO_MEDIAN_WEIGHT_RATIO, - UK_MIN_ESS_FRACTION, - UK_TERMINAL_GATE_SCHEMA_VERSION, - UKReleaseParityEvidence, UKZeroWeightStratumDeclaration, uk_degenerate_release_surface_gate, uk_export_surface_gate, uk_target_fit_gate, uk_target_surface_gate, - uk_terminal_gate_report, uk_weight_ess_gate, uk_weight_ratio_gate, uk_zero_weight_strata_gate, - write_uk_terminal_gate_report, ) from microcosm.build.uk_runtime.weighted_integrity import ( UKInputMassParityPolicy, @@ -719,13 +713,9 @@ "write_hmrc_replay_report", "UK_DEFAULT_ZERO_WEIGHT_STRATA", "UK_MAX_TARGET_ABS_RELATIVE_ERROR", - "UK_MAX_TO_MEDIAN_WEIGHT_RATIO", - "UK_MIN_ESS_FRACTION", - "UK_TERMINAL_GATE_SCHEMA_VERSION", "UKInputMassParityPolicy", "UKInputMassReference", "UKQRFTailConcentrationPolicy", - "UKReleaseParityEvidence", "UKZeroWeightStratumDeclaration", "load_uk_input_mass_reference", "load_uk_reviewed_exclusion_register", @@ -737,9 +727,7 @@ "uk_qrf_tail_concentration_gate", "uk_target_fit_gate", "uk_target_surface_gate", - "uk_terminal_gate_report", "uk_weight_ess_gate", "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", - "write_uk_terminal_gate_report", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_calibration.py index 68454c1f..ad1bb7ad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_calibration.py @@ -146,8 +146,12 @@ def materialize_uk_cgt_calibration_frame( f"UK CGT person {UK_CGT_SOURCE_COLUMN!r} values must be finite." ) + household_weight_by_id = pd.Series( + national_frame.weights_for("household").values, + index=household["household_id"].to_numpy(), + ) mapped_mass = person["person_household_id"].map( - household.set_index("household_id")["household_weight"] + household_weight_by_id ) if mapped_mass.isna().any() or not mapped_mass.gt(0.0).all(): raise ValueError( @@ -188,7 +192,7 @@ def materialize_uk_cgt_calibration_frame( EntitySchema(group_entities=("household",)), { "household": Weights( - household["household_weight"].to_numpy(dtype=float), + national_frame.weights_for("household").values, uk_household_weight_kind(national_frame), ) }, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index a264e532..a26764d9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -443,9 +443,11 @@ def impute_uk_capital_gains( if "capital_gains" not in person.columns: raise ValueError("Person table has no capital_gains column to redraw.") - weights_by_household = frame.table("household").set_index("household_id")[ - "household_weight" - ] + household = frame.table("household") + weights_by_household = pd.Series( + frame.weights_for("household").values, + index=household["household_id"], + ) missing_households = set(person["person_household_id"]) - set( weights_by_household.index ) @@ -539,11 +541,8 @@ def impute_uk_capital_gains( # through; the appended record is a conservation receipt, not a change — # the terminal family gate requires it, so a build whose CGT stage moved # mass or never ran fails by name. - household_mass = float( - pd.to_numeric( - frame.table("household")["household_weight"], errors="raise" - ).sum() - ) + weights = frame.weights_for("household") + household_mass = float(weights.total) receipt = MassChangeRecord( entity="household", old_total=household_mass, @@ -557,6 +556,7 @@ def impute_uk_capital_gains( household=frame.table("household"), time_period=time_period, weight_kind=uk_household_weight_kind(frame), + household_weights=weights.values, mass_log=(*frame.mass_log, receipt), ) validate_uk_national_frame(result_frame) @@ -577,9 +577,11 @@ def summarize_uk_cgt_imputation( calibration adjudication's question. """ person = after.table("person").reset_index(drop=True) - weights_by_household = after.table("household").set_index("household_id")[ - "household_weight" - ] + household = after.table("household") + weights_by_household = pd.Series( + after.weights_for("household").values, + index=household["household_id"], + ) weight = ( person["person_household_id"].map(weights_by_household).to_numpy(dtype=float) ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index cbdc303b..732a5233 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -31,6 +31,7 @@ effective_sample_size, ) from microcosm.calibrate.solve import CalibrationResult +from microcosm.frame import Frame __all__ = [ "UK_DIAGNOSTICS_SCHEMA_VERSION", @@ -402,7 +403,7 @@ def _target_pass_rates( def uk_calibration_diagnostics_payload( result: CalibrationResult, - household: pd.DataFrame, + frame: Frame, *, target_geography_levels: Mapping[str, object], target_registry: TargetRegistry, @@ -417,14 +418,11 @@ def uk_calibration_diagnostics_payload( """ registry = _require_uk_target_registry(target_registry) - if not isinstance(household, pd.DataFrame): - raise TypeError("UK diagnostic household data must be a pandas DataFrame.") - if "household_weight" not in household: - raise ValueError( - "UK diagnostic household data must contain 'household_weight'." - ) + if not isinstance(frame, Frame): + raise TypeError("UK diagnostic data must be a Frame.") + household = frame.table("household") result_weights = _as_weights(result.weights) - shipped_weights = _as_weights(household["household_weight"].to_numpy()) + shipped_weights = _as_weights(frame.weights_for("household").values) if shipped_weights.shape != result_weights.shape or not np.array_equal( shipped_weights, result_weights, @@ -465,7 +463,7 @@ def uk_calibration_diagnostics_payload( def write_uk_calibration_diagnostics( result: CalibrationResult, path: Path | str, - household: pd.DataFrame, + frame: Frame, *, target_geography_levels: Mapping[str, object], target_registry: TargetRegistry, @@ -478,7 +476,7 @@ def write_uk_calibration_diagnostics( encoded = json.dumps( uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=target_geography_levels, target_registry=target_registry, stratum_columns=stratum_columns, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py index d17063c4..2154f284 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py @@ -434,6 +434,7 @@ def retain_uk_frs_hmrc_leaves( household=frame.table("household"), time_period=time_period, weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, mass_log=frame.mass_log, ) validate_uk_national_frame(result_frame) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_calibration.py index efdb2af3..e2ba9a7b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_calibration.py @@ -228,7 +228,10 @@ def materialize_uk_hmrc_calibration_frame( ): raise RuntimeError("HMRC TI must equal derived TEI + TII exactly.") - positive_household_mass = household.set_index("household_id")["household_weight"] + positive_household_mass = pd.Series( + frame.weights_for("household").values, + index=household["household_id"].to_numpy(), + ) mapped_mass = person["person_household_id"].map(positive_household_mass) if mapped_mass.isna().any() or not mapped_mass.gt(0.0).all(): raise ValueError( @@ -294,7 +297,7 @@ def materialize_uk_hmrc_calibration_frame( EntitySchema(group_entities=("household",)), { "household": Weights( - household["household_weight"].to_numpy(dtype=float), + frame.weights_for("household").values, uk_household_weight_kind(frame), ) }, @@ -392,10 +395,8 @@ def _validate_materialization_inputs( raise ValueError( "HMRC target materialization requires rebuilt importance weights." ) - weights = pd.to_numeric( - frame.table("household")["household_weight"], errors="coerce" - ) - if weights.isna().any() or not weights.gt(0.0).all(): + weights = frame.weights_for("household").values + if not (weights > 0.0).all(): raise ValueError( "HMRC target materialization requires every household prior weight " "to be strictly positive." diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py index 510f4930..d67b823f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py @@ -67,7 +67,7 @@ replace_uk_spi_support_tables, support_channel_column, ) -from microcosm.frame import Frame, WeightKind +from microcosm.frame import Frame, WeightKind, engine_tables __all__ = [ "CERTIFIED_UK_CANDIDATE_FILENAME", @@ -543,10 +543,11 @@ def restore_uk_hmrc_income_family( build_period=time_period, ) + tables = engine_tables(frame, weighted_entities=("household",)) support = replace_uk_spi_support_tables( person=frame.table("person"), benunit=frame.table("benunit"), - household=frame.table("household"), + household=tables["household"], seed=seed, source_year=int(time_period), spi_prior_mass_share=spi_prior_mass_share, @@ -729,9 +730,11 @@ def _distributional_mass_shares(frame: Frame) -> dict[str, float]: ) if not spi_people.any(): raise RuntimeError("Rebuilt HMRC family contains no SPI support people.") - household_weights = frame.table("household").set_index("household_id")[ - "household_weight" - ] + household = frame.table("household") + household_weights = pd.Series( + frame.weights_for("household").values, + index=household["household_id"].to_numpy(), + ) mapped = pd.to_numeric( person["person_household_id"].map(household_weights), errors="coerce", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index 7059a3fa..56d5b83b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -64,10 +64,11 @@ UKInputMassParityPolicy, UKInputMassReference, UKQRFTailConcentrationPolicy, - UKReleaseParityEvidence, +) +from microcosm.build.uk_runtime.weighted_integrity import ( UKReviewedExclusion, + exclusion_evaluation_date, ) -from microcosm.build.uk_runtime.weighted_integrity import exclusion_evaluation_date from microcosm.frame import ( Frame, MassChangeRecord, @@ -346,7 +347,6 @@ def build_uk_national_dataset( calibration_diagnostics_sha256: str, stages: Sequence[UKNationalStage | PlanStage] | StagePlan = (), coverage_engine: Any | None = None, - parity_evidence: UKReleaseParityEvidence | None = None, input_mass_reference: UKInputMassReference | None = None, input_mass_policy: UKInputMassParityPolicy | None = None, qrf_tail_policy: UKQRFTailConcentrationPolicy | None = None, @@ -537,8 +537,6 @@ def build_uk_national_dataset( fit_weight_records = _stage_fit_weight_records(materialized_stages) if fit_weight_records is not None: artifacts["fit_weight_records"] = fit_weight_records - if parity_evidence is not None: - artifacts["parity_evidence"] = parity_evidence if input_mass_reference is not None: artifacts["input_mass_reference"] = input_mass_reference artifacts["input_mass_policy"] = input_mass_policy diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index d6aaca30..2690adfb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -14,24 +14,17 @@ new Frame via :func:`uk_national_frame` with an explicitly extended ``mass_log``; a weight-only update on unchanged tables goes through :meth:`Frame.with_weights`, which enforces the forward-only kind transition -and appends the record itself — and must also refresh the persisted -``household_weight`` column, because :func:`validate_uk_national_frame` -holds the column equal to the typed vector (the staging H5 exports the -column, so a silent disagreement would ship the wrong weights). +and appends the record itself. The ``household_weight`` column itself is a materialized export contract, not carrier state (``engine_tables`` regenerates it from the typed weights -at every export boundary). Dropping it from the in-build tables — the #612 -increment-2 charter item — is assessed and deferred: this module's own -contract makes the column load-bearing (required at construction, asserted -equal to the typed vector at validation), the #611-owned gate modules and -the spi/rowwise reader surface still read it, and the drop would re-open -the #618 carrier review for no behavioural gain. It is sequenced behind the -#611 consumer half and the reader moves, not silently abandoned. +at every export boundary). The H5 loader validates stored-vs-typed +agreement before consuming the column into the Frame's typed vector. """ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @@ -141,36 +134,42 @@ def uk_national_frame( household: pd.DataFrame, time_period: int | str, weight_kind: WeightKind = WeightKind.DESIGN, + household_weights: Sequence[float] | np.ndarray | pd.Series | None = None, mass_log: tuple[MassChangeRecord, ...] = (), ) -> Frame: """Assemble the UK national tables into a validated Frame. - The typed household weights are built from the ``household_weight`` - column, which stays on the table (the frame permits it because typed - weights exist) so the staging H5 keeps its column order on export. The - Frame constructor enforces the structural invariants the shadow carrier - never checked: group ids unique and sorted ascending, membership equality - in both directions, global column uniqueness, weight health, and the UK - invariant that a benunit's members share a household. + The typed household weights are built from ``household_weights`` when + supplied, otherwise from a legacy input ``household_weight`` column. The + stored carrier table never keeps that export column; materialization + regenerates it from the typed vector. The Frame constructor enforces the + structural invariants the shadow carrier never checked: group ids unique + and sorted ascending, membership equality in both directions, global + column uniqueness, weight health, and the UK invariant that a benunit's + members share a household. """ - if "household_weight" not in household.columns: - raise ValueError( - "household must carry a household_weight column; the UK staging " - "artifact exports it as a real column." - ) - # Validate on a stripped copy but store the caller's exact value: the - # carrier must never rewrite payload it merely transports. period = "" if time_period is None else str(time_period) if not period.strip(): raise ValueError("UK national frame time_period must be a non-empty string.") + if household_weights is None and "household_weight" not in household.columns: + raise ValueError( + "uk_national_frame requires household_weights or a household_weight " + "input column to seed the typed household vector." + ) + weight_values = ( + household_weights + if household_weights is not None + else household["household_weight"].to_numpy(dtype="float64") + ) weights = Weights( - values=household["household_weight"].to_numpy(dtype="float64"), + values=np.asarray(weight_values, dtype="float64"), kind=weight_kind, ) _assert_uk_benunit_nesting(person) + carrier_household = household.drop(columns=["household_weight"], errors="ignore") return Frame( - tables={"person": person, "benunit": benunit, "household": household}, + tables={"person": person, "benunit": benunit, "household": carrier_household}, schema=UK_NATIONAL_SCHEMA, weights={"household": weights}, mass_log=mass_log, @@ -238,10 +237,9 @@ def validate_uk_national_frame(frame: Frame) -> None: invariant via :meth:`Frame.revalidate`, then checks what only the UK contract knows — the exact export schema (person/benunit/household, household-only typed weights, no links), the time-period metadata, - agreement between the persisted ``household_weight`` column and the - typed vector, the UK invariant that a benunit's members share a - household, and agreement between the weight total and the latest - household :class:`MassChangeRecord`. + the UK invariant that a benunit's members share a household, and + agreement between the weight total and the latest household + :class:`MassChangeRecord`. """ if not isinstance(frame, Frame): @@ -268,17 +266,11 @@ def validate_uk_national_frame(frame: Frame) -> None: uk_time_period(frame) weights = frame.weights_for("household") household = frame.table("household") - if "household_weight" not in household.columns: - raise ValueError( - "UK national frame household table must carry the exported " - "household_weight column." - ) - column = household["household_weight"].to_numpy(dtype="float64") - if not np.array_equal(column, weights.values): + reserved = sorted(UK_EXPORTED_WEIGHT_COLUMNS & set(household.columns)) + if reserved: raise ValueError( - "household.household_weight column disagrees with the frame's " - "typed weights; a stage that replaced weights must refresh the " - "exported column." + "UK national Frame carrier must not persist exported weight " + f"column(s): {reserved}; use typed weights instead." ) household_records = [ record for record in frame.mass_log if record.entity == "household" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 9b7b4d04..e6e9f13b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -81,7 +81,6 @@ def _required_household_columns() -> tuple[str, ...]: return ( "household_id", - "household_weight", HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, *SPI_REPLACEMENT_STRATA_COLUMNS, ) @@ -349,25 +348,6 @@ def sample_uk_national_frame( sampled, factor = normalize_sampled_household_mass( sampled, target_mass=full_mass, source_name="UK national" ) - # The typed weights are authoritative; refresh the exported column in - # place so its position — and therefore the staging payload's column - # order — is preserved. The assignment relies on Frame.table returning - # the stored table, so verify it took rather than trust the invariant - # from two modules away (validate_uk_national_frame would also catch - # a stale column, but the failure should name its cause here). - sampled.table("household")["household_weight"] = sampled.weights_for( - "household" - ).values - refreshed = sampled.table("household")["household_weight"].to_numpy( - dtype="float64" - ) - if not np.array_equal(refreshed, sampled.weights_for("household").values): - raise ValueError( - "UK sample: the in-place household_weight refresh did not " - "persist; Frame.table stopped returning live table " - "references, so the exported column would keep its " - "pre-normalization values." - ) receipt["normalization_factor"] = factor receipt["normalized_household_mass"] = float( sampled.weights_for("household").total diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index aaee5029..72027c63 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -570,7 +570,7 @@ def _attach_source_lineage( def clone_uk_dataset_with_rowwise_geography( - dataset: Any | str | Path, + dataset: Frame | str | Path, crosswalk: pd.DataFrame, *, output_path: str | Path | None = None, @@ -588,16 +588,22 @@ def clone_uk_dataset_with_rowwise_geography( avoid_constituency_collisions: bool = True, source_lineage_modulus: int | None = None, ) -> UKRowwiseDatasetResult: - """Clone a UK single-year dataset object or H5 path with row-wise geography. - - The input's weight kind and mass log are carried, never overridden. An H5 - supplies them via the national metadata attrs; an attr-less H5 defaults to - ``WeightKind.DESIGN`` and an empty log, exactly as the national loader - reads it. An in-memory dataset object must declare - ``household_weight_kind``; its absent ``mass_log`` defaults to an empty - history. + """Clone a UK national frame or H5 path with row-wise geography. + + The input's weight kind and mass log are carried, never overridden. A + frame supplies them through its typed weights and mass log; an H5 supplies + them via the national metadata attrs, with an attr-less H5 retaining the + loader's documented ``WeightKind.DESIGN`` and empty-log semantics. The + duck-typed in-memory carrier retired with the #612 Frame migration — an + in-memory input must be a microcosm ``Frame``. """ + if not isinstance(dataset, Frame | str | Path): + raise TypeError( + "rowwise clone requires a microcosm Frame or a UK single-year H5 " + f"path, got {type(dataset).__name__}; the duck-typed in-memory " + "carrier retired with the #612 Frame migration." + ) tables = _dataset_tables(dataset, source_year=source_year) result = clone_uk_dataset_tables_with_rowwise_geography( person=tables["person"], @@ -772,7 +778,7 @@ def read_uk_single_year_weight_metadata( def _dataset_tables( - dataset: Any | str | Path, + dataset: Frame | str | Path, *, source_year: int | None, ) -> dict[str, Any]: @@ -794,37 +800,11 @@ def _dataset_tables( "household_weight_kind": uk_household_weight_kind(dataset), "mass_log": dataset.mass_log, } - missing = [ - name - for name in ("person", "benunit", "household") - if not hasattr(dataset, name) - ] - if missing: - raise ValueError(f"dataset is missing table attribute(s): {missing}.") - time_period = getattr(dataset, "time_period", None) - if not hasattr(dataset, "household_weight_kind"): - raise TypeError( - "dataset.household_weight_kind is required on in-memory datasets; " - "defaulting an absent kind would silently downgrade importance or " - "calibrated weights to design. Declare the kind explicitly " - "(H5 paths keep their documented attribute-less design default)." - ) - weight_kind = dataset.household_weight_kind - mass_log = getattr(dataset, "mass_log", ()) - if mass_log is None: - raise TypeError( - "dataset.mass_log must be a tuple of MassChangeRecord, not None; " - "omit the attribute entirely for an empty history." - ) - mass_log = tuple(mass_log) - return { - "person": dataset.person.copy(), - "benunit": dataset.benunit.copy(), - "household": dataset.household.copy(), - "time_period": _normalise_time_period(time_period, source_year=source_year), - "household_weight_kind": weight_kind, - "mass_log": mass_log, - } + raise TypeError( + "UK dataset table extraction requires a microcosm Frame or a UK " + f"single-year H5 path, got {type(dataset).__name__}; the duck-typed " + "in-memory carrier retired with the #612 Frame migration." + ) def _read_uk_single_year_h5(path: str | Path) -> dict[str, Any]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index f24b26f6..45755cd6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -11,45 +11,27 @@ from __future__ import annotations -import base64 -import binascii import functools -import hashlib -import hmac -import json import math -import os -import uuid -from collections.abc import Callable, Iterable, Mapping, Sequence -from dataclasses import dataclass, field +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import date -from pathlib import Path from types import MappingProxyType from typing import Any import numpy as np import pandas as pd -from microcosm.build.gate_battery import _evaluate_gate from microcosm.build.gates import ( - FitWeightRecord, - GateReport, GateResult, export_surface_gate, - weights_audit_gate, ) from microcosm.build.gates import ( target_surface_gate as _target_surface_gate, ) from microcosm.build.uk_runtime.diagnostics import uk_weight_summary -from microcosm.build.uk_runtime.national_frame import _uk_gate_surface -from microcosm.build.uk_runtime.release_input_coverage import ( - uk_release_input_coverage_gate, -) from microcosm.build.uk_runtime.weighted_integrity import ( UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE, - UK_INPUT_MASS_PARITY_GATE_NAME, - UK_QRF_TAIL_CONCENTRATION_GATE_NAME, UKInputMassParityPolicy, UKInputMassReference, UKQRFTailConcentrationPolicy, @@ -60,11 +42,8 @@ exclusion_evaluation_date, load_uk_reviewed_exclusion_register, uk_input_mass_parity_gate, - uk_input_mass_totals, - uk_qrf_tail_concentration_columns, uk_qrf_tail_concentration_gate, ) -from microcosm.frame import Frame __all__ = [ "UK_ALLOWED_EXTRA_EXPORT_COLUMNS", @@ -72,19 +51,11 @@ "UK_DEFAULT_ZERO_WEIGHT_STRATA", "UK_KNOWN_MISSING_REFERENCE_EXPORT_COLUMNS", "UK_MAX_TARGET_ABS_RELATIVE_ERROR", - "UK_MAX_TO_MEDIAN_WEIGHT_RATIO", - "UK_MIN_ESS_FRACTION", - "UK_TERMINAL_GATE_ATTESTATION_SCHEMA_VERSION", - "UK_TERMINAL_GATE_PRODUCER", - "UK_TERMINAL_GATE_SIGNATURE_ALGORITHM", - "UK_TERMINAL_GATE_SIGNING_KEY_ENV", "UK_REFERENCE_DATASET_NAME", "UK_REVIEWED_EXPORT_EXCLUSIONS", - "UK_TERMINAL_GATE_SCHEMA_VERSION", "UKInputMassParityPolicy", "UKInputMassReference", "UKQRFTailConcentrationPolicy", - "UKReleaseParityEvidence", "UKZeroWeightStratumDeclaration", "uk_default_degenerate_reviewed_exclusions", "uk_degenerate_release_surface_gate", @@ -93,120 +64,15 @@ "uk_qrf_tail_concentration_gate", "uk_target_fit_gate", "uk_target_surface_gate", - "uk_terminal_gate_policy_sha256", - "uk_terminal_gate_report", "uk_weight_ess_gate", "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", - "write_uk_terminal_gate_report", ] -UK_TERMINAL_GATE_SCHEMA_VERSION = 3 -UK_TERMINAL_GATE_ATTESTATION_SCHEMA_VERSION = 5 -UK_TERMINAL_GATE_PRODUCER = ( - "microcosm.build.uk_runtime.terminal_gates.uk_terminal_gate_report" -) -UK_TERMINAL_GATE_SIGNATURE_ALGORITHM = "hmac-sha256" -UK_TERMINAL_GATE_SIGNING_KEY_ENV = "POPULACE_UK_TERMINAL_GATE_SIGNING_KEY" UK_CANDIDATE_DATASET_NAME = "populace_uk_2023" UK_REFERENCE_DATASET_NAME = "enhanced_frs_2023_24_recalibrated" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 -# The certified June artifact is at 0.039953 ESS fraction. Its measured -# max/positive-median ratio is exactly 1,151.2542195939373 (maximum weight -# 18,652.802734375 / positive median 16.202157974243164). Under the #578 -# acceptance rule, a candidate must not regress the incumbent on battery -# observables: this boundary has no discretionary headroom. Raising it -# requires an explicit future adjudication. These are acceptance fences, not -# solver knobs. -UK_MIN_ESS_FRACTION = 0.01 -UK_MAX_TO_MEDIAN_WEIGHT_RATIO = 1_151.2542195939373 - -_UK_ALWAYS_APPLICABLE_GATE_NAMES = ( - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", -) -_UK_HMRC_GATE_NAMES = ("weights_audit",) -_UK_PARITY_GATE_NAMES = ("export_surface", "target_surface", "target_fit") -_UK_INPUT_MASS_GATE_NAMES = (UK_INPUT_MASS_PARITY_GATE_NAME,) -_UK_QRF_TAIL_GATE_NAMES = (UK_QRF_TAIL_CONCENTRATION_GATE_NAME,) -_UK_WEIGHT_SUMMARY_FIELDS = ( - "n_records", - "positive_weight_records", - "zero_weight_records", - "total_weight", - "effective_sample_size", - "ess_fraction", - "median_positive_weight", - "max_weight", - "max_to_median_positive_weight", - "top_1pct_weight_share", -) - - -def _canonical_sha256(value: object) -> str: - return hashlib.sha256(_canonical_json_bytes(value)).hexdigest() - - -def _canonical_json_bytes(value: object) -> bytes: - return json.dumps( - value, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - - -def _terminal_gate_signing_key() -> bytes: - """Read the 256-bit release-attestation key from the build environment.""" - - encoded = os.environ.get(UK_TERMINAL_GATE_SIGNING_KEY_ENV) - if not encoded: - raise RuntimeError( - f"{UK_TERMINAL_GATE_SIGNING_KEY_ENV} must contain a base64-encoded " - "32-byte key before writing a UK terminal gate report." - ) - try: - key = base64.b64decode(encoded, validate=True) - except (binascii.Error, ValueError) as exc: - raise RuntimeError( - f"{UK_TERMINAL_GATE_SIGNING_KEY_ENV} must be valid base64." - ) from exc - if len(key) != 32: - raise RuntimeError( - f"{UK_TERMINAL_GATE_SIGNING_KEY_ENV} must decode to exactly 32 bytes." - ) - return key - - -def _terminal_gate_signature(key: bytes, payload: object) -> str: - return hmac.new(key, _canonical_json_bytes(payload), hashlib.sha256).hexdigest() - - -def _validate_attested_release_identity( - release_id: object, - calibration_diagnostics_sha256: object, -) -> tuple[str, str]: - """Validate the external release coordinates covered by the signature.""" - - if not isinstance(release_id, str) or not release_id.strip(): - raise ValueError("UK terminal release_id must be a non-empty string.") - if ( - not isinstance(calibration_diagnostics_sha256, str) - or len(calibration_diagnostics_sha256) != 64 - or any( - character not in "0123456789abcdef" - for character in calibration_diagnostics_sha256 - ) - ): - raise ValueError( - "UK terminal calibration_diagnostics_sha256 must be a lowercase sha256." - ) - return release_id, calibration_diagnostics_sha256 - _SPI_FLAG = "household_is_spi_synthetic" _CAPITAL_GAINS_FLAG = "household_is_capital_gains_clone" @@ -356,105 +222,9 @@ def __post_init__(self) -> None: } -def _weighted_integrity_policy_payload(policy: object) -> object: - """Project an armed weighted-integrity policy into the sealed payload. - - ``None`` records that the gate is unarmed; an unexpected type is recorded - (not raised) so the policy digest still seals what the caller supplied - while the evaluator fails closed with the named type error. - """ - - if policy is None: - return None - if isinstance(policy, (UKInputMassParityPolicy, UKQRFTailConcentrationPolicy)): - return policy.policy_payload() - return {"invalid_type": f"{type(policy).__module__}.{type(policy).__qualname__}"} - - -def _terminal_gate_policy_payload( - *, - builtin_coverage_evaluator: bool, - reviewed_degenerate_exclusions: object, - zero_weight_declarations: Sequence[object], - minimum_ess_fraction: object, - maximum_max_to_median_ratio: object, - input_mass_policy: object = None, - qrf_tail_policy: object = None, -) -> dict[str, object]: - declarations: list[dict[str, object]] = [] - for declaration in zero_weight_declarations: - if isinstance(declaration, UKZeroWeightStratumDeclaration): - declarations.append( - { - "name": declaration.name, - "selector": dict(declaration.selector), - "maximum_zero_weight_rows": declaration.maximum_zero_weight_rows, - "reason": declaration.reason, - } - ) - else: - declarations.append( - { - "invalid_type": ( - f"{type(declaration).__module__}." - f"{type(declaration).__qualname__}" - ) - } - ) - try: - ess = float(minimum_ess_fraction) - except (TypeError, ValueError): - ess = repr(minimum_ess_fraction) - try: - ratio = float(maximum_max_to_median_ratio) - except (TypeError, ValueError): - ratio = repr(maximum_max_to_median_ratio) - return { - "coverage_evaluator": "builtin" if builtin_coverage_evaluator else "injected", - "reviewed_degenerate_exclusions": ( - {} - if reviewed_degenerate_exclusions is None - else { - name: record.policy_payload() - for name, record in sorted( - coerce_reviewed_exclusions( - reviewed_degenerate_exclusions, - label="UK degenerate-surface policy", - ).items() - ) - } - ), - "zero_weight_declarations": declarations, - "minimum_ess_fraction": ess, - "maximum_max_to_median_ratio": ratio, - "maximum_target_abs_relative_error": UK_MAX_TARGET_ABS_RELATIVE_ERROR, - "allowed_extra_export_columns": list(UK_ALLOWED_EXTRA_EXPORT_COLUMNS), - "known_missing_reference_export_columns": list( - UK_KNOWN_MISSING_REFERENCE_EXPORT_COLUMNS - ), - "reviewed_export_exclusions": dict( - sorted(UK_REVIEWED_EXPORT_EXCLUSIONS.items()) - ), - "input_mass_parity": _weighted_integrity_policy_payload(input_mass_policy), - "qrf_tail_concentration": _weighted_integrity_policy_payload(qrf_tail_policy), - } - - @functools.cache def uk_default_degenerate_reviewed_exclusions() -> Mapping[str, UKReviewedExclusion]: - """The committed degenerate-surface register (#630) — the policy of record. - - A ``None`` argument to :func:`uk_terminal_gate_report` resolves to this - register, and the frozen policy digest is computed over it, so deleting - or editing an entry moves the pinned literal (the intended tripwire). - Pass ``{}`` explicitly to run with no exclusions. - - Loaded lazily so importing this module never reads the filesystem — a - missing or malformed committed register surfaces as this call's clear - ``ValueError``, not an ``ImportError`` — cached so every caller seals - the same load, and wrapped read-only so the policy of record cannot be - mutated out from under the already-computed digest. - """ + """The committed degenerate-surface register (#630), loaded lazily.""" return MappingProxyType( load_uk_reviewed_exclusion_register( @@ -463,401 +233,6 @@ def uk_default_degenerate_reviewed_exclusions() -> Mapping[str, UKReviewedExclus ) -@functools.cache -def uk_terminal_gate_policy_sha256() -> str: - """Frozen digest of the default terminal-gate policy, exclusions sealed. - - Derived from the committed register, so it shares the lazy accessor's - contract: no import-time file I/O, one cached value per process. - """ - - return _canonical_sha256( - _terminal_gate_policy_payload( - builtin_coverage_evaluator=True, - reviewed_degenerate_exclusions=uk_default_degenerate_reviewed_exclusions(), - zero_weight_declarations=UK_DEFAULT_ZERO_WEIGHT_STRATA, - minimum_ess_fraction=UK_MIN_ESS_FRACTION, - maximum_max_to_median_ratio=UK_MAX_TO_MEDIAN_WEIGHT_RATIO, - ) - ) - - -@dataclass(frozen=True) -class UKReleaseParityEvidence: - """Complete real evidence needed to run the June parity-gate trio.""" - - candidate_columns: Iterable[str] - reference_columns: Iterable[str] - candidate_targets: Iterable[str] - reference_targets: Iterable[str] - target_relative_errors: Mapping[str, float] - - def __post_init__(self) -> None: - for field_name in ( - "candidate_columns", - "reference_columns", - "candidate_targets", - "reference_targets", - ): - raw = getattr(self, field_name) - materialized = tuple(sorted({str(value) for value in raw})) - if not materialized or any(not value for value in materialized): - raise ValueError(f"UK parity evidence {field_name} must be non-empty.") - object.__setattr__(self, field_name, materialized) - errors = { - str(name): float(error) - for name, error in self.target_relative_errors.items() - } - if not errors or any(not math.isfinite(error) for error in errors.values()): - raise ValueError( - "UK parity evidence target_relative_errors must be non-empty and " - "finite." - ) - if set(errors) != set(self.candidate_targets): - raise ValueError( - "UK parity evidence target_relative_errors must exactly cover " - "candidate_targets." - ) - object.__setattr__(self, "target_relative_errors", dict(sorted(errors.items()))) - - -def _gate_results_payload(results: Sequence[GateResult]) -> dict[str, object]: - return { - result.name: { - "passed": result.passed, - "failures": list(result.failures), - "details": dict(result.details), - } - for result in results - } - - -def _unsigned_terminal_gate_attestation( - results: Sequence[GateResult], - *, - release_id: str, - calibration_diagnostics_sha256: str, - policy_sha256: str, - evidence_sha256: Mapping[str, str], -) -> dict[str, object]: - """Return the provenance fields covered by the release signature.""" - - gates = _gate_results_payload(results) - return { - "schema_version": UK_TERMINAL_GATE_ATTESTATION_SCHEMA_VERSION, - "producer": UK_TERMINAL_GATE_PRODUCER, - "release_id": release_id, - "calibration_diagnostics_sha256": calibration_diagnostics_sha256, - "policy_sha256": policy_sha256, - "evaluated_gates": [result.name for result in results], - "evidence_sha256": dict(evidence_sha256), - "gate_results_sha256": _canonical_sha256(gates), - } - - -def _terminal_gate_report_payload( - results: Sequence[GateResult], - attestation: Mapping[str, object], - *, - signature_available: bool, -) -> dict[str, object]: - return { - "schema_version": UK_TERMINAL_GATE_SCHEMA_VERSION, - "enforced": True, - "passed": all(result.passed for result in results) and signature_available, - "gates": _gate_results_payload(results), - "attestation": dict(attestation), - } - - -def _release_dataset_evidence_payload( - results: Sequence[GateResult], -) -> dict[str, object]: - """Bind the attestation to the evaluated release's weight observables. - - The ratio gate receives the final shipped household-weight vector and - records the same ten-field summary written to ``uk_diagnostics.weights``. - Projecting that summary gives the publication contract an independently - reconstructible release-data digest. Erroring gates deliberately project - missing fields as null so a failed terminal report can still be written. - """ - - ratio = next((result for result in results if result.name == "weight_ratio"), None) - details = ratio.details if ratio is not None else {} - return { - "weights": { - field: _json_scalar(details.get(field)) - for field in _UK_WEIGHT_SUMMARY_FIELDS - } - } - - -@dataclass(frozen=True) -class _AttestedUKTerminalGateReport(GateReport): - """Aggregator-signed report sealed against post-evaluation mutation.""" - - release_id: str - calibration_diagnostics_sha256: str - policy_sha256: str - evidence_sha256: Mapping[str, str] - attestation: Mapping[str, object] - _signing_error: RuntimeError | None = field(repr=False, compare=False) - _sealed_sha256: str = field(init=False, repr=False, compare=False) - - def __post_init__(self) -> None: - release_id, calibration_diagnostics_sha256 = ( - _validate_attested_release_identity( - self.release_id, - self.calibration_diagnostics_sha256, - ) - ) - names = tuple(result.name for result in self.results) - if len(names) != len(set(names)): - raise ValueError("Attested UK terminal gate names must be unique.") - if len(self.policy_sha256) != 64 or any( - character not in "0123456789abcdef" for character in self.policy_sha256 - ): - raise ValueError("UK terminal policy digest must be a lowercase sha256.") - evidence = dict(sorted(self.evidence_sha256.items())) - if not evidence or any( - not isinstance(name, str) - or not isinstance(digest, str) - or len(digest) != 64 - or any(character not in "0123456789abcdef" for character in digest) - for name, digest in evidence.items() - ): - raise ValueError("UK terminal evidence digests must be named sha256s.") - evidence_names = set(evidence) - if "release_dataset" not in evidence_names: - raise ValueError( - "UK terminal evidence must include the release_dataset digest." - ) - unknown_evidence = sorted( - evidence_names - - { - "release_dataset", - "hmrc_spi_income", - "release_parity", - "input_mass_parity", - "qrf_tail_concentration", - } - ) - if unknown_evidence: - raise ValueError( - f"UK terminal evidence has unknown stages: {unknown_evidence}." - ) - expected_names = list(_UK_ALWAYS_APPLICABLE_GATE_NAMES) - if "hmrc_spi_income" in evidence_names: - expected_names.extend(_UK_HMRC_GATE_NAMES) - if "release_parity" in evidence_names: - expected_names.extend(_UK_PARITY_GATE_NAMES) - if "input_mass_parity" in evidence_names: - expected_names.extend(_UK_INPUT_MASS_GATE_NAMES) - if "qrf_tail_concentration" in evidence_names: - expected_names.extend(_UK_QRF_TAIL_GATE_NAMES) - if names != tuple(expected_names): - raise ValueError( - "UK terminal gate membership must follow the attested evidence " - f"stages; expected {expected_names}, got {list(names)}." - ) - object.__setattr__(self, "evidence_sha256", MappingProxyType(evidence)) - expected_unsigned = _unsigned_terminal_gate_attestation( - self.results, - release_id=release_id, - calibration_diagnostics_sha256=calibration_diagnostics_sha256, - policy_sha256=self.policy_sha256, - evidence_sha256=evidence, - ) - attestation = dict(self.attestation) - expected_fields = { - *expected_unsigned, - "signature_algorithm", - "signing_key_sha256", - "signature", - } - if set(attestation) != expected_fields: - raise ValueError( - "UK terminal attestation must contain the complete signed schema." - ) - if any( - attestation.get(name) != value for name, value in expected_unsigned.items() - ): - raise ValueError( - "UK terminal attestation must bind the evaluated gates and evidence." - ) - if ( - attestation.get("signature_algorithm") - != UK_TERMINAL_GATE_SIGNATURE_ALGORITHM - ): - raise ValueError("UK terminal attestation signature algorithm is invalid.") - signature_values = ( - attestation.get("signing_key_sha256"), - attestation.get("signature"), - ) - if self._signing_error is None: - if any( - not isinstance(value, str) - or len(value) != 64 - or any(character not in "0123456789abcdef" for character in value) - for value in signature_values - ): - raise ValueError( - "Signed UK terminal attestations require lowercase sha256 values." - ) - elif signature_values != (None, None): - raise ValueError( - "A failed UK terminal signature must not claim signature values." - ) - object.__setattr__(self, "attestation", MappingProxyType(attestation)) - object.__setattr__(self, "_sealed_sha256", self._current_attestation_sha256()) - - @property - def evaluated_gates(self) -> tuple[str, ...]: - return tuple(result.name for result in self.results) - - @property - def passed(self) -> bool: - """True iff the sealed terminal report is publishable.""" - - return bool(self.report_payload()["passed"]) - - def _current_attestation_sha256(self) -> str: - return _canonical_sha256( - _terminal_gate_report_payload( - self.results, - self.attestation, - signature_available=self._signing_error is None, - ) - ) - - def report_payload(self) -> dict[str, object]: - """Return the sealed aggregator output without granting signing power.""" - - if self._current_attestation_sha256() != self._sealed_sha256: - raise ValueError("UK terminal gate attestation changed after evaluation.") - return _terminal_gate_report_payload( - self.results, - self.attestation, - signature_available=self._signing_error is None, - ) - - -def _fit_evidence_payload( - records: tuple[object, ...] | None, - *, - required: bool, - materialization_error: Exception | None, -) -> dict[str, object]: - payload: dict[str, object] = {"required": required} - if materialization_error is not None: - payload["materialization_error"] = { - "type": type(materialization_error).__name__, - "message": str(materialization_error), - } - return payload - payload["fit_weight_records"] = [ - ( - {"fit_name": record.fit_name, "weight_kind": record.weight_kind} - if isinstance(record, FitWeightRecord) - else { - "invalid_type": ( - f"{type(record).__module__}.{type(record).__qualname__}" - ) - } - ) - for record in (records or ()) - ] - return payload - - -def _parity_evidence_payload(evidence: object) -> dict[str, object]: - if not isinstance(evidence, UKReleaseParityEvidence): - return { - "invalid_type": f"{type(evidence).__module__}.{type(evidence).__qualname__}" - } - return { - "candidate_columns": list(evidence.candidate_columns), - "reference_columns": list(evidence.reference_columns), - "candidate_targets": list(evidence.candidate_targets), - "reference_targets": list(evidence.reference_targets), - "target_relative_errors": dict(evidence.target_relative_errors), - } - - -def _input_mass_evidence_payload(reference: object) -> dict[str, object]: - """Bind the attestation to the frozen input-mass reference. - - An armed gate with no reference still produces a digest — recording the - absence the evaluator fails closed on — so the report cannot silently - drop the stage. - """ - - if reference is None: - return {"reference": None} - if not isinstance(reference, UKInputMassReference): - return { - "invalid_type": ( - f"{type(reference).__module__}.{type(reference).__qualname__}" - ) - } - return { - "reference": { - "identity": dict(reference.identity), - "totals": {name: float(total) for name, total in reference.totals.items()}, - } - } - - -def _json_tree(value: object) -> object: - if isinstance(value, Mapping): - return { - str(key): _json_tree(item) - for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) - } - if isinstance(value, (list, tuple)): - return [_json_tree(item) for item in value] - return _json_scalar(value) - - -_UK_QRF_TAIL_EVIDENCE_FIELDS = ( - "columns_checked", - "top_k", - "max_top_share", - "min_nonzero_records", - "top_share", - "carrier_counts", - "thin_columns", - "surface", -) - - -def _qrf_tail_evidence_payload(results: Sequence[GateResult]) -> dict[str, object]: - """Bind the attestation to the evaluated QRF tail observables. - - Like ``_release_dataset_evidence_payload``, this projects the gate's own - recorded observables (per-column top shares, carrier counts, and the - manifest-derived surface) rather than hashing raw column arrays, giving - consumers an independently reconstructible digest. Erroring gates - project missing fields as null so a failed report can still be written. - """ - - gate = next( - ( - result - for result in results - if result.name == UK_QRF_TAIL_CONCENTRATION_GATE_NAME - ), - None, - ) - details = gate.details if gate is not None else {} - return { - "tail_concentration": { - field: _json_tree(details.get(field)) - for field in _UK_QRF_TAIL_EVIDENCE_FIELDS - } - } - - def _entity_tables(dataset: Any) -> tuple[tuple[str, pd.DataFrame], ...]: if isinstance(dataset, Mapping): raw = tuple((entity, dataset.get(entity)) for entity in _STRUCTURAL_COLUMNS) @@ -1182,7 +557,7 @@ def uk_zero_weight_strata_gate( def uk_weight_ess_gate( weights: Sequence[float] | np.ndarray, *, - minimum_ess_fraction: float = UK_MIN_ESS_FRACTION, + minimum_ess_fraction: float, ) -> GateResult: """Require the shipped household weights to retain effective support.""" @@ -1208,7 +583,7 @@ def uk_weight_ess_gate( def uk_weight_ratio_gate( weights: Sequence[float] | np.ndarray, *, - maximum_max_to_median_ratio: float = UK_MAX_TO_MEDIAN_WEIGHT_RATIO, + maximum_max_to_median_ratio: float, ) -> GateResult: """Backstop a shipped-weight max/positive-median concentration blowout.""" @@ -1379,300 +754,3 @@ def _missing_fit_weight_evidence_gate() -> GateResult: ), details={"fits_checked": 0, "evidence_missing": True}, ) - - -def uk_terminal_gate_report( - frame: Frame, - coverage_engine: Any, - *, - release_id: str, - calibration_diagnostics_sha256: str, - input_coverage_evaluator: Callable[[], GateResult] | None = None, - reviewed_degenerate_exclusions: (Mapping[str, UKReviewedExclusion] | None) = None, - zero_weight_declarations: Sequence[UKZeroWeightStratumDeclaration] = ( - UK_DEFAULT_ZERO_WEIGHT_STRATA - ), - minimum_ess_fraction: float = UK_MIN_ESS_FRACTION, - maximum_max_to_median_ratio: float = UK_MAX_TO_MEDIAN_WEIGHT_RATIO, - fit_weight_records: Iterable[FitWeightRecord] | None = None, - require_fit_weight_records: bool = False, - parity_evidence: UKReleaseParityEvidence | None = None, - input_mass_reference: UKInputMassReference | None = None, - input_mass_policy: UKInputMassParityPolicy | None = None, - qrf_tail_policy: UKQRFTailConcentrationPolicy | None = None, - now: date | None = None, -) -> GateReport: - """Evaluate every evidenced UK terminal gate and seal its provenance. - - ``now`` (default: today, UTC) is the date reviewed-exclusion expiry is - evaluated against; tests inject fixed dates. - """ - - release_id, calibration_diagnostics_sha256 = _validate_attested_release_identity( - release_id, - calibration_diagnostics_sha256, - ) - # The legacy gate modules read the duck-attr evidence surface, and the - # coverage gate in particular reads its metadata attributes via - # getattr-with-default — handing it the raw Frame would silently degrade - # the checks, so the surface is built once here for every consumer. - surface = _uk_gate_surface(frame) - builtin_coverage_evaluator = input_coverage_evaluator is None - coverage = input_coverage_evaluator or ( - lambda: uk_release_input_coverage_gate(surface, coverage_engine) - ) - # None resolves to the committed register — the reviewed policy of - # record (#630); an explicit {} runs with no exclusions. The mapping is - # coerced and frozen ONCE here, so the gate and the sealed policy digest - # cannot observe different contents when a caller mutates its argument - # between the two reads (the attestation must describe the policy the - # gate actually ran under). - if reviewed_degenerate_exclusions is None: - reviewed_degenerate_exclusions = uk_default_degenerate_reviewed_exclusions() - reviewed_degenerate_exclusions = MappingProxyType( - coerce_reviewed_exclusions( - reviewed_degenerate_exclusions, label="UK degenerate-surface policy" - ) - ) - evaluation_date = exclusion_evaluation_date(now) - fit_stage_present = fit_weight_records is not None or require_fit_weight_records - materialized_fit_records: tuple[object, ...] | None = None - fit_materialization_error: Exception | None = None - if fit_weight_records is not None: - try: - materialized_fit_records = tuple(fit_weight_records) - except Exception as exc: # noqa: BLE001 - gate records the failed evidence - materialized_fit_records = () - fit_materialization_error = exc - evaluators: list[tuple[str, Callable[[], GateResult]]] = [ - ("uk_release_input_coverage", coverage), - ( - "degenerate_release_surface", - lambda: uk_degenerate_release_surface_gate( - surface, - reviewed_exclusions=reviewed_degenerate_exclusions, - now=evaluation_date, - ), - ), - ( - "zero_weight_strata", - lambda: uk_zero_weight_strata_gate( - surface.household, - declarations=zero_weight_declarations, - ), - ), - ( - "weight_ess", - lambda: uk_weight_ess_gate( - _household_weights(surface.household), - minimum_ess_fraction=minimum_ess_fraction, - ), - ), - ( - "weight_ratio", - lambda: uk_weight_ratio_gate( - _household_weights(surface.household), - maximum_max_to_median_ratio=maximum_max_to_median_ratio, - ), - ), - ] - - if fit_stage_present: - - def fit_weight_evaluator() -> GateResult: - if fit_materialization_error is not None: - raise fit_materialization_error - if fit_weight_records is None: - return _missing_fit_weight_evidence_gate() - if not materialized_fit_records: - return _missing_fit_weight_evidence_gate() - return weights_audit_gate(materialized_fit_records) - - evaluators.append( - ( - "weights_audit", - fit_weight_evaluator, - ) - ) - - if parity_evidence is not None: - - def checked_parity_evidence() -> UKReleaseParityEvidence: - if not isinstance(parity_evidence, UKReleaseParityEvidence): - raise TypeError("parity_evidence must be UKReleaseParityEvidence.") - return parity_evidence - - evaluators.extend( - ( - ( - "export_surface", - lambda: uk_export_surface_gate( - checked_parity_evidence().candidate_columns, - checked_parity_evidence().reference_columns, - ), - ), - ( - "target_surface", - lambda: uk_target_surface_gate( - checked_parity_evidence().candidate_targets, - checked_parity_evidence().reference_targets, - ), - ), - ( - "target_fit", - lambda: uk_target_fit_gate( - checked_parity_evidence().target_relative_errors, - ), - ), - ) - ) - - input_mass_armed = input_mass_reference is not None or input_mass_policy is not None - if input_mass_armed: - - def input_mass_evaluator() -> GateResult: - if input_mass_reference is None: - raise ValueError( - "input_mass_parity is armed but no UKInputMassReference " - "was supplied; a missing frozen reference is not a " - "passing gate." - ) - if input_mass_policy is None: - raise ValueError( - "input_mass_parity is armed without reviewed thresholds; " - "supply a UKInputMassParityPolicy measured per the #609 " - "measurement pass." - ) - return uk_input_mass_parity_gate( - uk_input_mass_totals(frame), - input_mass_reference, - policy=input_mass_policy, - now=evaluation_date, - ) - - evaluators.append((UK_INPUT_MASS_PARITY_GATE_NAME, input_mass_evaluator)) - - if qrf_tail_policy is not None: - - def qrf_tail_evaluator() -> GateResult: - if not isinstance(qrf_tail_policy, UKQRFTailConcentrationPolicy): - raise TypeError("qrf_tail_policy must be UKQRFTailConcentrationPolicy.") - values, weights, qrf_surface = uk_qrf_tail_concentration_columns(frame) - return uk_qrf_tail_concentration_gate( - values, - weights, - policy=qrf_tail_policy, - surface=qrf_surface, - now=evaluation_date, - ) - - evaluators.append((UK_QRF_TAIL_CONCENTRATION_GATE_NAME, qrf_tail_evaluator)) - - names = [name for name, _evaluator in evaluators] - duplicates = sorted({name for name in names if names.count(name) > 1}) - if duplicates: - raise ValueError(f"UK terminal gate names must be unique: {duplicates}.") - results = tuple(_evaluate_gate(name, evaluator) for name, evaluator in evaluators) - evidence_sha256 = { - "release_dataset": _canonical_sha256(_release_dataset_evidence_payload(results)) - } - if fit_stage_present: - evidence_sha256["hmrc_spi_income"] = _canonical_sha256( - _fit_evidence_payload( - materialized_fit_records, - required=require_fit_weight_records, - materialization_error=fit_materialization_error, - ) - ) - if parity_evidence is not None: - evidence_sha256["release_parity"] = _canonical_sha256( - _parity_evidence_payload(parity_evidence) - ) - if input_mass_armed: - evidence_sha256["input_mass_parity"] = _canonical_sha256( - _input_mass_evidence_payload(input_mass_reference) - ) - if qrf_tail_policy is not None: - evidence_sha256["qrf_tail_concentration"] = _canonical_sha256( - _qrf_tail_evidence_payload(results) - ) - policy_sha256 = _canonical_sha256( - _terminal_gate_policy_payload( - builtin_coverage_evaluator=builtin_coverage_evaluator, - reviewed_degenerate_exclusions=reviewed_degenerate_exclusions, - zero_weight_declarations=zero_weight_declarations, - minimum_ess_fraction=minimum_ess_fraction, - maximum_max_to_median_ratio=maximum_max_to_median_ratio, - input_mass_policy=input_mass_policy, - qrf_tail_policy=qrf_tail_policy, - ) - ) - unsigned_attestation = _unsigned_terminal_gate_attestation( - results, - release_id=release_id, - calibration_diagnostics_sha256=calibration_diagnostics_sha256, - policy_sha256=policy_sha256, - evidence_sha256=evidence_sha256, - ) - signing_error: RuntimeError | None = None - try: - signing_key = _terminal_gate_signing_key() - except RuntimeError as exc: - signing_key = None - signing_error = exc - attestation = { - **unsigned_attestation, - "signature_algorithm": UK_TERMINAL_GATE_SIGNATURE_ALGORITHM, - "signing_key_sha256": ( - hashlib.sha256(signing_key).hexdigest() if signing_key is not None else None - ), - } - if signing_key is None: - attestation["signature"] = None - else: - unsigned_report = _terminal_gate_report_payload( - results, - attestation, - signature_available=True, - ) - attestation["signature"] = _terminal_gate_signature( - signing_key, - unsigned_report, - ) - return _AttestedUKTerminalGateReport( - results, - release_id=release_id, - calibration_diagnostics_sha256=calibration_diagnostics_sha256, - policy_sha256=policy_sha256, - evidence_sha256=evidence_sha256, - attestation=attestation, - _signing_error=signing_error, - ) - - -def write_uk_terminal_gate_report( - report: GateReport, - path: str | Path, -) -> Path: - """Atomically persist an aggregator-signed report without signing input.""" - - if type(report) is not _AttestedUKTerminalGateReport: - raise TypeError( - "UK terminal gate report writer requires the attested report " - "returned by uk_terminal_gate_report()." - ) - output = Path(path) - payload = report.report_payload() - encoded = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n" - output.parent.mkdir(parents=True, exist_ok=True) - temporary = output.with_name(f".{output.name}.{uuid.uuid4().hex}.tmp") - try: - temporary.write_text(encoded, encoding="utf-8") - temporary.replace(output) - finally: - temporary.unlink(missing_ok=True) - if report._signing_error is not None: - raise RuntimeError( - f"{report._signing_error} Unsigned failed report was written to {output}." - ) from report._signing_error - return output diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index 95f57d06..9de10b3b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -23,9 +23,8 @@ Thresholds carry **no committed defaults**: the US numbers are calibrated to US incidents and the #609 measurement pass has not yet adjudicated UK boundaries. Arming either gate requires explicit policy values; once the -measurement numbers are adjudicated, the constants belong next to -:data:`~microcosm.build.uk_runtime.terminal_gates.UK_MAX_TO_MEDIAN_WEIGHT_RATIO` -with the same derivation-comment discipline. +measurement numbers are adjudicated, the schema-4 manifest parameters should +carry the same derivation-comment discipline as the weight-ratio threshold. What the first measurement pass against the pinned enhanced-FRS incumbent (sha ``584ae33d…``, 2026-08-04) established, so that no later reader diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 22485574..57355391 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -290,6 +290,14 @@ def test_declares_the_full_june_battery(self, manifest) -> None: # Legacy behaviour: every evaluated failure raises, so every # declared entry blocks release. assert all(g.criticality == "release_blocking" for g in manifest.gates) + + def test_only_the_weights_audit_blocks_on_absent_evidence(self, manifest) -> None: + # "An absent audit is not a passing audit" — the retired schema-3 + # path blocked every posture on a missing fit-weight audit, and the + # battery keeps that strictness via the entry flag (#654, #691 + # review). No other entry opts out of the dev-posture leniency. + flagged = [g.id for g in manifest.gates if g.evidence_absent_blocks] + assert flagged == ["uk_weights_audit"] assert all(g.not_applicable is None for g in manifest.gates) def test_gate_names_are_country_neutral(self, manifest) -> None: @@ -299,15 +307,12 @@ def test_gate_names_are_country_neutral(self, manifest) -> None: assert by_id["uk_qrf_tail_concentration"] == "tail_concentration" assert not any(name.startswith("uk_") for name in by_id.values()) - def test_thresholds_match_the_legacy_module_constants(self, manifest) -> None: + def test_thresholds_match_the_schema4_manifest(self, manifest) -> None: params = {gate.id: gate.parameters for gate in manifest.gates} - assert ( - params["uk_weight_ess"]["minimum_ess_fraction"] - == terminal_gates.UK_MIN_ESS_FRACTION - ) + assert params["uk_weight_ess"]["minimum_ess_fraction"] == 0.01 assert ( params["uk_weight_ratio"]["maximum_max_to_median_ratio"] - == terminal_gates.UK_MAX_TO_MEDIAN_WEIGHT_RATIO + == 1_151.2542195939373 ) assert ( params["uk_target_fit"]["max_abs_relative_error"] @@ -398,6 +403,25 @@ def test_unknown_gate_function_is_refused(self, tmp_path) -> None: with pytest.raises(ValueError, match="unknown gate function 'vibes'"): load_country_spec(package_dir) + def test_non_bool_evidence_absent_blocks_is_refused(self, tmp_path) -> None: + files = _minimal_package() + files["gates.json"]["gates"][0]["evidence_absent_blocks"] = "yes" + package_dir = _write_package(tmp_path, files) + with pytest.raises(ValueError, match="evidence_absent_blocks must be"): + load_country_spec(package_dir) + + def test_evidence_absent_blocks_on_an_excused_entry_is_refused( + self, tmp_path + ) -> None: + files = _minimal_package() + entry = files["gates.json"]["gates"][0] + entry.pop("parameters", None) + entry["not_applicable"] = "reviewed: no surface yet" + entry["evidence_absent_blocks"] = True + package_dir = _write_package(tmp_path, files) + with pytest.raises(ValueError, match="mutually exclusive"): + load_country_spec(package_dir) + def test_all_diagnostic_gates_are_refused(self, tmp_path) -> None: files = _minimal_package() files["gates.json"]["gates"][0]["criticality"] = "diagnostic" diff --git a/packages/microcosm-build/tests/test_gate_battery.py b/packages/microcosm-build/tests/test_gate_battery.py index 6272442e..b9b791ac 100644 --- a/packages/microcosm-build/tests/test_gate_battery.py +++ b/packages/microcosm-build/tests/test_gate_battery.py @@ -384,6 +384,43 @@ def test_blocks_release_candidates_and_marks_dev_builds( assert report["gates"]["t"]["status"] == "evidence_absent" assert report["shippable"] is False + def test_evidence_absent_blocks_flag_blocks_every_posture( + self, tmp_path, signing_env + ): + # The dev-posture leniency is opt-out per entry: a manifest entry + # declaring evidence_absent_blocks blocks the default build too, + # while its report status stays honestly evidence_absent (the + # legacy UK weights-audit strictness, ported in #654/#691 review). + manifest = _manifest( + [_entry("t", gate="weights_audit", evidence_absent_blocks=True)], + ["terminal"], + ) + run = GateBatteryRun( + manifest, + release_id="xx-test-build", + report_path=tmp_path / "strict_absent.json", + release_candidate=False, + ) + run.run_phase("terminal", EvidenceContext()) + assert run.enforce("terminal", mode=BlockingMode.MARKS_ARTIFACT) is True + report = json.loads(run.report_path.read_text()) + assert report["gates"]["t"]["status"] == "evidence_absent" + assert report["shippable"] is False + # The armed flag rides both digests: an identical manifest without + # the flag hashes differently. + unflagged = _manifest([_entry("t", gate="weights_audit")], ["terminal"]) + baseline = GateBatteryRun( + unflagged, + release_id="xx-test-build", + report_path=tmp_path / "lenient_absent.json", + release_candidate=False, + ) + baseline.run_phase("terminal", EvidenceContext()) + assert baseline.enforce("terminal", mode=BlockingMode.MARKS_ARTIFACT) is False + lenient = json.loads(baseline.report_path.read_text()) + assert lenient["policy_sha256"] != report["policy_sha256"] + assert lenient["gates_manifest_sha256"] != report["gates_manifest_sha256"] + def test_diagnostic_entries_never_block(self, tmp_path, signing_env): manifest = _manifest( [ diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index f8567082..1d0d8c97 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -1,22 +1,15 @@ """The UK consumer half of the gate battery (microcosm#611 increment 1). -The behaviour-preservation contract: ``evaluate_phase`` over ``uk/gates.json`` -with ``UK_GATE_REGISTRY`` must reproduce the legacy ``uk_terminal_gate_report`` -verdicts gate for gate over identical synthetic evidence — same ``passed``, -same failure lines, same details — with exactly two result names re-minted -onto the shared vocabulary. Where the two paths deliberately differ (the -legacy report *omits* unevidenced gates; the battery records them as -``evidence_absent`` and blocks release candidates only), the difference is -asserted here as a positive statement, not papered over. - -Fixtures are synthetic throughout: no UKDS unit records, same discipline as -the legacy battery tests. +Fixtures are synthetic throughout: no UKDS unit records. The schema-3 +aggregator retired in #654; these tests pin the battery-side behavior that +survived the differential receipt. """ from __future__ import annotations import base64 from datetime import date, datetime +from types import SimpleNamespace from unittest.mock import patch import numpy as np @@ -56,25 +49,15 @@ UKInputMassParityPolicy, UKInputMassReference, UKQRFTailConcentrationPolicy, - UKReleaseParityEvidence, - uk_terminal_gate_report, ) from microcosm.frame import engine_tables KEY = base64.b64encode(b"\x07" * 32).decode("ascii") -RELEASE_ID = "populace-uk-2023-frs-k535080" -DIAGNOSTICS_SHA256 = "c" * 64 #: The shared exclusion-expiry clock, fixed inside the committed register's #: validity window (approved 2026-08-10, expires 2027-02-10) so the suite #: never drifts across an expiry boundary. CLOCK = date(2026, 9, 1) -#: Neutral declared name -> the legacy result name the bindings re-mint. -LEGACY_NAMES = { - "release_input_coverage": "uk_release_input_coverage", - "tail_concentration": "qrf_tail_concentration", -} - VALIDATE_REFERENCE = ( "microcosm.build.uk_runtime.weighted_integrity._validate_input_mass_reference" ) @@ -122,7 +105,7 @@ def _coverage() -> GateResult: ) -def _parity(**overrides) -> UKReleaseParityEvidence: +def _parity(**overrides) -> SimpleNamespace: fields = { "candidate_columns": {"person.age"}, "reference_columns": {"person.age"}, @@ -131,7 +114,7 @@ def _parity(**overrides) -> UKReleaseParityEvidence: "target_relative_errors": {"ons/population": 0.01}, } fields.update(overrides) - return UKReleaseParityEvidence(**fields) + return SimpleNamespace(**fields) def _reference() -> UKInputMassReference: @@ -156,9 +139,9 @@ def _qrf_policy() -> UKQRFTailConcentrationPolicy: def _fixture_coverage_registry(): """The UK registry with the coverage gate fed by the same fixture the - legacy tests inject, so both differential sides see identical coverage - evidence (the real coverage gate has its own dedicated tests). The - fixture mints the legacy name, so the re-minting path stays exercised.""" + differential harness used before retiring schema 3. The real coverage + gate has its own dedicated tests. The fixture mints the legacy name, so + the re-minting path stays exercised.""" return { **UK_GATE_REGISTRY, @@ -172,16 +155,7 @@ def _fixture_coverage_registry(): } -def _run_both(tables, *, parity=None, fit_records=None, armed=True, clock=CLOCK): - """Run the legacy battery and the declared battery over one evidence set. - - Both sides are built from the same tables and the same evidence objects - in one place — evidence asymmetry between the sides would read as a - false differential failure. That includes the exclusion-expiry clock: - the legacy aggregator threads ``now`` and the battery threads the - ``exclusions_evaluated_on`` artifact, both set to the same date here. - """ - +def _run_battery(tables, *, parity=None, fit_records=None, armed=True, clock=CLOCK): person, benunit, household = tables frame = uk_national_frame( person=person, benunit=benunit, household=household, time_period="2023" @@ -190,13 +164,10 @@ def _run_both(tables, *, parity=None, fit_records=None, armed=True, clock=CLOCK) "coverage_engine": object(), "exclusions_evaluated_on": clock, } - legacy_kwargs: dict[str, object] = {"now": clock} if fit_records is not None: artifacts["fit_weight_records"] = fit_records - legacy_kwargs["fit_weight_records"] = fit_records if parity is not None: artifacts["parity_evidence"] = parity - legacy_kwargs["parity_evidence"] = parity if armed: reference = _reference() input_mass_policy = _input_mass_policy() @@ -204,50 +175,17 @@ def _run_both(tables, *, parity=None, fit_records=None, armed=True, clock=CLOCK) artifacts["input_mass_reference"] = reference artifacts["input_mass_policy"] = input_mass_policy artifacts["qrf_tail_policy"] = qrf_policy - legacy_kwargs["input_mass_reference"] = reference - legacy_kwargs["input_mass_policy"] = input_mass_policy - legacy_kwargs["qrf_tail_policy"] = qrf_policy # Small synthetic totals exercise battery behavior without disclosing # the licensed 131-column reference (same patch as the legacy tests); # the binding's declared-pin check compares spec to runtime constant and # needs no patching. with patch(VALIDATE_REFERENCE, return_value=None): - legacy = uk_terminal_gate_report( - frame, - object(), - release_id=RELEASE_ID, - calibration_diagnostics_sha256=DIAGNOSTICS_SHA256, - input_coverage_evaluator=_coverage, - **legacy_kwargs, - ) - battery = evaluate_phase( + return evaluate_phase( load_country_spec("uk").gates, "terminal", EvidenceContext(frame=frame, artifacts=artifacts), registry=_fixture_coverage_registry(), ) - return legacy, battery - - -def _assert_identical_verdicts(legacy, battery) -> None: - legacy_by_name = {result.name: result for result in legacy.results} - evaluated = [ - outcome - for outcome in battery.outcomes - if outcome.status in (GateStatus.PASSED, GateStatus.FAILED) - ] - assert [LEGACY_NAMES.get(o.entry.gate, o.entry.gate) for o in evaluated] == [ - result.name for result in legacy.results - ] - for outcome in evaluated: - legacy_result = legacy_by_name[ - LEGACY_NAMES.get(outcome.entry.gate, outcome.entry.gate) - ] - result = outcome.result - assert result.name == outcome.entry.gate - assert result.passed == legacy_result.passed, outcome.entry.id - assert result.failures == legacy_result.failures, outcome.entry.id - assert dict(result.details) == dict(legacy_result.details), outcome.entry.id class TestUKSurfaceAdapter: @@ -313,54 +251,44 @@ def test_missing_evidence_names_its_keys(self, uk_gates) -> None: ) -class TestDifferentialAgainstLegacyBattery: - def test_fully_armed_battery_matches_gate_for_gate(self) -> None: - legacy, battery = _run_both( +class TestBatteryRegressions: + def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: + battery = _run_battery( _tables(), parity=_parity(), fit_records=(FitWeightRecord("spi_qrf", "importance"),), ) - _assert_identical_verdicts(legacy, battery) by_id = {o.entry.id: o for o in battery.outcomes} passed = [ entry_id for entry_id, o in by_id.items() if o.status is GateStatus.PASSED ] assert len(passed) == 10 - # The armed QRF gate fails identically on both sides: the tiny - # synthetic frame carries none of the declared QRF output columns. - # Failure-text parity over a real failure, for free. qrf = by_id["uk_qrf_tail_concentration"] assert qrf.status is GateStatus.FAILED - assert qrf.result.failures == ( - legacy.results[-1].failures # qrf is the last legacy gate - ) + assert "declared QRF output is absent" in qrf.result.failures[0] - def test_empty_fit_records_fail_identically(self) -> None: + def test_empty_fit_records_fail_closed(self) -> None: # Present-but-empty is not absent: a fit stage that ran and emitted - # nothing is a failed audit on both sides, never a vacuous pass - # (the shared binding alone would pass it; the UK override keeps - # the legacy guard). - legacy, battery = _run_both(_tables(), fit_records=()) + # nothing is a failed audit, never a vacuous pass. + battery = _run_battery(_tables(), fit_records=()) - _assert_identical_verdicts(legacy, battery) audit = {o.entry.id: o for o in battery.outcomes}["uk_weights_audit"] assert audit.status is GateStatus.FAILED assert "an absent audit is not a passing audit" in (audit.result.failures[0]) - def test_seeded_defects_fail_identically(self) -> None: + def test_seeded_defects_fail_the_expected_gates(self) -> None: blown = _tables(weights=[1.0, 1.0, 1.0, 1.0e9]) seeded_parity = _parity( candidate_columns={"person.age", "person.unreviewed_extra"}, target_relative_errors={"ons/population": -0.40}, ) - legacy, battery = _run_both( + battery = _run_battery( blown, parity=seeded_parity, fit_records=(FitWeightRecord("spi_qrf", "none"),), ) - _assert_identical_verdicts(legacy, battery) failed = {o.entry.id for o in battery.outcomes if o.status is GateStatus.FAILED} assert { "uk_weight_ratio", @@ -371,28 +299,14 @@ def test_seeded_defects_fail_identically(self) -> None: class TestUnevidencedArms: - """The chartered semantic difference, stated as a positive assertion. + """Missing evidence is explicit; it blocks release candidates, plus any + entry whose manifest declares absence non-excusable in every posture + (``uk_weights_audit`` — "an absent audit is not a passing audit", the + legacy strictness ported during the #654 retirement).""" - The legacy report *omits* gates whose evidence is absent (sealed by its - membership contract); the battery lists every declared entry and records - the gap as ``evidence_absent`` with the missing keys named — blocking - release candidates only. The A2 orchestration swap inherits exactly this - delta.""" + def test_battery_records_evidence_absent(self, uk_gates) -> None: + battery = _run_battery(_tables(), armed=False) - def test_legacy_omits_where_the_battery_records_evidence_absent( - self, uk_gates - ) -> None: - legacy, battery = _run_both(_tables(), armed=False) - - legacy_names = {result.name for result in legacy.results} - assert legacy_names == { - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - } - _assert_identical_verdicts(legacy, battery) absent = { o.entry.id: o.reason for o in battery.outcomes @@ -409,32 +323,22 @@ def test_legacy_omits_where_the_battery_records_evidence_absent( for reason in absent.values(): assert reason.startswith("missing evidence: ") - assert battery.blocking_outcomes(release_candidate=False) == () + # The audit's absence blocks even the default posture — its status + # stays honestly evidence_absent; only the enforcement is strict. + default_blocked = { + o.entry.id for o in battery.blocking_outcomes(release_candidate=False) + } + assert default_blocked == {"uk_weights_audit"} blocked = { o.entry.id for o in battery.blocking_outcomes(release_candidate=True) } assert blocked == set(absent) - def test_absent_but_required_fit_evidence_is_the_named_delta(self) -> None: - # Legacy: a production fit stage without records is an explicit - # failure. Battery: the absent artifact is a named evidence gap that - # blocks release candidates. Same shipping decision, different - # taxonomy — asserted so the A2 review can lean on it. + def test_absent_fit_evidence_is_named(self) -> None: person, benunit, household = _tables() frame = uk_national_frame( person=person, benunit=benunit, household=household, time_period="2023" ) - legacy = uk_terminal_gate_report( - frame, - object(), - release_id=RELEASE_ID, - calibration_diagnostics_sha256=DIAGNOSTICS_SHA256, - input_coverage_evaluator=_coverage, - require_fit_weight_records=True, - ) - legacy_audit = {r.name: r for r in legacy.results}["weights_audit"] - assert legacy_audit.passed is False - battery = evaluate_phase( load_country_spec("uk").gates, "terminal", @@ -456,7 +360,7 @@ class TestExclusionDiscipline: ) def test_every_exclusion_gate_shares_the_injected_clock(self) -> None: - _legacy, battery = _run_both( + battery = _run_battery( _tables(), parity=_parity(), fit_records=(FitWeightRecord("spi_qrf", "importance"),), @@ -468,17 +372,19 @@ def test_every_exclusion_gate_shares_the_injected_clock(self) -> None: } assert set(stamps.values()) == {CLOCK.isoformat()}, stamps - def test_an_expired_register_behaves_identically_on_both_sides(self) -> None: - # Past the committed register's expiry the exclusion is out of - # force on both paths; whatever the verdict, it must be the same - # verdict — the differential contract holds at every clock value. - legacy, battery = _run_both( + def test_an_expired_register_fails_closed(self) -> None: + battery = _run_battery( _tables(), parity=_parity(), fit_records=(FitWeightRecord("spi_qrf", "importance"),), clock=date(2027, 3, 1), ) - _assert_identical_verdicts(legacy, battery) + failed = {o.entry.id for o in battery.outcomes if o.status is GateStatus.FAILED} + assert { + "uk_degenerate_release_surface", + "uk_qrf_tail_concentration", + } <= failed + assert "uk_input_mass_parity" not in failed def test_review_override_is_loud_in_the_evidence_payload(self) -> None: binding = UK_GATE_REGISTRY["degenerate_release_surface"] diff --git a/packages/microcosm-build/tests/test_uk_diagnostics.py b/packages/microcosm-build/tests/test_uk_diagnostics.py index a2302d20..9dc8f65c 100644 --- a/packages/microcosm-build/tests/test_uk_diagnostics.py +++ b/packages/microcosm-build/tests/test_uk_diagnostics.py @@ -108,8 +108,20 @@ def _diagnostics_case(*, with_skipped: bool = False): registry.to_target_set(), weights=final_weights, ) - shipped = household.assign(household_weight=final_weights) - return result, shipped, registry, geography + diagnostic_frame = Frame( + { + "person": frame.table("person"), + "household": household, + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + final_weights, + WeightKind.DESIGN, + ) + }, + ) + return result, diagnostic_frame, registry, geography def test_uk_weight_summary_reports_kish_ess_and_concentration() -> None: @@ -242,11 +254,11 @@ def test_zero_weight_strata_validates_alignment_and_columns() -> None: def test_payload_preserves_common_schema_and_adds_versioned_uk_evidence() -> None: - result, household, registry, geography = _diagnostics_case() + result, frame, registry, geography = _diagnostics_case() payload = uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=registry, build={"release_id": "fixture"}, @@ -285,7 +297,7 @@ def test_payload_preserves_common_schema_and_adds_versioned_uk_evidence() -> Non assert uk["weights"]["top_1pct_weight_share"] == payload["top_1pct_weight_share"] assert uk["weights"]["zero_weight_records"] == 1 assert uk["weights"]["ess_fraction"] == pytest.approx( - payload["effective_sample_size"] / len(household) + payload["effective_sample_size"] / frame.n("household") ) assert uk["target_pass_rates_by_geography_level"] == [ { @@ -333,7 +345,7 @@ def test_payload_preserves_common_schema_and_adds_versioned_uk_evidence() -> Non def test_payload_requires_exact_explicit_geography_mapping() -> None: - result, household, registry, geography = _diagnostics_case() + result, frame, registry, geography = _diagnostics_case() missing = dict(geography) missing.pop("national_target@2023") extra = {**geography, "not_a_target@2023": "national"} @@ -343,32 +355,32 @@ def test_payload_requires_exact_explicit_geography_mapping() -> None: with pytest.raises(ValueError, match="must exactly cover"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=invalid, target_registry=registry, ) with pytest.raises(ValueError, match="Unknown UK target geography level"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=unknown, target_registry=registry, ) with pytest.raises(TypeError, match="must map declared target names"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=list(geography), target_registry=registry, ) def test_skipped_target_counts_as_a_geography_non_pass() -> None: - result, household, registry, geography = _diagnostics_case(with_skipped=True) + result, frame, registry, geography = _diagnostics_case(with_skipped=True) payload = uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=registry, ) @@ -395,64 +407,78 @@ def test_skipped_target_counts_as_a_geography_non_pass() -> None: with pytest.raises(ValueError, match="must exactly cover"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=missing_skipped, target_registry=registry, ) def test_payload_requires_a_valid_matching_uk_registry() -> None: - result, household, registry, geography = _diagnostics_case() + result, frame, registry, geography = _diagnostics_case() with pytest.raises(TypeError, match="require a TargetRegistry"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=object(), ) with pytest.raises(ValueError, match="country == 'uk'"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=TargetRegistry(registry.specs, country="us"), ) with pytest.raises(ValueError, match="non-empty registry"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=TargetRegistry((), country="uk"), ) with pytest.raises(ValueError, match="exactly partition"): uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=TargetRegistry(registry.specs[:-1], country="uk"), ) def test_payload_requires_the_exact_shipped_weight_vector() -> None: - result, household, registry, geography = _diagnostics_case() - household.loc[0, "household_weight"] += 1.0 + result, frame, registry, geography = _diagnostics_case() + bad_weights = frame.weights_for("household").values.copy() + bad_weights[0] += 1.0 + mismatched = Frame( + { + "person": frame.table("person"), + "household": frame.table("household"), + }, + frame.schema, + { + "household": Weights( + bad_weights, + frame.weights_for("household").kind, + ) + }, + ) with pytest.raises(ValueError, match="must exactly match"): uk_calibration_diagnostics_payload( result, - household, + mismatched, target_geography_levels=geography, target_registry=registry, ) def test_writer_round_trips_strict_json(tmp_path: Path) -> None: - result, household, registry, geography = _diagnostics_case() + result, frame, registry, geography = _diagnostics_case() path = write_uk_calibration_diagnostics( result, tmp_path / "calibration_diagnostics.json", - household, + frame, target_geography_levels=geography, target_registry=registry, ) @@ -460,7 +486,7 @@ def test_writer_round_trips_strict_json(tmp_path: Path) -> None: assert json.loads(path.read_text(encoding="utf-8")) == ( uk_calibration_diagnostics_payload( result, - household, + frame, target_geography_levels=geography, target_registry=registry, ) @@ -470,7 +496,7 @@ def test_writer_round_trips_strict_json(tmp_path: Path) -> None: write_uk_calibration_diagnostics( result, path, - household, + frame, target_geography_levels=geography, target_registry=registry, build={"not_json": float("nan")}, @@ -483,7 +509,7 @@ def test_writer_preserves_prior_bytes_when_atomic_replace_fails( tmp_path: Path, monkeypatch, ) -> None: - result, household, registry, geography = _diagnostics_case() + result, frame, registry, geography = _diagnostics_case() path = tmp_path / "calibration_diagnostics.json" prior = b'{"prior":true}\n' path.write_bytes(prior) @@ -496,7 +522,7 @@ def fail_replace(_temporary, _output): write_uk_calibration_diagnostics( result, path, - household, + frame, target_geography_levels=geography, target_registry=registry, ) diff --git a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py index 0c055a27..b7befa78 100644 --- a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py +++ b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py @@ -418,6 +418,7 @@ def test_candidate_clone_identity_mismatch_fails_closed(tmp_path: Path) -> None: benunit=dataset.table("benunit"), household=household, time_period="2023", + household_weights=dataset.weights_for("household").values, ) with pytest.raises(ValueError, match="person IDs do not reverse"): diff --git a/packages/microcosm-build/tests/test_uk_hmrc_calibration.py b/packages/microcosm-build/tests/test_uk_hmrc_calibration.py index 4db612ee..bcebfc78 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_calibration.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_calibration.py @@ -149,6 +149,11 @@ def _with( household=frame.table("household") if household is None else household, time_period=uk_time_period(frame) if time_period is None else time_period, weight_kind=uk_household_weight_kind(frame), + household_weights=( + None + if household is not None and "household_weight" in household + else frame.weights_for("household").values + ), mass_log=frame.mass_log, ) @@ -298,6 +303,7 @@ def test_materialization_fails_closed_when_one_component_has_no_band_support() - def test_materialization_requires_strictly_positive_household_prior() -> None: dataset, targets = _feasible_dataset_and_targets() household = dataset.table("household").copy() + household["household_weight"] = dataset.weights_for("household").values household.loc[0, "household_weight"] = 0.0 broken = _with(dataset, household=household) diff --git a/packages/microcosm-build/tests/test_uk_hmrc_restoration.py b/packages/microcosm-build/tests/test_uk_hmrc_restoration.py index 073194eb..9e3e5a90 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_restoration.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_restoration.py @@ -109,6 +109,7 @@ def _tampered_dataset() -> Frame: benunit=frame.table("benunit"), household=frame.table("household"), time_period=HMRC_SPI_BUILD_PERIOD, + household_weights=frame.weights_for("household").values, ) @@ -229,7 +230,7 @@ def _support_and_imputation( benunit = pd.DataFrame({"benunit_id": np.arange(1, row_count + 1)}) mass_record = MassChangeRecord( entity="household", - old_total=float(dataset.table("household")["household_weight"].sum()), + old_total=float(dataset.weights_for("household").total), new_total=float(sum(household_weights)), declared_factor=1.0, reason="reviewed test allocation to one positive-mass SPI channel", @@ -509,6 +510,7 @@ def test_restoration_binds_loaded_candidate_bytes_before_source_io( benunit=base.table("benunit"), household=base.table("household"), time_period=HMRC_SPI_BUILD_PERIOD, + household_weights=base.weights_for("household").values, ) write_uk_national_frame(replacement, candidate_path) loaded_replacement, replacement_provenance = load_uk_national_frame(candidate_path) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 15b62ede..71101a33 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -160,7 +160,7 @@ def test_ladder_clone_assigns_gates_and_conserves(toy_ladder, tmp_path) -> None: assert len(frame_household) == 8 assert len(frame_person) == 10 assert len(frame_benunit) == 8 - assert frame_household["household_weight"].sum() == pytest.approx(33.0) + assert result.frame.weights_for("household").total == pytest.approx(33.0) assert frame_household["household_id"].is_unique # In-memory carrier: per-entity clone-index names (Frame's flattening # rule forbids one shared name across entity tables). @@ -212,9 +212,9 @@ def test_ladder_clone_assigns_gates_and_conserves(toy_ladder, tmp_path) -> None: assert household["household_id"].is_unique assert person["person_id"].is_unique assert set(person["person_household_id"]) <= set(household["household_id"]) - # Export payload schema is unchanged by the Frame carrier: the - # artifact clone column keeps its legacy name and position on all - # three tables, and no per-entity name leaks into the H5. + # The artifact clone column keeps its legacy name and position on all + # three tables, no per-entity name leaks into the H5, and typed + # household weights are materialized into the export payload. for table in (person, benunit, household): assert "clone_index" in table.columns assert not any( @@ -229,14 +229,14 @@ def test_ladder_clone_assigns_gates_and_conserves(toy_ladder, tmp_path) -> None: "clone_index", ] assert benunit.columns.tolist() == ["benunit_id", "clone_index"] - assert household.columns.tolist()[:6] == [ + assert household.columns.tolist()[:5] == [ "household_id", - "household_weight", "region", "source_household_id", "source_household_key", "clone_index", ] + assert household["household_weight"].sum() == pytest.approx(33.0) def test_ladder_clone_refuses_vintage_mismatch(toy_ladder) -> None: @@ -495,13 +495,15 @@ def test_ladder_clone_pins_per_copy_weights_and_fk_alignment(toy_ladder) -> None ladder, _ = toy_ladder result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=2) household = result.frame.table("household") + household_weights = result.frame.weights_for("household").values person = result.frame.table("person") household_clone_column = ladder_clone_index_column("household") # Every source copy carries exactly its divided weight. for clone_index in (0, 1): - copy = household[household[household_clone_column] == clone_index] + mask = household[household_clone_column] == clone_index + copy = household[mask] weights = dict( - zip(copy["source_household_id"], copy["household_weight"], strict=True) + zip(copy["source_household_id"], household_weights[mask], strict=True) ) assert weights == { 1: pytest.approx(1.5), diff --git a/packages/microcosm-build/tests/test_uk_local_rowwise.py b/packages/microcosm-build/tests/test_uk_local_rowwise.py index f440acb2..86790b6b 100644 --- a/packages/microcosm-build/tests/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/test_uk_local_rowwise.py @@ -155,7 +155,7 @@ def test_rowwise_doctrine_solve_uses_base_weights_directly() -> None: assert record.old_total == pytest.approx(float(np.sum(base))) assert record.new_total == pytest.approx(float(np.sum(result.weights))) np.testing.assert_allclose( - result.frame.table("household")["household_weight"].to_numpy(), + result.frame.weights_for("household").values, result.weights, ) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index 2016363f..f6e15c05 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -29,7 +29,6 @@ from microcosm.build.uk_runtime.release_input_coverage import ( uk_release_input_coverage_gate, ) -from microcosm.build.uk_runtime.terminal_gates import UKReleaseParityEvidence from microcosm.frame import Frame, MassChangeRecord, WeightKind TEST_UK_RELEASE_ID = "populace-uk-2023-frs-k535080" @@ -66,10 +65,23 @@ def _toy_gate_registry() -> dict[str, UKGateBinding]: manifest preflight are pass-throughs (both have their own tests) and every gate without a binding is a named ``evidence_absent`` gap — non-blocking off the release-candidate posture, exactly the legacy - fixture's effect of reporting only the coverage verdict. + fixture's effect of reporting only the coverage verdict. The one + exception is the weights audit: its manifest entry declares + ``evidence_absent_blocks`` (an absent audit is not a passing audit, + in every posture), so the seam registry binds it as a pass-through — + the strict-absence behavior has its own tests. """ return { + "weights_audit": UKGateBinding( + name="weights_audit", + evaluator=lambda context, parameters: GateResult( + name="weights_audit", + passed=True, + details={"toy_audit": True}, + ), + needs_frame=False, + ), "release_input_coverage": UKGateBinding( name="release_input_coverage", evaluator=_toy_coverage_evaluator, @@ -111,6 +123,7 @@ def _replace_person(frame: Frame, person: pd.DataFrame) -> Frame: household=frame.table("household"), time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, mass_log=frame.mass_log, ) @@ -286,35 +299,36 @@ def test_driver_validates_the_uk_residue_after_each_stage( ) -> None: """The driver's post-stage validate is load-bearing, not decorative. - A stage returning ``frame.with_weights(...)`` with a stale exported - ``household_weight`` column constructs a perfectly valid Frame — the - kernel permits the column when typed weights exist — so only the - driver's ``validate_uk_national_frame`` call can stop the wrong weights - from shipping. The stage-side rejection tests all raise inside the - stage's own frame construction; this one can only fail at the driver - seam. + A stage can directly construct a kernel-valid Frame carrying the exported + ``household_weight`` column. Only the driver's + ``validate_uk_national_frame`` call can stop that column from returning + to the in-build carrier. """ pytest.importorskip("tables") - from microcosm.frame import CONSERVE_MASS, Weights input_h5 = tmp_path / "base.h5" _write_two_row_h5(input_h5) - def redistribute_without_refreshing_column(frame: Frame) -> Frame: - weights = frame.weights_for("household") - return frame.with_weights( - "household", - Weights(values=weights.values[::-1].copy(), kind=weights.kind), - mass=CONSERVE_MASS, + def return_export_column(frame: Frame) -> Frame: + return Frame( + { + "person": frame.table("person"), + "benunit": frame.table("benunit"), + "household": frame.table("household").assign(household_weight=999.0), + }, + frame.schema, + {"household": frame.weights_for("household")}, + metadata=frame.metadata, + mass_log=frame.mass_log, ) - with pytest.raises(ValueError, match="refresh the exported column"): + with pytest.raises(ValueError, match="must not persist exported weight"): _run_national_build( input_h5=input_h5, staging_h5=tmp_path / "staging.h5", stages=( - UKNationalStage("stale_column", redistribute_without_refreshing_column), + UKNationalStage("export_column", return_export_column), ), coverage_engine=object(), ) @@ -424,7 +438,7 @@ def recording_writer(frame, path): staged, staged_provenance = load_uk_national_frame(staging_h5) assert staged_provenance.source_h5 == staging_h5.resolve() assert staged.person["employment_income"].tolist() == [50_000.0] - assert staged.table("household")["household_weight"].tolist() == [2.0] + assert staged.weights_for("household").values.tolist() == [2.0] diagnostic = json.loads(coverage_json.read_text()) assert diagnostic["enforced"] is True assert diagnostic["input_coverage"]["passed"] is True @@ -846,6 +860,10 @@ def test_national_build_real_terminal_batch_passes_before_staging( result = _run_national_build( input_h5=input_h5, staging_h5=staging_h5, + # The audit's absence blocks every posture (evidence_absent_blocks), + # so a healthy staging pass needs the HMRC stage's audit evidence — + # exactly what the real pipeline supplies. + stages=(UKNationalStage("hmrc_spi_income", _RecordedFitStage()),), coverage_engine=object(), terminal_gate_path=terminal_json, gate_registry=None, # the real UK registry @@ -866,9 +884,9 @@ def test_national_build_real_terminal_batch_passes_before_staging( "uk_zero_weight_strata": "passed", "uk_weight_ess": "passed", "uk_weight_ratio": "passed", + "uk_weights_audit": "passed", # The legacy report omitted unevidenced gates; the battery names # every gap — non-blocking off the release-candidate posture. - "uk_weights_audit": "evidence_absent", "uk_export_surface": "evidence_absent", "uk_target_surface": "evidence_absent", "uk_target_fit": "evidence_absent", @@ -932,7 +950,7 @@ def test_national_build_real_terminal_batch_writes_all_findings_before_raise( assert not staging_h5.exists() -def test_national_build_parity_trio_evaluates_with_evidence_absent_without( +def test_national_build_parity_trio_is_evidence_absent( monkeypatch, tmp_path, ) -> None: @@ -941,43 +959,22 @@ def test_national_build_parity_trio_evaluates_with_evidence_absent_without( input_h5 = tmp_path / "healthy.h5" _write_two_row_h5(input_h5) _stub_real_coverage(monkeypatch, _passing_gate) - parity = UKReleaseParityEvidence( - candidate_columns=("person.employment_income",), - reference_columns=("person.employment_income",), - candidate_targets=("population",), - reference_targets=("population",), - target_relative_errors={"population": 0.0}, - ) - with_evidence = _run_national_build( + result = _run_national_build( input_h5=input_h5, staging_h5=tmp_path / "staging.h5", stages=(UKNationalStage("hmrc_spi_income", _RecordedFitStage()),), coverage_engine=object(), - parity_evidence=parity, terminal_gate_path=tmp_path / "terminal_gates.json", - gate_registry=None, # the real UK registry + gate_registry=None, ) - gates = with_evidence.gate_report["gates"] + gates = result.gate_report["gates"] assert gates["uk_weights_audit"]["status"] == "passed" assert gates["uk_weights_audit"]["details"]["resolved_weight_kinds"] == { "uk_frs_only_spi_fill": "importance", "uk_spi_2022_23_income": "design", } - for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): - assert gates[entry_id]["status"] == "passed", entry_id - - without_evidence = _run_national_build( - input_h5=input_h5, - staging_h5=tmp_path / "staging2.h5", - stages=(UKNationalStage("hmrc_spi_income", _RecordedFitStage()),), - coverage_engine=object(), - terminal_gate_path=tmp_path / "terminal_gates2.json", - gate_registry=None, - ) - - gates = without_evidence.gate_report["gates"] for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): assert gates[entry_id]["status"] == "evidence_absent", entry_id assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" @@ -1133,6 +1130,7 @@ def break_links(frame: Frame) -> Frame: benunit=frame.table("benunit"), household=frame.table("household"), time_period=None, + household_weights=frame.weights_for("household").values, ), "time_period must be a non-empty string", ), @@ -1216,6 +1214,7 @@ def test_national_staging_h5_loads_through_policyengine_uk(tmp_path) -> None: household=frame.table("household"), time_period=uk_time_period(frame), weight_kind=WeightKind.IMPORTANCE, + household_weights=frame.weights_for("household").values, mass_log=( MassChangeRecord( entity="household", diff --git a/packages/microcosm-build/tests/test_uk_national_frame.py b/packages/microcosm-build/tests/test_uk_national_frame.py index b3c36621..6fcf7d38 100644 --- a/packages/microcosm-build/tests/test_uk_national_frame.py +++ b/packages/microcosm-build/tests/test_uk_national_frame.py @@ -106,6 +106,7 @@ def test_construction_accessors_and_residue_validation() -> None: np.testing.assert_array_equal( frame.weights_for("household").values, np.array([10.0, 20.0, 30.0]) ) + assert "household_weight" not in frame.table("household") validate_uk_national_frame(frame) @@ -130,10 +131,9 @@ def test_construction_enforces_frame_linkage_invariants() -> None: _frame(benunit=orphaned) -def test_validate_rejects_column_vector_disagreement() -> None: - # Construct a frame whose persisted column and typed vector disagree — - # only reachable by direct Frame construction, which is the point: the - # residue validation catches drift the constructor helper cannot produce. +def test_validate_rejects_exported_weight_column_on_the_carrier() -> None: + # Only reachable by direct Frame construction; the canonical constructor + # consumes household_weight into the typed vector and strips the column. frame = Frame( tables={ "person": person_frame(), @@ -148,7 +148,7 @@ def test_validate_rejects_column_vector_disagreement() -> None: }, metadata={"time_period": "2023"}, ) - with pytest.raises(ValueError, match="disagrees with the frame's typed weights"): + with pytest.raises(ValueError, match="must not persist exported weight"): validate_uk_national_frame(frame) @@ -226,19 +226,16 @@ def test_validate_rejects_non_household_typed_weights() -> None: validate_uk_national_frame(frame) -def test_weight_only_update_must_refresh_the_exported_column() -> None: +def test_weight_only_update_keeps_the_carrier_columnless() -> None: frame = _frame() - # Same total, different distribution: the kernel accepts the conserving - # replacement, but the persisted household_weight column is now stale — - # the UK residue validation refuses to let it ship. updated = frame.with_weights( "household", Weights(values=np.array([20.0, 10.0, 30.0]), kind=WeightKind.DESIGN), mass=CONSERVE_MASS, ) - with pytest.raises(ValueError, match="refresh the exported column"): - validate_uk_national_frame(updated) + assert "household_weight" not in updated.table("household") + validate_uk_national_frame(updated) def test_write_load_round_trip_with_provenance(tmp_path: Path) -> None: diff --git a/packages/microcosm-build/tests/test_uk_national_sampling.py b/packages/microcosm-build/tests/test_uk_national_sampling.py index 709bdc3d..d701b97d 100644 --- a/packages/microcosm-build/tests/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/test_uk_national_sampling.py @@ -212,37 +212,13 @@ def test_sampled_frame_is_normalized_refreshed_and_valid() -> None: validate_uk_national_frame(sampled) weights = sampled.weights_for("household") assert float(weights.total) == pytest.approx(full_mass) - np.testing.assert_array_equal( - sampled.table("household")["household_weight"].to_numpy(dtype="float64"), - weights.values, - ) + assert "household_weight" not in sampled.table("household") record = sampled.mass_log[-1] assert record.entity == "household" assert "composition-preserving" in record.reason assert receipt["normalized_household_mass"] == pytest.approx(full_mass) -@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError") -def test_weight_column_refresh_verifies_the_live_reference() -> None: - """If Frame.table ever returns copies, the in-place household_weight - refresh silently no-ops and the staging payload would export - pre-normalization weights; the sampler must catch that locally instead - of relying on validate_uk_national_frame two calls later. (The pandas - chained-assignment warning is this exact failure mode firing on the - injected copy, so the test suppresses it.)""" - - frame = _source_family_frame() - stored_table = Frame.table - - def copying_table(self: Frame, name: str) -> pd.DataFrame: - return stored_table(self, name).copy() - - with pytest.MonkeyPatch.context() as patch: - patch.setattr(Frame, "table", copying_table) - with pytest.raises(ValueError, match="refresh did not persist"): - sample_uk_national_frame(frame, fraction=0.5, seed=3) - - def test_numeric_region_codes_are_refused() -> None: """Region strata labels are formatted values, so an integer code in one build and a float code in another would silently split a stratum; the @@ -256,6 +232,7 @@ def test_numeric_region_codes_are_refused() -> None: benunit=frame.table("benunit"), household=household, time_period="2023", + household_weights=frame.weights_for("household").values, ) with pytest.raises(ValueError, match="must contain non-empty strings"): sample_uk_national_frame(numeric, fraction=0.5, seed=1) @@ -295,6 +272,7 @@ def test_missing_lineage_columns_fail_closed() -> None: benunit=frame.table("benunit"), household=household, time_period="2023", + household_weights=frame.weights_for("household").values, ) with pytest.raises(ValueError, match="missing \\['clone_index'\\]"): sample_uk_national_frame(stripped, fraction=0.5, seed=1) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dataset.py b/packages/microcosm-build/tests/test_uk_rowwise_dataset.py index b388e5dc..dc3f49c2 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dataset.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dataset.py @@ -16,23 +16,6 @@ class FakeUKDataset: time_period = "2023" - # In-memory datasets must declare their weight kind explicitly; only H5 - # inputs keep the attribute-less design default. - household_weight_kind = WeightKind.DESIGN - - def __init__( - self, - *, - person: pd.DataFrame, - benunit: pd.DataFrame, - household: pd.DataFrame, - ): - self.person = person - self.benunit = benunit - self.household = household - - -class FakeUKDatasetWithoutPeriod: household_weight_kind = WeightKind.DESIGN def __init__( @@ -170,23 +153,20 @@ def test_clone_uk_dataset_tables_assigns_geography_and_remaps_links() -> None: ) -def test_clone_uk_dataset_object_uses_dataset_time_period() -> None: +def test_clone_uk_dataset_rejects_duck_typed_object() -> None: dataset = FakeUKDataset( person=person_frame(), benunit=benunit_frame(), household=household_frame(), ) - result = clone_uk_dataset_with_rowwise_geography( - dataset, - crosswalk_frame(), - n_clones=1, - seed=1, - ) - - assert result.time_period == "2023" - assert result.household["household_id"].tolist() == [1, 2] - assert result.person["person_household_id"].tolist() == [1, 2, 2] + with pytest.raises(TypeError, match="duck-typed in-memory carrier retired"): + clone_uk_dataset_with_rowwise_geography( + dataset, + crosswalk_frame(), + n_clones=1, + seed=1, + ) def test_clone_uk_dataset_accepts_a_frame_without_downgrading_kind() -> None: @@ -223,23 +203,6 @@ def test_clone_uk_dataset_accepts_a_frame_without_downgrading_kind() -> None: assert result.person["person_household_id"].tolist() == [1, 2, 2] -def test_clone_uk_dataset_object_can_use_source_year_as_period() -> None: - dataset = FakeUKDatasetWithoutPeriod( - person=person_frame(), - benunit=benunit_frame(), - household=household_frame(), - ) - - result = clone_uk_dataset_with_rowwise_geography( - dataset, - crosswalk_frame(), - source_year=2024, - ) - - assert result.time_period == "2024" - assert result.household["source_household_key"].tolist() == ["2024:1", "2024:2"] - - def test_validate_uk_rowwise_dataset_tables_rejects_broken_household_link() -> None: result = clone_uk_dataset_tables_with_rowwise_geography( person=person_frame(), diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index 0eb9139b..a7c83e74 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -426,7 +426,7 @@ def test_clone_entity_frame_refuses_int64_overflow() -> None: ) -def test_dataset_object_metadata_carried_and_required() -> None: +def test_dataset_object_metadata_carrier_is_retired() -> None: class SeamLike: time_period = "2023" household_weight_kind = WeightKind.IMPORTANCE @@ -445,55 +445,7 @@ def __init__(self) -> None: self.benunit = benunit_frame() self.household = household_frame() - class MissingKind: - time_period = "2023" - - def __init__(self) -> None: - self.person = person_frame() - self.benunit = benunit_frame() - self.household = household_frame() - - class NoMassLog: - time_period = "2023" - household_weight_kind = WeightKind.DESIGN - - def __init__(self) -> None: - self.person = person_frame() - self.benunit = benunit_frame() - self.household = household_frame() - - class NoneLog: - time_period = "2023" - household_weight_kind = WeightKind.DESIGN - mass_log = None - - def __init__(self) -> None: - self.person = person_frame() - self.benunit = benunit_frame() - self.household = household_frame() - - carried = clone_uk_dataset_with_rowwise_geography( - SeamLike(), crosswalk_frame(), n_clones=1, seed=3 - ) - assert carried.household_weight_kind is WeightKind.IMPORTANCE - assert len(carried.mass_log) == 2 - - # An in-memory dataset without a declared kind hard-fails: defaulting - # here would silently downgrade importance/calibrated weights to design. - with pytest.raises(TypeError, match="household_weight_kind"): - clone_uk_dataset_with_rowwise_geography( - MissingKind(), crosswalk_frame(), n_clones=1, seed=3 - ) - - # An absent mass_log still defaults to an empty history once the kind - # is declared; only an explicit None is rejected. - no_history = clone_uk_dataset_with_rowwise_geography( - NoMassLog(), crosswalk_frame(), n_clones=1, seed=3 - ) - assert no_history.household_weight_kind is WeightKind.DESIGN - assert len(no_history.mass_log) == 1 - - with pytest.raises(TypeError, match="mass_log"): + with pytest.raises(TypeError, match="duck-typed in-memory carrier retired"): clone_uk_dataset_with_rowwise_geography( - NoneLog(), crosswalk_frame(), n_clones=1, seed=3 + SeamLike(), crosswalk_frame(), n_clones=1, seed=3 ) diff --git a/packages/microcosm-build/tests/test_uk_stage_checkpoints.py b/packages/microcosm-build/tests/test_uk_stage_checkpoints.py index f71c5da5..fde2d964 100644 --- a/packages/microcosm-build/tests/test_uk_stage_checkpoints.py +++ b/packages/microcosm-build/tests/test_uk_stage_checkpoints.py @@ -97,7 +97,7 @@ def test_uk_no_extension_checkpoint_keeps_its_schema_2_byte_golden( assert completed.path.name == "000_retain.frame.h5" assert hashlib.sha256(completed.path.read_bytes()).hexdigest() == ( - "7fd5d25833f395b9eac57fcb0bc6537a344862a024ee981daf167949722a17ee" + "65952080dbebd4149051659c0b367a5384ff5ad9206a6f8d90b1dba89ff5b631" ) @@ -154,9 +154,7 @@ def test_nested_frame_metadata_round_trips_through_the_stage_record( } ), "benunit": pd.DataFrame({"benunit_id": [11]}), - "household": pd.DataFrame( - {"household_id": [101], "household_weight": [10.0]} - ), + "household": pd.DataFrame({"household_id": [101]}), }, EntitySchema(group_entities=("benunit", "household")), {"household": Weights(np.array([10.0], dtype=np.float64), WeightKind.DESIGN)}, @@ -203,9 +201,7 @@ def test_set_metadata_round_trips_with_an_unchanged_content_identity( } ), "benunit": pd.DataFrame({"benunit_id": [11]}), - "household": pd.DataFrame( - {"household_id": [101], "household_weight": [10.0]} - ), + "household": pd.DataFrame({"household_id": [101]}), }, EntitySchema(group_entities=("benunit", "household")), {"household": Weights(np.array([10.0], dtype=np.float64), WeightKind.DESIGN)}, diff --git a/packages/microcosm-build/tests/test_uk_terminal_gates.py b/packages/microcosm-build/tests/test_uk_terminal_gates.py index 8a152392..17149f8a 100644 --- a/packages/microcosm-build/tests/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/test_uk_terminal_gates.py @@ -1,12 +1,9 @@ -"""UK terminal-gate batching and seeded-defect coverage.""" +"""UK terminal gate evaluators.""" from __future__ import annotations -import base64 -import hashlib -import hmac -import json import math +from datetime import UTC, date, datetime from types import SimpleNamespace from unittest.mock import patch @@ -14,31 +11,29 @@ import pandas as pd import pytest -from microcosm.build.gates import FitWeightRecord, GateReport, GateResult -from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.terminal_gates import ( UK_DEFAULT_ZERO_WEIGHT_STRATA, - UK_MAX_TO_MEDIAN_WEIGHT_RATIO, - UK_TERMINAL_GATE_PRODUCER, - UK_TERMINAL_GATE_SIGNATURE_ALGORITHM, - UK_TERMINAL_GATE_SIGNING_KEY_ENV, UKInputMassParityPolicy, UKInputMassReference, UKQRFTailConcentrationPolicy, - UKReleaseParityEvidence, UKZeroWeightStratumDeclaration, uk_default_degenerate_reviewed_exclusions, uk_degenerate_release_surface_gate, uk_export_surface_gate, + uk_input_mass_parity_gate, + uk_qrf_tail_concentration_gate, uk_target_fit_gate, uk_target_surface_gate, - uk_terminal_gate_policy_sha256, - uk_terminal_gate_report, + uk_weight_ess_gate, uk_weight_ratio_gate, - write_uk_terminal_gate_report, + uk_zero_weight_strata_gate, ) -TEST_UK_TERMINAL_GATE_SIGNING_KEY = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" +TEST_MIN_ESS_FRACTION = 0.01 +TEST_MAX_TO_MEDIAN_WEIGHT_RATIO = 1_151.2542195939373 +VALIDATE_REFERENCE = ( + "microcosm.build.uk_runtime.weighted_integrity._validate_input_mass_reference" +) def _entry(reason: str, *, expires_on: str = "2027-02-10") -> dict[str, str]: @@ -53,18 +48,6 @@ def _entry(reason: str, *, expires_on: str = "2027-02-10") -> dict[str, str]: } -TEST_UK_RELEASE_ID = "populace-uk-2023-frs-k535080" -TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256 = "c" * 64 - - -@pytest.fixture(autouse=True) -def _trusted_terminal_gate_signing_key(monkeypatch) -> None: - monkeypatch.setenv( - UK_TERMINAL_GATE_SIGNING_KEY_ENV, - TEST_UK_TERMINAL_GATE_SIGNING_KEY, - ) - - def _dataset( *, n: int = 4, @@ -98,58 +81,6 @@ def _dataset( ) -def _coverage(*, passed: bool = True) -> GateResult: - return GateResult( - name="uk_release_input_coverage", - passed=passed, - failures=() if passed else ("seeded coverage defect",), - details={"fixture": True}, - ) - - -def _frame_of(dataset): - return uk_national_frame( - person=dataset.person, - benunit=dataset.benunit, - household=dataset.household, - time_period="2023", - ) - - -def _report(dataset=None, **kwargs): - # Small synthetic totals exercise battery behavior without disclosing the - # licensed 131-column reference; trust-anchor behavior has dedicated tests. - with patch( - "microcosm.build.uk_runtime.weighted_integrity._validate_input_mass_reference", - return_value=None, - ): - return uk_terminal_gate_report( - _frame_of(_dataset() if dataset is None else dataset), - object(), - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - input_coverage_evaluator=lambda: _coverage(), - **kwargs, - ) - - -def _gates(report) -> dict[str, dict[str, object]]: - return report.to_manifest()["gates"] - - -def test_healthy_synthetic_release_passes_the_mandatory_batch() -> None: - report = _report() - - assert report.passed - assert list(_gates(report)) == [ - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - ] - - @pytest.mark.parametrize( ("signal", "detail_key"), [ @@ -162,10 +93,10 @@ def test_each_degenerate_column_class_produces_its_named_finding( signal, detail_key, ) -> None: - gate = _gates(_report(_dataset(signal=signal)))["degenerate_release_surface"] + gate = uk_degenerate_release_surface_gate(_dataset(signal=signal)) - assert gate["passed"] is False - assert gate["details"][detail_key] == ["person.employment_income"] + assert not gate.passed + assert gate.details[detail_key] == ["person.employment_income"] def test_reviewed_degenerate_exclusion_is_recorded_and_stale_entries_fail() -> None: @@ -191,11 +122,11 @@ def test_reviewed_degenerate_exclusion_is_recorded_and_stale_entries_fail() -> N def test_undeclared_zero_weight_stratum_produces_named_finding() -> None: dataset = _dataset(weights=[0.0, 1.0, 1.0, 1.0]) - gate = _gates(_report(dataset))["zero_weight_strata"] + gate = uk_zero_weight_strata_gate(dataset.household) - assert gate["passed"] is False - assert gate["details"]["unmatched_zero_weight_rows"] == 1 - assert "match no declared stratum" in gate["failures"][0] + assert not gate.passed + assert gate.details["unmatched_zero_weight_rows"] == 1 + assert "match no declared stratum" in gate.failures[0] def test_zero_weight_stratum_beyond_declaration_produces_named_finding() -> None: @@ -210,13 +141,11 @@ def test_zero_weight_stratum_beyond_declaration_produces_named_finding() -> None reason="No zero rows are expected in the healthy fixture.", ) - gate = _gates(_report(dataset, zero_weight_declarations=(declaration,)))[ - "zero_weight_strata" - ] + gate = uk_zero_weight_strata_gate(dataset.household, declarations=(declaration,)) - assert gate["passed"] is False - assert gate["details"]["declared_strata"][0]["zero_weight_rows"] == 1 - assert "exceed the declared maximum" in gate["failures"][0] + assert not gate.passed + assert gate.details["declared_strata"][0]["zero_weight_rows"] == 1 + assert "exceed the declared maximum" in gate.failures[0] def test_missing_zero_weight_selector_columns_fail_even_with_positive_weights() -> None: @@ -229,13 +158,13 @@ def test_missing_zero_weight_selector_columns_fail_even_with_positive_weights() inplace=True, ) - gate = _gates(_report(dataset))["zero_weight_strata"] + gate = uk_zero_weight_strata_gate(dataset.household) - assert gate["passed"] is False + assert not gate.passed assert all( - row["missing_selector_columns"] for row in gate["details"]["declared_strata"] + row["missing_selector_columns"] for row in gate.details["declared_strata"] ) - assert "selector column(s) are missing" in gate["failures"][0] + assert "selector column(s) are missing" in gate.failures[0] def test_default_declarations_name_both_june_100k_zero_strata() -> None: @@ -259,173 +188,60 @@ def test_ess_collapse_produces_named_finding() -> None: weights = np.ones(200, dtype=float) weights[0] = 10_000.0 - gate = _gates(_report(_dataset(n=200, weights=weights)))["weight_ess"] + gate = uk_weight_ess_gate( + weights, + minimum_ess_fraction=TEST_MIN_ESS_FRACTION, + ) - assert gate["passed"] is False - assert gate["details"]["ess_fraction"] < 0.01 - assert "ESS fraction" in gate["failures"][0] + assert not gate.passed + assert gate.details["ess_fraction"] < TEST_MIN_ESS_FRACTION + assert "ESS fraction" in gate.failures[0] def test_ratio_blowout_produces_named_finding() -> None: - report = _report( - _dataset(weights=[1.0, 1.0, 1.0, 20.0]), + gate = uk_weight_ratio_gate( + [1.0, 1.0, 1.0, 20.0], maximum_max_to_median_ratio=10.0, ) - gate = _gates(report)["weight_ratio"] - - assert gate["passed"] is False - assert gate["details"]["max_to_median_positive_weight"] == 20.0 - assert "Max/positive-median" in gate["failures"][0] - -def test_ratio_boundary_is_pinned_to_the_certified_june_measurement() -> None: - assert UK_MAX_TO_MEDIAN_WEIGHT_RATIO == 1_151.2542195939373 + assert not gate.passed + assert gate.details["max_to_median_positive_weight"] == 20.0 + assert "Max/positive-median" in gate.failures[0] def test_certified_june_weight_ratio_passes_at_the_inclusive_boundary() -> None: - # Ratio-preserving replay of the certified June H5 (sha256 f17306ccb2aad7ff - # 0130be3589b560afb2e2a12a943570911cd0c77f07934833). The gate observes - # only the positive median and maximum, so the full 535,080-row vector is - # trimmed to those real values plus a representative shipped zero. + # Ratio-preserving replay of the certified June H5. The gate observes only + # the positive median and maximum, so the full vector is trimmed to those + # real values plus a representative shipped zero. positive_median = 16.202157974243164 maximum = 18_652.802734375 - gate = uk_weight_ratio_gate([0.0, positive_median, positive_median, maximum]) + gate = uk_weight_ratio_gate( + [0.0, positive_median, positive_median, maximum], + maximum_max_to_median_ratio=TEST_MAX_TO_MEDIAN_WEIGHT_RATIO, + ) assert gate.details["max_to_median_positive_weight"] == ( - UK_MAX_TO_MEDIAN_WEIGHT_RATIO + TEST_MAX_TO_MEDIAN_WEIGHT_RATIO ) assert gate.details["maximum_max_to_median_ratio"] == ( - UK_MAX_TO_MEDIAN_WEIGHT_RATIO + TEST_MAX_TO_MEDIAN_WEIGHT_RATIO ) assert gate.passed def test_immediate_nextafter_weight_ratio_fails_with_distinct_full_precision() -> None: - just_above = math.nextafter(UK_MAX_TO_MEDIAN_WEIGHT_RATIO, math.inf) + just_above = math.nextafter(TEST_MAX_TO_MEDIAN_WEIGHT_RATIO, math.inf) - gate = uk_weight_ratio_gate([0.0, 1.0, 1.0, just_above]) + gate = uk_weight_ratio_gate( + [0.0, 1.0, 1.0, just_above], + maximum_max_to_median_ratio=TEST_MAX_TO_MEDIAN_WEIGHT_RATIO, + ) assert just_above == 1_151.2542195939375 assert gate.details["max_to_median_positive_weight"] == just_above assert not gate.passed assert repr(just_above) in gate.failures[0] - assert repr(UK_MAX_TO_MEDIAN_WEIGHT_RATIO) in gate.failures[0] - - -def test_gate_evaluation_error_does_not_mask_later_findings() -> None: - report = uk_terminal_gate_report( - _frame_of(_dataset(signal=0.0)), - object(), - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - input_coverage_evaluator=lambda: (_ for _ in ()).throw( - RuntimeError("seeded coverage crash") - ), - ) - gates = _gates(report) - - assert list(gates) == [ - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - ] - assert gates["uk_release_input_coverage"]["passed"] is False - assert gates["degenerate_release_surface"]["passed"] is False - assert gates["weight_ratio"]["passed"] is True - - -def test_malformed_release_surface_is_refused_before_any_gate_runs() -> None: - """The report takes the validated Frame carrier (#611 A4): a duck record - that lost its household table is unrepresentable, so the refusal happens - loudly at the evidence seam instead of producing the all-gates-failed - batch the duck-typed report used to return.""" - - dataset = _dataset() - del dataset.household - - with pytest.raises(TypeError, match="must be a Frame"): - uk_terminal_gate_report( - dataset, - object(), - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - input_coverage_evaluator=lambda: _coverage(passed=False), - ) - orphaned = _dataset(n=2) - orphaned.benunit = pd.DataFrame({"benunit_id": [201, 202, 999]}) - with pytest.raises(ValueError, match="referenced by no person"): - _frame_of(orphaned) - - -def test_bad_optional_evidence_is_contained_by_each_named_gate() -> None: - def broken_records(): - raise RuntimeError("seeded record materialization failure") - yield # pragma: no cover - - report = _report( - fit_weight_records=broken_records(), - parity_evidence=object(), - ) - gates = _gates(report) - - assert gates["weights_audit"]["passed"] is False - assert ( - "seeded record materialization failure" in gates["weights_audit"]["failures"][0] - ) - for name in ("export_surface", "target_surface", "target_fit"): - assert gates[name]["passed"] is False - assert "must be UKReleaseParityEvidence" in gates[name]["failures"][0] - - -def test_fit_audit_is_absent_without_evidence_and_required_missing_fails() -> None: - absent = _gates(_report()) - required = _gates(_report(require_fit_weight_records=True)) - - assert "weights_audit" not in absent - assert required["weights_audit"]["passed"] is False - assert required["weights_audit"]["details"]["evidence_missing"] is True - - -def test_fit_audit_uses_real_records_and_rejects_unweighted_fit() -> None: - passing = _gates( - _report(fit_weight_records=(FitWeightRecord("spi_qrf", "importance"),)) - ) - failing = _gates(_report(fit_weight_records=(FitWeightRecord("spi_qrf", "none"),))) - - assert passing["weights_audit"]["passed"] is True - assert failing["weights_audit"]["passed"] is False - assert failing["weights_audit"]["details"]["unweighted_fits"] == ["spi_qrf"] - - -def test_parity_trio_is_absent_without_evidence_and_present_with_evidence() -> None: - absent = _gates(_report()) - evidence = UKReleaseParityEvidence( - candidate_columns={"person.age"}, - reference_columns={"person.age"}, - candidate_targets={"ons/population"}, - reference_targets={"ons/population"}, - target_relative_errors={"ons/population": 0.01}, - ) - present = _gates(_report(parity_evidence=evidence)) - - assert {"export_surface", "target_surface", "target_fit"}.isdisjoint(absent) - assert all( - present[name]["passed"] - for name in ("export_surface", "target_surface", "target_fit") - ) - - -def test_parity_evidence_must_be_complete_and_nonvacuous() -> None: - with pytest.raises(ValueError, match="exactly cover candidate_targets"): - UKReleaseParityEvidence( - candidate_columns={"person.age"}, - reference_columns={"person.age"}, - candidate_targets={"ons/population"}, - reference_targets={"ons/population"}, - target_relative_errors={"different": 0.01}, - ) + assert repr(TEST_MAX_TO_MEDIAN_WEIGHT_RATIO) in gate.failures[0] def test_ported_june_parity_gates_retain_their_named_failures() -> None: @@ -460,18 +276,6 @@ def test_ported_june_parity_gates_reject_empty_evidence() -> None: assert "evidence is empty" in " ".join(fit.failures) -def test_unevidenced_gates_are_omitted_not_stubbed_as_passes() -> None: - """Item-4 gates joined the battery evidence-gated; item 5 remains future.""" - - names = set(_gates(_report())) - - assert { - "input_mass_parity", - "qrf_tail_concentration", - "delivered_take_up", - }.isdisjoint(names) - - def _input_mass_reference(totals=None) -> UKInputMassReference: return UKInputMassReference( totals=({"employment_income": 10.0} if totals is None else totals), @@ -494,449 +298,80 @@ def _qrf_tail_policy(**overrides) -> UKQRFTailConcentrationPolicy: return UKQRFTailConcentrationPolicy(**fields) -def test_armed_weighted_integrity_gates_join_membership_and_attestation() -> None: - """Removing either armed gate from the battery must fail this test.""" - - report = _report( - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(), - qrf_tail_policy=_qrf_tail_policy(), - ) - gates = _gates(report) - - assert list(gates) == [ - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - "input_mass_parity", - "qrf_tail_concentration", - ] - assert gates["input_mass_parity"]["passed"] is True - assert set(report.evidence_sha256) == { - "release_dataset", - "input_mass_parity", - "qrf_tail_concentration", - } - assert report.attestation["evaluated_gates"][-2:] == [ - "input_mass_parity", - "qrf_tail_concentration", - ] +def _input_mass_gate(candidate_totals, reference=None, *, policy=None): + with patch(VALIDATE_REFERENCE, return_value=None): + return uk_input_mass_parity_gate( + candidate_totals, + _input_mass_reference() if reference is None else reference, + policy=_input_mass_policy() if policy is None else policy, + ) -def test_zeroed_input_column_fails_the_armed_battery_by_name() -> None: - report = _report( - _dataset(signal=0.0), - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(), - ) - gate = _gates(report)["input_mass_parity"] +def test_zeroed_input_column_fails_by_name() -> None: + gate = _input_mass_gate({"employment_income": 0.0}) - assert not report.passed - assert gate["passed"] is False - assert "employment_income" in gate["failures"][0] - assert "mass is zero" in gate["failures"][0] - assert gate["details"]["reference_identity"]["filename"] == ( + assert not gate.passed + assert gate.name == "input_mass_parity" + assert "employment_income" in gate.failures[0] + assert "mass is zero" in gate.failures[0] + assert gate.details["reference_identity"]["filename"] == ( "enhanced_frs_2023_24.h5" ) -def test_999_permille_mass_loss_fails_the_armed_battery_by_name() -> None: - report = _report( - _dataset(signal=[0.001, 0.002, 0.003, 0.004]), - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(), - ) - gate = _gates(report)["input_mass_parity"] - - assert gate["passed"] is False - assert "employment_income" in gate["failures"][0] - assert "-99.9%" in gate["failures"][0] - - -def test_concentrated_qrf_output_fails_the_armed_battery_by_name() -> None: - n = 10 - values = np.ones(n) - values[0] = 1_000.0 - dataset = _dataset(n=n) - dataset.person["self_employment_income"] = values - - report = _report(dataset, qrf_tail_policy=_qrf_tail_policy()) - gate = _gates(report)["qrf_tail_concentration"] - - assert not report.passed - assert gate["passed"] is False - assert "self_employment_income" in gate["failures"][0] - assert gate["details"]["surface"]["declared_qrf_outputs"] >= 47 - - -def test_armed_gate_without_reference_or_thresholds_fails_closed() -> None: - missing_reference = _gates(_report(input_mass_policy=_input_mass_policy()))[ - "input_mass_parity" - ] - missing_policy = _gates(_report(input_mass_reference=_input_mass_reference()))[ - "input_mass_parity" - ] - - assert missing_reference["passed"] is False - assert "no UKInputMassReference" in missing_reference["failures"][0] - assert missing_policy["passed"] is False - assert "#609 measurement pass" in missing_policy["failures"][0] - - -def test_weighted_integrity_evaluator_crash_does_not_mask_pending_gates() -> None: - report = _report( - input_mass_reference=object(), - input_mass_policy=_input_mass_policy(), - qrf_tail_policy=object(), - ) - gates = _gates(report) +def test_999_permille_mass_loss_fails_by_name() -> None: + gate = _input_mass_gate({"employment_income": 0.01}) - assert gates["input_mass_parity"]["passed"] is False - assert "must be UKInputMassReference" in gates["input_mass_parity"]["failures"][0] - assert gates["qrf_tail_concentration"]["passed"] is False - assert ( - "must be UKQRFTailConcentrationPolicy" - in gates["qrf_tail_concentration"]["failures"][0] - ) - # Pending gates still evaluated and reported (#547). - assert gates["weight_ratio"]["passed"] is True - - -def test_weighted_integrity_thresholds_are_bound_into_the_policy_digest() -> None: - unarmed = _report() - armed = _report( - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(), - ) - retuned = _report( - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(relative_tolerance=0.25), - ) - - digests = { - report.attestation["policy_sha256"] for report in (unarmed, armed, retuned) - } - assert len(digests) == 3 + assert not gate.passed + assert "employment_income" in gate.failures[0] + assert "-99.9%" in gate.failures[0] -def test_stale_weighted_integrity_exclusions_fail_the_armed_battery() -> None: - report = _report( - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy( +def test_stale_weighted_integrity_exclusions_fail() -> None: + gate = _input_mass_gate( + {"employment_income": 10.0}, + policy=_input_mass_policy( reviewed_exclusions={ "employment_income": _entry("Seeded stale entry."), } ), ) - gate = _gates(report)["input_mass_parity"] - assert gate["passed"] is False - assert "Stale reviewed input-mass exclusions" in gate["failures"][0] - - -def test_terminal_report_writer_round_trips_strict_atomic_json(tmp_path) -> None: - report = _report() - output = tmp_path / "terminal_gates.json" - - written = write_uk_terminal_gate_report(report, output) - payload = json.loads(written.read_text(encoding="utf-8")) - - assert payload["schema_version"] == 3 - assert payload["enforced"] is True - assert payload["passed"] is True - assert payload["gates"] == _gates(report) - attestation = payload["attestation"] - assert attestation["schema_version"] == 5 - assert attestation["producer"] == UK_TERMINAL_GATE_PRODUCER - assert attestation["release_id"] == TEST_UK_RELEASE_ID - assert ( - attestation["calibration_diagnostics_sha256"] - == TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256 - ) - assert attestation["policy_sha256"] != uk_terminal_gate_policy_sha256() - assert attestation["evaluated_gates"] == [ - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - ] - assert set(attestation["evidence_sha256"]) == {"release_dataset"} - assert len(attestation["gate_results_sha256"]) == 64 - assert attestation["signature_algorithm"] == UK_TERMINAL_GATE_SIGNATURE_ALGORITHM - assert len(attestation["signing_key_sha256"]) == 64 - assert len(attestation["signature"]) == 64 - unsigned_report = { - **payload, - "attestation": { - key: value for key, value in attestation.items() if key != "signature" - }, - } - expected_signature = hmac.new( - base64.b64decode(TEST_UK_TERMINAL_GATE_SIGNING_KEY), - json.dumps( - unsigned_report, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode(), - hashlib.sha256, - ).hexdigest() - assert hmac.compare_digest(attestation["signature"], expected_signature) - assert list(tmp_path.glob(".terminal_gates.json.*.tmp")) == [] - - -def test_terminal_report_writer_persists_before_missing_signing_key_raise( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.delenv(UK_TERMINAL_GATE_SIGNING_KEY_ENV) - report = _report() - output = tmp_path / "terminal_gates.json" - - assert report.passed is False - assert report.passed == report.report_payload()["passed"] - with pytest.raises(RuntimeError, match="Unsigned failed report was written"): - write_uk_terminal_gate_report(report, output) - - payload = json.loads(output.read_text(encoding="utf-8")) - assert payload["passed"] is False - assert payload["attestation"]["signature"] is None - assert payload["attestation"]["signing_key_sha256"] is None - - -@pytest.mark.parametrize( - ("encoded_key", "match"), - [ - ("not-base64!", "must be valid base64"), - (base64.b64encode(b"x" * 31).decode(), "exactly 32 bytes"), - ], -) -def test_terminal_report_signer_rejects_malformed_or_wrong_length_key( - monkeypatch, - tmp_path, - encoded_key: str, - match: str, -) -> None: - monkeypatch.setenv(UK_TERMINAL_GATE_SIGNING_KEY_ENV, encoded_key) - report = _report() - output = tmp_path / "terminal_gates.json" - - with pytest.raises(RuntimeError, match=match): - write_uk_terminal_gate_report(report, output) - - payload = json.loads(output.read_text(encoding="utf-8")) - assert payload["passed"] is False - assert payload["attestation"]["signature"] is None - assert payload["attestation"]["signing_key_sha256"] is None - - -def test_terminal_signature_canonicalizes_dict_order_and_float_formatting() -> None: - from microcosm.build.uk_runtime import terminal_gates - - left = json.loads('{"z":1e0,"nested":{"beta":2.50,"alpha":3.0}}') - right = json.loads('{"nested":{"alpha":3e0,"beta":25e-1},"z":1.0}') - key = base64.b64decode(TEST_UK_TERMINAL_GATE_SIGNING_KEY) - - assert terminal_gates._canonical_json_bytes(left) == ( - terminal_gates._canonical_json_bytes(right) - ) - assert bytes.fromhex(terminal_gates._terminal_gate_signature(key, left)) == ( - bytes.fromhex(terminal_gates._terminal_gate_signature(key, right)) - ) + assert not gate.passed + assert "Stale reviewed input-mass exclusions" in gate.failures[0] -def test_terminal_report_writer_rejects_sol_composed_raw_parity_trio( - tmp_path, -) -> None: - report = GateReport( - ( - uk_export_surface_gate({"person.age"}, {"person.age"}), - uk_target_surface_gate({"ons/population"}, {"ons/population"}), - uk_target_fit_gate({"ons/population": 0.0}), +def test_weighted_integrity_type_errors_are_named() -> None: + with pytest.raises(TypeError, match="reference must be UKInputMassReference"): + uk_input_mass_parity_gate( + {"employment_income": 10.0}, + object(), + policy=_input_mass_policy(), ) - ) - assert report.passed - output = tmp_path / "terminal_gates.json" - - with pytest.raises(TypeError, match="returned by uk_terminal_gate_report"): - write_uk_terminal_gate_report(report, output) - - assert not output.exists() - - -def test_private_constructor_cannot_mint_sol_raw_parity_trio() -> None: - """Even importing underscored internals cannot omit mandatory gates.""" - - from microcosm.build.uk_runtime import terminal_gates - - results = ( - uk_export_surface_gate({"person.age"}, {"person.age"}), - uk_target_surface_gate({"ons/population"}, {"ons/population"}), - uk_target_fit_gate({"ons/population": 0.0}), - ) - - with pytest.raises(ValueError, match="membership must follow"): - terminal_gates._AttestedUKTerminalGateReport( - results, - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - policy_sha256=uk_terminal_gate_policy_sha256(), - evidence_sha256={ - "release_dataset": "a" * 64, - "release_parity": "b" * 64, - }, - attestation={}, - _signing_error=None, + with pytest.raises(TypeError, match="policy must be UKQRFTailConcentrationPolicy"): + uk_qrf_tail_concentration_gate( + {"self_employment_income": [1.0, 2.0]}, + {"self_employment_income": [1.0, 1.0]}, + policy=object(), ) -def test_private_constructor_cannot_drop_evidenced_weighted_integrity_gates() -> None: - """An attested report naming increment-4 evidence must carry the gates.""" - - from microcosm.build.uk_runtime import terminal_gates - - healthy = _report( - input_mass_reference=_input_mass_reference(), - input_mass_policy=_input_mass_policy(), - qrf_tail_policy=_qrf_tail_policy(), - ) - trimmed_results = tuple( - result - for result in healthy.results - if result.name not in ("input_mass_parity", "qrf_tail_concentration") - ) - - with pytest.raises(ValueError, match="membership must follow"): - terminal_gates._AttestedUKTerminalGateReport( - trimmed_results, - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - policy_sha256=uk_terminal_gate_policy_sha256(), - evidence_sha256=dict(healthy.evidence_sha256), - attestation={}, - _signing_error=None, - ) - - -def test_terminal_report_writer_cannot_resign_aggregator_output( - monkeypatch, - tmp_path, -) -> None: - """The public persistence seam is not a signing oracle for caller input.""" - - report = _report() - original_key_id = report.attestation["signing_key_sha256"] - original_signature = report.attestation["signature"] - monkeypatch.setenv( - UK_TERMINAL_GATE_SIGNING_KEY_ENV, - base64.b64encode(b"x" * 32).decode(), - ) +def test_concentrated_qrf_output_fails_by_name() -> None: + values = np.ones(10) + values[0] = 1_000.0 - output = write_uk_terminal_gate_report( - report, - tmp_path / "terminal_gates.json", + gate = uk_qrf_tail_concentration_gate( + {"self_employment_income": values}, + {"self_employment_income": np.ones(10)}, + policy=_qrf_tail_policy(), ) - payload = json.loads(output.read_text(encoding="utf-8")) - - assert payload["attestation"]["signing_key_sha256"] == original_key_id - assert payload["attestation"]["signature"] == original_signature - - -def test_production_terminal_report_pins_policy_and_evidence_membership( - monkeypatch, - tmp_path, -) -> None: - from microcosm.build.uk_runtime import terminal_gates - monkeypatch.setattr( - terminal_gates, - "uk_release_input_coverage_gate", - lambda _dataset, _engine: _coverage(), - ) - evidence = UKReleaseParityEvidence( - candidate_columns={"person.age"}, - reference_columns={"person.age"}, - candidate_targets={"ons/population"}, - reference_targets={"ons/population"}, - target_relative_errors={"ons/population": 0.01}, - ) - report = uk_terminal_gate_report( - _frame_of(_dataset()), - object(), - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - fit_weight_records=(FitWeightRecord("spi_qrf", "importance"),), - parity_evidence=evidence, - ) - output = write_uk_terminal_gate_report( - report, - tmp_path / "terminal_gates.json", - ) - payload = json.loads(output.read_text(encoding="utf-8")) - - # The #630 source_year reviewed exclusion entered the committed register - # and #610's schema 2 sealed its full approval receipt (approver, - # adjudication, dates), so the frozen digest moved — the intended - # tripwire; the pre-#630 digest stays pinned in microcosm-data for the - # grandfathered June release. - assert uk_terminal_gate_policy_sha256() == ( - "ae93bd10a02362a523eb077bcbd32b362cef31f0447acbc40537df696e30c757" - ) - assert payload["attestation"]["policy_sha256"] == (uk_terminal_gate_policy_sha256()) - assert payload["attestation"]["evaluated_gates"] == [ - "uk_release_input_coverage", - "degenerate_release_surface", - "zero_weight_strata", - "weight_ess", - "weight_ratio", - "weights_audit", - "export_surface", - "target_surface", - "target_fit", - ] - assert set(payload["attestation"]["evidence_sha256"]) == { - "release_dataset", - "hmrc_spi_income", - "release_parity", - } - weight_details = payload["gates"]["weight_ratio"]["details"] - weight_fields = ( - "n_records", - "positive_weight_records", - "zero_weight_records", - "total_weight", - "effective_sample_size", - "ess_fraction", - "median_positive_weight", - "max_weight", - "max_to_median_positive_weight", - "top_1pct_weight_share", - ) - release_evidence = { - "weights": {field: weight_details[field] for field in weight_fields} - } - encoded = json.dumps( - release_evidence, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode() - assert payload["attestation"]["evidence_sha256"]["release_dataset"] == ( - hashlib.sha256(encoded).hexdigest() - ) + assert not gate.passed + assert gate.name == "qrf_tail_concentration" + assert "self_employment_income" in gate.failures[0] def test_committed_degenerate_register_is_the_policy_of_record() -> None: - """None resolves to the committed #630 register; the digest seals it. - - The register asserts the structural approval receipt only — never the - reason prose, which must stay freely editable (any edit still moves the - frozen digest, so rewording is visible without a prose pin here). - """ - register = uk_default_degenerate_reviewed_exclusions() assert set(register) == {"household.source_year"} record = register["household.source_year"] @@ -948,31 +383,15 @@ def test_committed_degenerate_register_is_the_policy_of_record() -> None: def test_policy_of_record_is_immutable_and_loaded_once() -> None: - """The default register cannot drift from the already-computed digest. - - A mutable module-level mapping would let any caller (or a sloppy test) - change the policy of record for the rest of the process while the frozen - digest kept attesting the committed one — the exact failure the digest - exists to prevent. The accessor returns one cached read-only mapping, and - loading is lazy so a broken committed register surfaces as this call's - ValueError rather than an ImportError at module import. - """ - register = uk_default_degenerate_reviewed_exclusions() assert register is uk_default_degenerate_reviewed_exclusions() with pytest.raises(TypeError): register["household.source_year"] = None # type: ignore[index] with pytest.raises(AttributeError): register.pop # noqa: B018 - MappingProxyType exposes no mutators - assert uk_terminal_gate_policy_sha256() == uk_terminal_gate_policy_sha256() def test_expired_degenerate_exclusion_fails_with_renewal_context() -> None: - """Honored through expires_on; strictly after, the combined message names - the approver, the adjudication, and the lapse date.""" - - from datetime import date - entry = _entry("Fixture broadcast, admitted.") honored = uk_degenerate_release_surface_gate( _dataset(signal=7.0), @@ -995,23 +414,10 @@ def test_expired_degenerate_exclusion_fails_with_renewal_context() -> None: "microcosm#610) — renew the adjudication or remove the entry." in expired.failures[0] ) - # Expired-but-still-degenerate is not stale: the column carries no signal. assert expired.details["stale_exclusions"] == [] def test_out_of_force_exclusions_fail_at_every_column_state() -> None: - """The register cannot rot silently just because its column moved. - - Adversarial-review finding (three independent lenses): an expired entry - whose column was absent (dormant) or had regained signal produced no - failure — the build went green and the published report was only - rejected downstream by the contract's expired_exclusions expectation. - Out-of-force entries now fail the gate at every column state, with - receipt context rather than the stale "remove them" message. - """ - - from datetime import date - after_expiry = date(2027, 2, 11) dormant = uk_degenerate_release_surface_gate( _dataset(signal=7.0), @@ -1032,7 +438,7 @@ def test_out_of_force_exclusions_fail_at_every_column_state() -> None: assert "renew the adjudication or remove the entries" in combined[0] regained = uk_degenerate_release_surface_gate( - _dataset(), # employment_income varies: the column carries signal now + _dataset(), reviewed_exclusions={ "person.employment_income": _entry("Fixture broadcast, admitted.") }, @@ -1040,19 +446,12 @@ def test_out_of_force_exclusions_fail_at_every_column_state() -> None: ) assert not regained.passed assert regained.details["expired_exclusions"] == ["person.employment_income"] - # Receipt context wins over the stale message for out-of-force entries. assert regained.details["stale_exclusions"] == [] assert len(regained.failures) == 1 assert "renew the adjudication" in regained.failures[0] def test_premature_degenerate_exclusion_never_suppresses() -> None: - """A receipt whose approved_on is still in the future is not an - approval: it must not suppress today (adversarial-review finding — a - typo'd future year would have silently suppressed for years).""" - - from datetime import date - before_approval = date(2026, 8, 9) live = uk_degenerate_release_surface_gate( _dataset(signal=7.0), @@ -1078,53 +477,9 @@ def test_premature_degenerate_exclusion_never_suppresses() -> None: def test_exclusion_clocks_reject_datetimes() -> None: - """datetime is a date subclass; letting one through would compare - timestamps against dates or leak a timestamp into - exclusions_evaluated_on (adversarial-review finding).""" - - from datetime import UTC, datetime - with pytest.raises(TypeError, match="must be a datetime.date"): uk_degenerate_release_surface_gate( - _dataset(), reviewed_exclusions={}, now=datetime.now(UTC) - ) - with pytest.raises(TypeError, match="must be a datetime.date"): - _report(now=datetime.now(UTC)) - - -def test_report_seals_the_register_snapshot_it_ran_under() -> None: - """The gate and the attested policy digest must observe one register. - - Adversarial-review finding: the mapping was read once at gate time and - again at digest time, with caller-controlled evaluators running in - between — a mutation there produced an attestation describing a policy - the gate never ran under. The report now coerces and freezes the - mapping once at entry, before any evaluator runs. - """ - - mutable = {"person.employment_income": _entry("Fixture broadcast, admitted.")} - baseline = _report( - _dataset(signal=7.0), - reviewed_degenerate_exclusions=dict(mutable), - ) - - def mutating_coverage(): - mutable.clear() - return _coverage() - - with patch( - "microcosm.build.uk_runtime.weighted_integrity._validate_input_mass_reference", - return_value=None, - ): - mutated = uk_terminal_gate_report( - _frame_of(_dataset(signal=7.0)), - object(), - release_id=TEST_UK_RELEASE_ID, - calibration_diagnostics_sha256=(TEST_UK_CALIBRATION_DIAGNOSTICS_SHA256), - input_coverage_evaluator=mutating_coverage, - reviewed_degenerate_exclusions=mutable, + _dataset(), + reviewed_exclusions={}, + now=datetime.now(UTC), ) - assert mutated.attestation["policy_sha256"] == baseline.attestation["policy_sha256"] - degenerate = _gates(mutated)["degenerate_release_surface"] - assert degenerate["passed"] is True - assert "person.employment_income" in degenerate["details"]["reviewed_exclusions"] diff --git a/packages/microcosm-build/tests/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/test_uk_weighted_integrity.py index 4e5c4936..15323505 100644 --- a/packages/microcosm-build/tests/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/test_uk_weighted_integrity.py @@ -732,10 +732,10 @@ def test_uk_totals_are_the_shared_helper_minus_exported_weight_columns() -> None """The UK wrapper must not reinvent the shared numeric semantics. One frame through both helpers: per-column weighted totals must be - identical, except that the wrapper removes the exported weight column — - ``household_weight`` is a real, engine-known column on the UK frame - (the materialized export contract), so the shared helper totals its - squared mass and only the wrapper can say it is plumbing, not mass. + identical. The in-build UK carrier no longer persists the exported + ``household_weight`` column, so the wrapper's removal is a compatibility + no-op on carrier Frames; export materialization remains the boundary that + writes the column from the typed vector. Anchored to a hand computation once, so the wrapper is pinned to the shared semantics rather than merely to itself. """ @@ -768,14 +768,9 @@ def test_uk_totals_are_the_shared_helper_minus_exported_weight_columns() -> None shared_totals = input_mass_totals(frame) uk_totals = uk_input_mass_totals(frame) - # The wrapper exists exactly for this key: sum of squared weights is not - # input mass. - assert shared_totals["household_weight"] == 2.0**2 + 5.0**2 - assert uk_totals == { - name: total - for name, total in shared_totals.items() - if name != "household_weight" - } + assert "household_weight" not in frame.table("household") + assert "household_weight" not in shared_totals + assert uk_totals == shared_totals # NaN fills to 0, booleans total weighted True mass, weights broadcast # through membership — asserted against hand computation once. assert uk_totals["employment_income"] == 30_000.0 * 2.0 + 12_000.0 * 5.0 diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 4d90473f..5537b0b1 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -329,13 +329,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "2586535bcae393e5d09a01a47bab5a55e310089044e84780d6f5c270a077d006" + "b147b50369e1f8b851f843e89b4a490b0a8c3b6a92e32b6bdb7bf40ca454cd0c" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "6308ee13c1bfaeb9840524b5a70b3ecedf6db0476e1284780508e5489fa3662b" + "6a98915343a7add9f469e9adebde5a9c85fdcbb3ecefee870df7b77c2b658e81" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "48993fd9ed4f8cc41cd6d4063026d40776e634962301ac38e6944bd1cc6927ee" + "da0039af7d84d0dd7c5bac2016aa94dee938c787ff2400743e1b875cb81adfc6" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index ee2cbd21..5e6626cf 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -101,13 +101,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "2586535bcae393e5d09a01a47bab5a55e310089044e84780d6f5c270a077d006" + "b147b50369e1f8b851f843e89b4a490b0a8c3b6a92e32b6bdb7bf40ca454cd0c" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "6308ee13c1bfaeb9840524b5a70b3ecedf6db0476e1284780508e5489fa3662b" + "6a98915343a7add9f469e9adebde5a9c85fdcbb3ecefee870df7b77c2b658e81" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "48993fd9ed4f8cc41cd6d4063026d40776e634962301ac38e6944bd1cc6927ee" + "da0039af7d84d0dd7c5bac2016aa94dee938c787ff2400743e1b875cb81adfc6" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index b723f99e..4e30e739 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -13,8 +13,6 @@ from itertools import combinations from pathlib import Path -import pandas as pd - from microcosm.build.country_spec import country_stage_plan, load_country_spec from microcosm.build.gate_battery import GateBatteryBlockedError from microcosm.build.logbook import canonical_json_bytes @@ -881,11 +879,10 @@ def _main_recording( append_phase(state, "candidate_verified") append_phase(state, "inputs_pinned") # This staging path performs no calibration and therefore has no real - # target-surface or target-fit evidence. Leave parity_evidence absent; - # the terminal report omits that trio instead of inventing passes. - # The weighted-integrity pair (#609) follows the same rule: it joins - # the battery only when the caller arms it with a frozen reference - # and measured thresholds. + # target-surface or target-fit evidence; the schema-4 battery records + # the missing evidence explicitly. The weighted-integrity pair (#609) + # joins only when the caller arms it with a frozen reference and + # measured thresholds. gate_path_argument = ( {"input_coverage_path": legacy_input_coverage_path} if legacy_input_coverage_path is not None @@ -1128,9 +1125,6 @@ def _aggregate_build_record( } for record in result.frame.mass_log ] - household_weights = pd.to_numeric( - result.frame.table("household")["household_weight"], errors="raise" - ) release_evidence = dict(result.gate_report["release_evidence"]) return { "schema_version": 3, @@ -1165,7 +1159,9 @@ def _aggregate_build_record( "household": len(result.frame.table("household")), }, "household_weight_kind": uk_household_weight_kind(result.frame).value, - "household_weight_total": float(household_weights.sum()), + "household_weight_total": float( + result.frame.weights_for("household").total + ), "mass_changes": mass_changes, }, "source_rows": { diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 93eabf56..065dd330 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -63,6 +63,7 @@ validate_geography_coverage, write_geography_crosswalk, ) +from microcosm.frame import engine_tables CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" DATASET_FILENAME_TEMPLATE = "populace_uk_{source_year}_rowwise.h5" @@ -1315,7 +1316,9 @@ def _rowwise_summary( if isinstance(result, UKLadderRowwiseDatasetResult): person = result.frame.table("person") benunit = result.frame.table("benunit") - household = result.frame.table("household") + household = engine_tables(result.frame, weighted_entities=("household",))[ + "household" + ] weight_kind = uk_household_weight_kind(result.frame) mass_log = result.frame.mass_log time_period = uk_time_period(result.frame) diff --git a/tools/measure_uk_weighted_integrity_baselines.py b/tools/measure_uk_weighted_integrity_baselines.py index fd038fec..d9a2e4be 100644 --- a/tools/measure_uk_weighted_integrity_baselines.py +++ b/tools/measure_uk_weighted_integrity_baselines.py @@ -6,7 +6,7 @@ weighted mass share and carrier count for every declared QRF output. This tool produces those numbers so they can be posted on #578 and the gate boundaries set at the measured edge with no discretionary headroom — the -same discipline that pinned ``UK_MAX_TO_MEDIAN_WEIGHT_RATIO``. +same discipline that pinned the schema-4 weight-ratio threshold. It is a diagnostic recorder only: it never gates, and release builds do not run it. Each ``--h5`` must be a UK national single-year artifact (person,