From b2e10cb8a04e74ffe18c9e2a2aa7fd911a099e6b Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 25 Jun 2026 23:34:25 +0200 Subject: [PATCH] Add the MEOS-API pin manifest, generator policy doc, and ecosystem orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEOS-API is the catalog producer (the root of the chain), not a generated binding. Add tools/pin/compose-order.txt (its enrichment/projection PRs), GENERATION.md (the producer framing: run.py reads the MEOS pin headers and emits output/meos-idl.json, the single source of truth every binding consumes), and tools/ecosystem-generate.sh (the turnkey orchestrator: given a pin, build the catalog + libmeos, then drive each binding's own regen-from-pin.sh in dependency order — JVM consumers after the JMEOS jar, PyMEOS after PyMEOS-CFFI). --- GENERATION.md | 47 +++++++++++++++++++++ tools/ecosystem-generate.sh | 82 +++++++++++++++++++++++++++++++++++++ tools/pin/compose-order.txt | 43 +++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 GENERATION.md create mode 100755 tools/ecosystem-generate.sh create mode 100644 tools/pin/compose-order.txt diff --git a/GENERATION.md b/GENERATION.md new file mode 100644 index 0000000..e5b485c --- /dev/null +++ b/GENERATION.md @@ -0,0 +1,47 @@ +# MEOS-API generation — the catalog producer and the ecosystem chain + +MEOS-API is the **root** of the per-binding generator policy: it is the **catalog producer**, +not a generated binding. Every other repo is a projection *of* this catalog. + +## The policy (ecosystem-wide) + +Every MobilityDB language/surface binding is a **pure projection of the MEOS-API catalog**, +and each binding owns its own generator, in its own repo, in a canonical layout. The single +source of truth is the **catalog** this repo produces: `output/meos-idl.json`, generated from +the MEOS C headers. + +## What MEOS-API generates + +`run.py ` parses the MEOS public headers with libclang and emits +`output/meos-idl.json`: every function, struct, and enum with signatures, ownership, shape +(output arrays / nullability), recovered collapsed C types (`bool`/`int64`/`Timestamp`/… +that the preprocessor flattens to `int`), `@ingroup` groups, the `@sqlfn` SQL-name map, and +the portable bare-name aliases. The `generator/` modules project the catalog onto the +language-**agnostic** service contracts (OpenAPI, MCP, the runtime server, the OGC Moving +Features projection) — the surfaces that need no foreign toolchain. Language bindings live in +their own repos and generate from this catalog. + +## The chain (do not invert) + +``` +MobilityDB pin + -> MEOS-API run.py -> output/meos-idl.json (+ libmeos.so built from the same pin) + -> JMEOS (jar) -> { MobilitySpark, MobilityFlink, MobilityKafka } + -> PyMEOS-CFFI -> PyMEOS + -> GoMEOS / MEOS.NET / meos-rs / MobilityDuck / MobilityNebula +``` + +## Turnkey: regenerate the whole ecosystem from a pin + +`tools/ecosystem-generate.sh ` runs the chain in dependency order: build the catalog +(`run.py`) + `libmeos.so` from the pin, then invoke each binding's own +`tools/regen-from-pin.sh` in the order above (the JVM consumers after the JMEOS jar; PyMEOS +after PyMEOS-CFFI). Each binding owns its regeneration; this script only sequences them. See +the script header for the repo + frontier-branch table it drives (each binding's frontier is +recorded in its own `tools/pin/compose-order.txt`). + +## Pinning + +The catalog is reproducible from a MobilityDB `ecosystem-pin-*`: +`MDB_SRC_ROOT= python3 run.py /meos/include`. MEOS-API's own +`tools/pin/compose-order.txt` governs *this repo's* enrichment/projection PR accumulate. diff --git a/tools/ecosystem-generate.sh b/tools/ecosystem-generate.sh new file mode 100755 index 0000000..215cfef --- /dev/null +++ b/tools/ecosystem-generate.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# ecosystem-generate.sh — regenerate the WHOLE MobilityDB binding ecosystem from a pin. +# +# Usage: tools/ecosystem-generate.sh [WORKDIR] +# +# It runs the chain in DEPENDENCY ORDER (the JVM consumers after the JMEOS jar; PyMEOS after +# PyMEOS-CFFI). Each binding OWNS its regeneration (its own tools/regen-from-pin.sh, driven by +# its own tools/pin/compose-order.txt); this script only sequences them and produces the +# catalog + libmeos.so the chain starts from. +# +# GH is the only source of truth: every repo is fetched fresh from GitHub at its FRONTIER +# branch (recorded below + in each binding's compose-order.txt). WORKDIR defaults to a $HOME +# path (NEVER /tmp — it is reaped). +set -euo pipefail + +PIN_REF="${1:?usage: ecosystem-generate.sh [workdir]}" +WORK="${2:-$HOME/ecosystem-gen}" +MDB="${MDB_REPO:-$HOME/src/MobilityDB}" +MEOSAPI="$(cd "$(dirname "$0")/.." && pwd)" # this repo +mkdir -p "$WORK" + +# Repo -> frontier branch (the codegen lives in these OPEN PRs until merged; verified 2026-06-25). +# Once a binding's generator PR merges, switch its frontier to the default branch. +# binding owner/repo frontier-branch +# JMEOS estebanzimanyi/JMEOS feat/facade-surface-22a (PR #28) +# GoMEOS estebanzimanyi/GoMEOS codegen/flat-wrappers-22a (PR #5) +# MEOS.NET estebanzimanyi/MEOS.NET work/portable-aliases (PR #5) +# PyMEOS-CFFI estebanzimanyi/PyMEOS-CFFI bump/meos-1.4 (PR #19) +# PyMEOS estebanzimanyi/PyMEOS feat/oo-dispatch-consumer (PR #95) +# MobilitySpark estebanzimanyi/MobilitySpark feat/generated-dispatch (PR #28) +# MobilityFlink estebanzimanyi/MobilityFlink consolidate/flink-benchmark (PR #31) +# MobilityKafka estebanzimanyi/MobilityKafka consolidate/kafka-benchmark (no PR — topology gap) +# MobilityNebula estebanzimanyi/MobilityNebula feat/nebula-codegen-generator-infra (PR #170) +# meos-rs MobilityDB/meos-rs main (bindgen today; catalog migration pending) +# MobilityDuck MobilityDB/MobilityDuck main (generator is WIP, not yet on GH) + +log() { printf '\n=== %s ===\n' "$*" >&2; } +regen() { # owner/repo frontier -- runs the binding's own regen-from-pin.sh + local slug="$1" branch="$2" dir="$WORK/${1##*/}" + rm -rf "$dir"; git clone --quiet --branch "$branch" "https://github.com/$slug" "$dir" \ + || { echo "SKIP $slug ($branch unavailable)"; return 0; } + if [ -x "$dir/tools/regen-from-pin.sh" ]; then + ( cd "$dir" && CATALOG="$CATALOG" LIBMEOS="$LIBMEOS" JMEOS_JAR="${JMEOS_JAR:-}" \ + tools/regen-from-pin.sh "$PIN" ) || echo "WARN: $slug regen returned non-zero" + else + echo "NOTE: $slug has no tools/regen-from-pin.sh yet (frontier $branch) — see its GENERATION.md" + fi +} + +# ── PHASE 0 — pin -> catalog + libmeos.so (the root every binding consumes) ── +log "PHASE 0: resolve pin + build catalog + libmeos" +git -C "$MDB" fetch origin --tags --force --quiet +PIN="$(git -C "$MDB" rev-parse "${PIN_REF}^{commit}" 2>/dev/null || git -C "$MDB" rev-parse "$PIN_REF")" +PINWT="$WORK/pin"; rm -rf "$PINWT"; git -C "$MDB" worktree add --detach "$PINWT" "$PIN" +( cd "$MEOSAPI" && MDB_SRC_ROOT="$PINWT" python3 run.py "$PINWT/meos/include" ) +export CATALOG="$MEOSAPI/output/meos-idl.json" +# all-families libmeos (the runtime every binding loads); see generation-starts-from-building-so +cmake -S "$PINWT" -B "$PINWT/build-allfam" -DMEOS=ON -DCBUFFER=ON -DJSON=ON -DNPOINT=ON \ + -DPOSE=ON -DRGEO=ON -DQUADBIN=ON -DH3=ON \ + -DH3_INCLUDE_DIR=/usr/include/h3 -DH3_LIBRARY=/usr/lib/x86_64-linux-gnu/libh3.so >/dev/null +cmake --build "$PINWT/build-allfam" --target meos -j"$(nproc)" +export LIBMEOS="$PINWT/build-allfam/libmeos.so" + +# ── PHASE 1 — JMEOS jar + the non-JVM bindings (all consume the catalog) ── +log "PHASE 1: JMEOS jar + non-JVM bindings" +regen estebanzimanyi/JMEOS feat/facade-surface-22a +export JMEOS_JAR="$WORK/JMEOS/jar/JMEOS.jar" # produced by JMEOS's regen-from-pin.sh +regen estebanzimanyi/GoMEOS codegen/flat-wrappers-22a +regen estebanzimanyi/MEOS.NET work/portable-aliases +regen estebanzimanyi/PyMEOS-CFFI bump/meos-1.4 +regen MobilityDB/meos-rs main +regen MobilityDB/MobilityDuck main +regen estebanzimanyi/MobilityNebula feat/nebula-codegen-generator-infra + +# ── PHASE 2 — consumers of phase-1 artifacts ── +log "PHASE 2: PyMEOS (needs CFFI) + JVM stream consumers (need the JMEOS jar)" +regen estebanzimanyi/PyMEOS feat/oo-dispatch-consumer +regen estebanzimanyi/MobilitySpark feat/generated-dispatch +regen estebanzimanyi/MobilityFlink consolidate/flink-benchmark +regen estebanzimanyi/MobilityKafka consolidate/kafka-benchmark + +log "DONE — catalog=$CATALOG libmeos=$LIBMEOS jmeos=$JMEOS_JAR ; per-binding output under $WORK" diff --git a/tools/pin/compose-order.txt b/tools/pin/compose-order.txt new file mode 100644 index 0000000..7316112 --- /dev/null +++ b/tools/pin/compose-order.txt @@ -0,0 +1,43 @@ +# USER-APPROVED-PIN-WRITE — creating MEOS-API's pin manifest (user 2026-06-25, per-binding +# generator policy rollout). New file in the MEOS-API repo, NOT a mutation of MobilityDB's +# pin tooling. +# +# MEOS-API pin — THE canonical, dependency-ordered fold manifest (per-binding policy). +# +# MEOS-API is the ROOT of the chain: it is the catalog PRODUCER, not a generated binding. +# `run.py` reads the MEOS pin's `meos/include` headers and emits `output/meos-idl.json` — the +# single source of truth every binding consumes. This manifest lists MEOS-API's own enrichment +# + projection PRs. (policy: generator-per-binding-canonical-policy) +# +# Format: # role. '?' = membership/order UNCONFIRMED. +# base = current origin/master. Derived from the live DAG (gh pr list, this turn). + +# ── WAVE 0 — CATALOG CORRECTNESS (parser / type recovery / enrichment) ── +2 feat/shape-metadata # shape annotations in meos-meta.json for binding codegens +16 feat/generate-shape-from-headers # generate fn shape (output arrays + nullability) from headers +20 fix/recover-jsonb-jsonpath-types # recover Jsonb/JsonPath collapsed C types +22 fix/recover-quadbin-type # recover Quadbin collapsed C type +17 feat/sql-name-chain # MEOS-C -> MobilityDB-C -> SQL name chain from @csqlfn/@sqlfn/@sqlop +18 feat/sqlfn-name-map # attach the @sqlfn SQL-name map to the catalog +19 feat/portable-aliases-comparison-families # camelCase comparison families (tempEq/everEq/alwaysEq) +21 feat/catalog-doxygen-groups # attach the doxygen @ingroup module group to the catalog +4 feat/service-enrichment # derive service-projection metadata (category/network/wire) + +# ── WAVE 1 — CATALOG PROJECTIONS (language-AGNOSTIC; these legitimately live in MEOS-API) ── +5 feat/openapi-generator # OpenAPI 3.1 contract from the enriched catalog +6 feat/mcp-generator # MCP tool manifest from the enriched catalog +13 feat/movfeat-openapi-generator # OGC API – Moving Features OpenAPI projection +10 feat/object-model # explicit MEOS object model (class lattice, methods, error contract) +7 feat/runtime-server # contract-driven runtime HTTP server with a pluggable MEOS engine +12 feat/openapi-validator-gate # CI: openapi-validate workflow + +# ── WAVE 2 — INTEGRATION / DOCS ── +14 feat/integration-train # Waves 0–5 dependency-ordered manifest + verifier +9 docs/cross-repo-handoff # cross-repo handoff brief for consuming meos-api.json + +# ════════════════════════════════════════════════════════════════════════════════════ +# Language BINDINGS (PyMEOS/JMEOS/GoMEOS/MEOS.NET/meos-rs/Duck/Spark/Flink/Kafka/Nebula) are +# NOT in this manifest — each is a separate repo with its OWN compose-order.txt, generated FROM +# this catalog. The dependency-ordered chain that regenerates the whole ecosystem from a pin is +# `tools/ecosystem-generate.sh`. See GENERATION.md. +# ════════════════════════════════════════════════════════════════════════════════════