MobilityDB · estebanzimanyi · Jul 16, 2026 · Jul 16, 2026
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -20,22 +20,25 @@ jobs:
           java-version: "21"
           cache: maven
 
-      # Read the pinned MobilityDB commit from the single recorded surface file
-      # so no SHA is duplicated across this workflow.
-      - name: Read the pinned MobilityDB commit
-        id: meos
-        run: echo "sha=$(tr -d '[:space:]' < tools/meos-source-commit.txt)" >> "$GITHUB_OUTPUT"
-
-      # Build and install libmeos (all families, + spatial_ref_sys.csv /
-      # ways1000.csv under /usr/local/share) through the shared MEOS build
-      # recipe instead of an inline copy of it. build-jmeos.sh then only builds
-      # the JMEOS jar against this installed libmeos.
-      - name: Build and install libmeos via the shared provision-meos action
+      # Derive the MEOS catalog (meos-idl.json via run.py) and build+install the
+      # all-families libmeos.so (+ spatial_ref_sys.csv / ways1000.csv under
+      # /usr/local/share) from upstream MobilityDB master through the shared
+      # provision-meos action. Tracking master (not a pinned commit) keeps the
+      # source, the catalog, the JMEOS jar and the libmeos the tests load all moving
+      # together, so the generated facades can never drift from the surface they run
+      # against. build-jmeos.sh then only builds the JMEOS jar against this libmeos.
+      - name: Provision MEOS catalog + libmeos
+        id: provision
         uses: MobilityDB/MEOS-API/.github/actions/provision-meos@master
         with:
-          mobilitydb-ref: ${{ steps.meos.outputs.sha }}
+          mobilitydb-ref: master
           build-libmeos: "true"
 
+      - name: Stage the derived MEOS catalog
+        # codegen_jvm.py reads tools/meos-idl.json; place the derived catalog there.
+        # The catalog is gitignored — it only ever exists as a build artifact.
+        run: cp "${{ steps.provision.outputs.catalog-path }}" tools/meos-idl.json
+
       - name: Build JMEOS.jar against the installed libmeos
         # libmeos is already built and installed under /usr/local by the
         # provision-meos action, so build-jmeos.sh skips its own libmeos build

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,10 @@
 # Maven build artefacts
 target/
 
+# Derived generator input — the MEOS catalog is staged in CI from upstream
+# MobilityDB master (provision-meos), never committed.
+tools/meos-idl.json
+
 # JMEOS native dependency — generated by build-jmeos.sh, never committed
 .build-jmeos/
 kafka-streams-app/jar/

diff --git a/GENERATION.md b/GENERATION.md
@@ -6,57 +6,57 @@ per-binding generator policy.
 ## The policy (ecosystem-wide)
 
 Every MobilityDB language/surface binding is a **pure projection of the MEOS-API catalog**,
-and **each binding owns its own generator, in its own repo**, in a canonical layout. The
-single source of truth is the **MEOS C API** (via the MEOS-API catalog `meos-idl.json`,
-generated from the MEOS headers). A binding is an independent, plug-and-play module that
-owns its generation.
+and the JVM bindings **share one generator**. The single source of truth is the **MEOS C
+API** (via the MEOS-API catalog `meos-idl.json`, generated from the MEOS headers). A binding
+is an independent, plug-and-play module that owns its generation.
 
-Each binding repo satisfies the same invariants: in-repo generator; catalog/jar input from
-a specific MobilityDB commit; thin language projection; no committed native binaries.
+Each binding repo satisfies the same invariants: the shared generator; catalog/jar input
+derived in CI from upstream MobilityDB master; thin language projection; no committed native
+binaries; no committed generated sources.
 
 ## MobilityKafka scope: generated MEOS facades over the JMEOS surface
 
 MobilityKafka is a **consumer** binding: it binds the **JMEOS jar** (the JVM FFI projection
-of the catalog), not MEOS-API directly. Its generator **`tools/codegen_facades.py`** reads
-the JMEOS raw-FFI surface (intersected with the streaming-relevance baseline) and emits the
-`org.mobilitydb.meos.MeosOps*` 1:1 forwarder facades the Kafka Streams app consumes
-(`--engine kafka`). The facades are a *consumer* projection — they live here, not in JMEOS,
-so the JMEOS FFI line and the facade line do not diverge.
+of the catalog), not MEOS-API directly. Its generator is the shared
+**`tools/codegen_jvm.py --engine kafka`**, the single generator vendored identically by every
+JVM binding (MobilitySpark, MobilityFlink, MobilityKafka); the `flink` and `kafka` engines
+emit the `org.mobilitydb.meos.MeosOps*` 1:1 forwarder facades the Kafka Streams app consumes.
+The facades are a *consumer* projection — they live here, not in JMEOS, so the JMEOS FFI line
+and the facade line do not diverge.
+
+## Full surface, grouped by the catalog object model
+
+`codegen_jvm.py --engine kafka` emits a facade for **every** function on the bundled JMEOS
+`functions.GeneratedFunctions` surface, grouped by the MEOS-API catalog object model: one
+`MeosOps<Class>` per object-model class plus one `MeosOpsFree<Header>` per source header for
+the free functions, with a shared `MeosOpsRuntime` that probes libmeos once per JVM. Each
+forwarder carries a runtime guard: functions whose catalog return type is sequence-typed
+(build a whole `TSequence`/`SeqSet`, inherently non-streamable) throw
+`UnsupportedOperationException`; all others forward to `GeneratedFunctions` behind the
+`MEOS_AVAILABLE` probe. The class/role/header are read straight from the catalog's
+`objectModel`, and the sequence check from `returnType.canonical` — no separate classifier.
+
+The `MeosOps*` facades are emitted at build time and are **not committed**: Maven
+`generate-sources` runs `tools/codegen_jvm.py --engine kafka` into `target/generated-facades`,
+and `build-helper` adds it as a source root. The sole hand-written class under
+`org.mobilitydb.meos` is `MeosSetSetJoin`.
 
 ## The build chain — no committed binaries
 
-`build-jmeos.sh` reproduces the whole native/JVM chain, so the repository carries no jar or
+`build-jmeos.sh` reproduces the native/JVM chain, so the repository carries no jar or
 `libmeos.so`:
 
 ```
-MobilityDB @ tools/meos-source-commit.txt
-  → build-jmeos.sh: cmake -DMEOS=ON            → libmeos.so   (families CBUFFER/NPOINT/POSE)
-  → build-jmeos.sh: JMEOS main                 → JMEOS.jar    → installed com.mobilitydb:jmeos:1.4.0
-  → tools/codegen_facades.py --engine kafka    → org.mobilitydb.meos.MeosOps* facades
+MobilityDB @ master
+  → provision-meos (MEOS-API/run.py + cmake -DMEOS=ON -DALL) → meos-idl.json + libmeos.so
+  → build-jmeos.sh: JMEOS main                               → JMEOS.jar → com.mobilitydb:jmeos:1.4.0
+  → tools/codegen_jvm.py --engine kafka  (full jar surface)  → org.mobilitydb.meos.MeosOps* facades
   → kafka-streams-app  (mvn test)
 ```
 
-`build-jmeos.sh` tracks upstream MobilityDB master and MobilityDB/JMEOS main (recorded as
-immutable head SHAs, overridable via the environment). The `libmeos.so` it builds must come
-from the **same commit** the JMEOS facade surface was generated against — surface-match, else
-runtime symbol faults; only the symbols the app calls must be present (jnr resolves lazily).
-
-## The streaming-relevance baseline (generator input)
-
-`tools/codegen_facades.py` emits a facade only for functions in the **streaming-relevant**
-tiers, read from `tools/baseline/streaming-relevance-baseline.json`. That baseline is itself
-**generated and reproducible** — it is not hand-maintained. It is produced by
-**`tools/classify_streaming_relevance.py`**, a deterministic classifier: the tier of a
-function is decided purely by its name, its object-model role, and its number of temporal
-parameters (zero per-function judgement), so the same MEOS catalog always yields the same
-baseline.
-
-To refresh the baseline (e.g. after bumping `tools/meos-source-commit.txt`): rebuild the
-catalog at the tracked commit with MEOS-API, then
-
-```
-tools/regen_baseline.sh <path-to-meos-idl.json>
-```
-
-and commit the diff. Because the classifier is deterministic, an unchanged catalog
-regenerates the baseline byte-for-byte.
+CI derives the catalog + all-families `libmeos.so` from upstream MobilityDB master through
+the shared `provision-meos` action, stages the catalog to `tools/meos-idl.json`, and lets
+`build-jmeos.sh` build the JMEOS jar from JMEOS `main` against that libmeos. Tracking master
+(not a pinned commit) keeps the source, the catalog, the jar and the libmeos the tests load
+all moving together, so the generated facades can never drift from the surface they run
+against.
diff --git a/build-jmeos.sh b/build-jmeos.sh
@@ -24,9 +24,8 @@
 # an ordinary dependency — no committed jar/so required.
 #
 # The refs below track upstream MobilityDB master and MobilityDB/JMEOS main — the
-# surfaces this project is generated against. They are recorded as immutable head
-# SHAs (overridable env vars) so a build is reproducible; bump them to the current
-# master/main tips when refreshing the generated surface (see GENERATION.md).
+# surfaces this project is generated against — as branch names (overridable env
+# vars), so a build always picks up the current master/main tips.
 #
 set -euo pipefail
 
@@ -36,12 +35,12 @@ set -euo pipefail
 # MobilityDB master — the surface the JMEOS facade is generated against, so the
 # built libmeos.so matches the facade catalog. Overridable via the environment.
 MOBILITYDB_REPO="${MOBILITYDB_REPO:-https://github.com/MobilityDB/MobilityDB.git}"
-MOBILITYDB_REF="${MOBILITYDB_REF:-d984d747acc1fcdee895ebaf7517912d596ea598}"  # master 2026-07-10
+MOBILITYDB_REF="${MOBILITYDB_REF:-master}"  # track upstream master (only used when libmeos is not pre-installed)
 
 # JMEOS main — functions.GeneratedFunctions (built at build-time from the committed
 # catalog) plus the org.mobilitydb.meos.MeosOps* facades.
 JMEOS_REPO="${JMEOS_REPO:-https://github.com/MobilityDB/JMEOS.git}"
-JMEOS_REF="${JMEOS_REF:-5275e7d44cf9a62b731b2c3c2c9aa4ccebafc604}"  # main 2026-07-10
+JMEOS_REF="${JMEOS_REF:-main}"  # track upstream main
 
 # Maven coordinates the jar is installed under (must match kafka-streams-app/pom.xml).
 JMEOS_GROUP_ID="${JMEOS_GROUP_ID:-com.mobilitydb}"
@@ -156,6 +155,20 @@ clone_at "${JMEOS_REPO}" "${JMEOS_REF}" "${JMEOS_DIR}"
 # JMEOS' build bundles src/libmeos.so into the jar and JarLibraryLoader extracts it.
 cp -f "${LIBMEOS_SO}" "${JMEOS_DIR}/jmeos-core/src/libmeos.so"
 
+# JMEOS main derives functions.GeneratedFunctions from codegen/input/meos-idl.json
+# but tracks the MEOS-API catalog (it no longer commits it). Stage the catalog
+# derived by provision-meos — the workflow puts it at tools/meos-idl.json — so the
+# jar is built from the same master surface as the libmeos and the facades.
+MEOS_CATALOG="${MEOS_CATALOG:-${SCRIPT_DIR}/tools/meos-idl.json}"
+if [ -s "${MEOS_CATALOG}" ]; then
+  mkdir -p "${JMEOS_DIR}/codegen/input"
+  cp -f "${MEOS_CATALOG}" "${JMEOS_DIR}/codegen/input/meos-idl.json"
+  log "Staged MEOS catalog into ${JMEOS_DIR}/codegen/input/meos-idl.json"
+else
+  echo "error: MEOS catalog not found at ${MEOS_CATALOG}; stage it (provision-meos) first" >&2
+  exit 1
+fi
+
 log "Building JMEOS.jar"
 # FunctionsGenerator lives in the codegen module, which jmeos-core does not
 # declare as a Maven dependency — so '-am' will not build it. Compile it first

diff --git a/kafka-streams-app/pom.xml b/kafka-streams-app/pom.xml
@@ -120,6 +120,53 @@
 
     <build>
         <plugins>
+            <!-- Generate the org.mobilitydb.meos.MeosOps* facades from the installed JMEOS
+                 jar's surface at build time (into target/, never committed), via the single
+                 generator vendored identically by every JVM binding. The kafka engine emits
+                 the same MeosOps* forwarders the streams app consumes. -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <id>generate-facades</id>
+                        <phase>generate-sources</phase>
+                        <goals><goal>exec</goal></goals>
+                        <configuration>
+                            <executable>python3</executable>
+                            <arguments>
+                                <argument>${project.basedir}/../tools/codegen_jvm.py</argument>
+                                <argument>--engine</argument>
+                                <argument>kafka</argument>
+                                <argument>--catalog</argument>
+                                <argument>${project.basedir}/../tools/meos-idl.json</argument>
+                                <argument>--jar</argument>
+                                <argument>${settings.localRepository}/com/mobilitydb/jmeos/1.4.0/jmeos-1.4.0.jar</argument>
+                                <argument>--out</argument>
+                                <argument>${project.build.directory}/generated-facades</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <version>3.5.0</version>
+                <executions>
+                    <execution>
+                        <id>add-generated-facades</id>
+                        <phase>generate-sources</phase>
+                        <goals><goal>add-source</goal></goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.build.directory}/generated-facades/src/main/java</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.12.1</version>

diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/meos/MeosOpsBigintSet.java b/kafka-streams-app/src/main/java/org/mobilitydb/meos/MeosOpsBigintSet.java