diff --git a/deps.edn b/deps.edn index a3655e6..090e92b 100644 --- a/deps.edn +++ b/deps.edn @@ -11,7 +11,7 @@ ;; `idx-stats` no longer restores the subtree to rebuild it (replikativ/persistent-sorted-set#24). org.replikativ/persistent-sorted-set {:mvn/version "0.5.141"} ;; konserve for durable storage - org.replikativ/konserve {:mvn/version "0.9.350" + org.replikativ/konserve {:mvn/version "0.9.375" :exclusions [org.clojure/clojurescript com.github.pkpkpk/cljs-node-io com.github.pkpkpk/fress @@ -65,7 +65,11 @@ javax.servlet/javax.servlet-api]}} :aliases - {:dev + ;; Sibling checkout, for co-development across the replikativ stack. + ;; Not required to build — konserve 0.9.375 carries `store-id`. + {:local {:override-deps {org.replikativ/konserve {:local/root "../konserve"}}} + + :dev {:extra-paths ["dev" "test" "bench" "notebooks"] :extra-deps {nrepl/nrepl {:mvn/version "1.3.0"} cider/cider-nrepl {:mvn/version "0.50.2"} @@ -211,7 +215,7 @@ ;; Use published deps (for CI/release) :release {:override-deps {org.replikativ/persistent-sorted-set {:mvn/version "0.5.141"} - org.replikativ/konserve {:mvn/version "0.9.350" + org.replikativ/konserve {:mvn/version "0.9.375" :exclusions [org.clojure/clojurescript com.github.pkpkpk/cljs-node-io com.github.pkpkpk/fress diff --git a/doc/gc-and-store-ownership.md b/doc/gc-and-store-ownership.md new file mode 100644 index 0000000..cec8877 --- /dev/null +++ b/doc/gc-and-store-ownership.md @@ -0,0 +1,116 @@ +# GC and store ownership + +Why stratum's garbage collection looks different from datahike's, scriptum's and +proximum's, and what would have to be decided to make them the same. + +Written while landing `konserve.gc-guard` across the stack. + +## Two patterns already exist + +| adapter | where its data lives | `mark-from-key-map` | +| --- | --- | --- | +| stratum | datahike's store (shared) | real — contributes its reachable keys | +| proximum | its own store (`:store-config`) | `#{}` | +| scriptum | its own path (`-ft` sidecar) | `#{}` | + +## Consistency splits into two axes + +**Axis A — the collection mechanism.** Guard the values-then-pointer window, +derive the sweep cutoff from `konserve.gc-guard/cutoff` *before* marking, sweep +with `konserve.gc/sweep!`. + +- datahike: yes +- scriptum: yes (as of the konserve backing) +- proximum: yes (as of `feat/konserve-gc-guard`) +- stratum: no — a per-store JVM monitor, no timestamp cutoff, deletion limited + to three explicitly enumerated key families + +**Axis B — store ownership.** Own your store and sweep it (scriptum, proximum) +versus share a store and contribute marks (stratum). + +## Stratum's position on A follows from its position on B + +`konserve.gc/sweep!` is allow-list: it deletes every key that is not in the +whitelist and is older than the cutoff. That is only safe for a store you own +outright. + +Stratum shares datahike's store — stated in +`datahike/index/secondary/stratum.clj`: + +> Stratum shares datahike's store […] stratum writes to datahike's konserve +> store, so datahike's GC must preserve stratum's keys. + +So stratum's explicit-family deletion is not an oversight; it is the adaptation +that sharing forces. Porting it to `sweep!` would require a whitelist covering +every key datahike owns, in the one configuration where getting that wrong +destroys another library's data. + +Stratum is already safe on both of its paths: + +- **Embedded**: datahike opens `guard/writing!` *before* the secondary flush — + `datahike/writing.cljc` names stratum in the comment, because `-sec-flush` + writes konserve keys from inside it — and absorbs stratum's marks via + `sec/mark-from-key-map :stratum`. +- **Standalone**: `with-storage-lock` serializes `gc!` against `sync!`. + +## The decision is B, not A + +**Model 1 — every library owns its store.** Stratum moves out of datahike's +store, adopts A wholesale, and its `mark-from-key-map` becomes `#{}` like the +others. + +- Buys: one mechanism everywhere; no cross-library mark protocol; a mark bug can + only damage its own data. +- Costs: N stores to configure, back up and replicate; no structural sharing + across libraries; a migration for stratum. + +**Model 2 — one store, many marks.** Scriptum and proximum move into datahike's +store and implement real marks. + +- Buys: one store to operate; one GC; secondary data co-located with the datoms + that reference it. +- Costs: every `mark-from-key-map` must be exhaustively correct or datahike's + sweep deletes that library's data. One library's mark bug becomes another's + data loss. + +Two things that landed this week change what is available here. Scriptum's +konserve backing makes Model 2 *possible* for it at all — under the path design +its data was files, which is why its `-sec-mark` still reads "Scriptum uses +filesystem, not konserve — nothing to mark". And `konserve.gc-guard` makes +Model 2 *safe*, being the shared safe point across libraries in one store. + +Weak recommendation: **Model 1.** Failure containment is worth more than +operational tidiness, and landing this work surfaced two independent mark bugs — +proximum's mark never whitelisted branch heads, so every GC deleted the head +commit of every branch. Model 2 asks you to bet against exactly that class of +bug. + +## Tripwire + +If scriptum's konserve backing ever shares datahike's store, +`mark-from-key-map :scriptum` returning `#{}` becomes a data-loss bug. It is +correct today only because scriptum owns its store. + +## Fixed here: the GC/sync lock key + +`with-storage-lock` allocated its monitor in a map keyed by the store +*reference*, reasoning that record equality would make two connections to one +path share a lock. Measured, they do not: + +``` +identical? false = false same hash? false +``` + +So a process that connected twice got two independent monitors and no exclusion +at all — 0.16 ms of contention where the lock should have blocked for 300 ms. + +The accompanying claim that "under-locking is impossible: distinct keys always +get distinct locks" was backwards. Two distinct keys for one physical store *is* +under-locking. The safe direction is the opposite, and it is the same rule the +GC safe point follows: the key may be **coarser** than the physical store — two +stores collapsing onto one monitor merely over-locks — but never **finer**. + +The lock now keys on konserve's `:id`, which is coarser (a logical identity, +shared by replicas across machines) and within one JVM names one store. Stores +built through a backend constructor carry no id and fall back to the object, +which is the old behaviour. diff --git a/src/stratum/dataset.clj b/src/stratum/dataset.clj index d01ea10..ec59c7d 100644 --- a/src/stratum/dataset.clj +++ b/src/stratum/dataset.clj @@ -336,7 +336,7 @@ ;; sentinel themselves. (doseq [[col-name col-data] columns-field] (idx/idx-append! (:index col-data) - (dict-encode col-data (get row col-name)))) + (dict-encode col-data (get row col-name)))) (set! row-count-val (unchecked-inc row-count-val)) this)) @@ -382,9 +382,9 @@ (append! this (:set opts))) this) (let [{vf-col :from-col vt-col :to-col vt-unit :unit} valid-cfg - close-vt-val (or (coerce-temporal-value (:valid-from tx-meta) vt-unit) - (now-in-unit vt-unit)) - n (long row-count-val) + close-vt-val (or (coerce-temporal-value (:valid-from tx-meta) vt-unit) + (now-in-unit vt-unit)) + n (long row-count-val) ;; Pass 1: classify every matching row. A row is :close-safe ;; iff it's currently open (vt=MAX) AND its vf is strictly ;; before the new write's vf — i.e. closing it to new-vf @@ -408,22 +408,22 @@ ;; (row-vt <= new-vf). Ignored. ;; :overlap — anything else; can't proceed without ;; auto-split. - {:keys [close-safe overlaps]} - (loop [i 0 acc {:close-safe (transient []) - :overlaps (transient [])}] - (if (>= i n) - {:close-safe (persistent! (:close-safe acc)) - :overlaps (persistent! (:overlaps acc))} - (let [row (materialize-row columns-field i)] - (if-let [row-set (match-fn row)] - (let [row-vf (long (get row vf-col)) - row-vt (long (get row vt-col)) - open? (= row-vt Long/MAX_VALUE) - klass (cond - (and open? (< row-vf close-vt-val)) :close-safe + {:keys [close-safe overlaps]} + (loop [i 0 acc {:close-safe (transient []) + :overlaps (transient [])}] + (if (>= i n) + {:close-safe (persistent! (:close-safe acc)) + :overlaps (persistent! (:overlaps acc))} + (let [row (materialize-row columns-field i)] + (if-let [row-set (match-fn row)] + (let [row-vf (long (get row vf-col)) + row-vt (long (get row vt-col)) + open? (= row-vt Long/MAX_VALUE) + klass (cond + (and open? (< row-vf close-vt-val)) :close-safe ;; Closed row entirely before new-vf — no overlap - (and (not open?) (<= row-vt close-vt-val)) :no-conflict - :else :overlaps)] + (and (not open?) (<= row-vt close-vt-val)) :no-conflict + :else :overlaps)] ;; A key is struck off only when the row it named is ;; actually going to carry the new values. `:no-conflict` ;; means the matched row is closed and entirely in the @@ -432,16 +432,16 @@ ;; validity window abuts an existing closed one was ;; silently dropped: the old row was not updated (correct) ;; and the new row was never appended (not). - (when-not (= :no-conflict klass) - (when-let [k (matcher-key opts row)] - (.remove ^java.util.Set unmatched k))) - (recur (inc i) - (if (= :no-conflict klass) - acc + (when-not (= :no-conflict klass) + (when-let [k (matcher-key opts row)] + (.remove ^java.util.Set unmatched k))) + (recur (inc i) + (if (= :no-conflict klass) + acc ;; `row-set` travels with the row: under the keyed ;; form each matched row carries its own values. - (update acc klass conj! [i row row-set])))) - (recur (inc i) acc))))) + (update acc klass conj! [i row row-set])))) + (recur (inc i) acc))))) ;; Auto-split partitions overlaps by row-vf vs new-vf: ;; row-vf < new-vf → partial left overlap → TRUNCATE ;; (set row-vt = new-vf, keep the @@ -453,24 +453,24 @@ ;; new-vt is always MAX in upsert!/retract!. They become ;; reachable once Phase D wires SQL `FOR PORTION OF ;; VALID_TIME FROM x TO y`.) - {auto-truncate :truncate auto-drop :drop} - (if (and (seq overlaps) auto-split?) - (group-by (fn [[_i r _s]] - (if (< (long (get r vf-col)) close-vt-val) - :truncate - :drop)) - overlaps) - {:truncate [] :drop []}) - - _ (when (and (seq overlaps) (not auto-split?)) - (throw (ex-info "upsert! would overlap existing rows' vt-windows" - {:where (:where opts) - :by (:by opts) - :new-window [close-vt-val Long/MAX_VALUE] - :overlaps (mapv (fn [[i r _s]] - (assoc (select-keys r [vf-col vt-col]) :row-idx i)) - overlaps) - :hint "pass :auto-split? true to split overlapping rows, or restructure the write"})))] + {auto-truncate :truncate auto-drop :drop} + (if (and (seq overlaps) auto-split?) + (group-by (fn [[_i r _s]] + (if (< (long (get r vf-col)) close-vt-val) + :truncate + :drop)) + overlaps) + {:truncate [] :drop []}) + + _ (when (and (seq overlaps) (not auto-split?)) + (throw (ex-info "upsert! would overlap existing rows' vt-windows" + {:where (:where opts) + :by (:by opts) + :new-window [close-vt-val Long/MAX_VALUE] + :overlaps (mapv (fn [[i r _s]] + (assoc (select-keys r [vf-col vt-col]) :row-idx i)) + overlaps) + :hint "pass :auto-split? true to split overlapping rows, or restructure the write"})))] ;; Mutation order is load-bearing: ;; 1. set-at! everything (close-safe + truncate) — indices stay ;; stable across set-at!. @@ -479,64 +479,64 @@ ;; truncate) would point at the wrong row. ;; 3. append! the new rows (close-safe merged-row, or degenerate ;; insert). - (if system-cfg + (if system-cfg ;; ----- Bitemporal path (SCD2-on-both-axes) ----- ;; Every mutation of a row's vt-window is also a system-time ;; event: close the old row's _system_to, append the ;; replacement(s) with a fresh _system_from. This preserves ;; the audit chain so `FOR SYSTEM_TIME AS OF ` still ;; sees the pre-surgery state. - (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] - (doseq [[i prev-row row-set] close-safe] - (let [orig-vf (long (get prev-row vf-col))] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val} - {:vf close-vt-val :vt Long/MAX_VALUE :data row-set}] - tx-meta))) - (doseq [[i prev-row _s] auto-truncate] - (let [orig-vf (long (get prev-row vf-col))] + (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] + (doseq [[i prev-row row-set] close-safe] + (let [orig-vf (long (get prev-row vf-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val} + {:vf close-vt-val :vt Long/MAX_VALUE :data row-set}] + tx-meta))) + (doseq [[i prev-row _s] auto-truncate] + (let [orig-vf (long (get prev-row vf-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val}] + tx-meta))) + (doseq [[i prev-row _s] auto-drop] (replace-row-bitemporal! this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val}] - tx-meta))) - (doseq [[i prev-row _s] auto-drop] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [] - tx-meta)) + [] + tx-meta)) ;; INSERT-where-nothing-matched. The predicate form has one such ;; case — no row matched at all — while the keyed form has one per ;; key that named no row, which is why `unmatched` is a set rather ;; than a boolean. - (doseq [[k row-set] (unmatched-inserts opts unmatched)] - (append! this (assoc row-set (:by opts) k) - (assoc tx-meta :valid-from close-vt-val - :system-from system-now))) - (when (and (nil? (:by opts)) (empty? close-safe)) - (append! this (:set opts) - (assoc tx-meta :valid-from close-vt-val - :system-from system-now)))) + (doseq [[k row-set] (unmatched-inserts opts unmatched)] + (append! this (assoc row-set (:by opts) k) + (assoc tx-meta :valid-from close-vt-val + :system-from system-now))) + (when (and (nil? (:by opts)) (empty? close-safe)) + (append! this (:set opts) + (assoc tx-meta :valid-from close-vt-val + :system-from system-now)))) ;; ----- Valid-only path (existing in-place mutation) ----- - (do - (doseq [[i _prev-row _s] close-safe] - (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) - (doseq [[i _r _s] auto-truncate] - (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) - (when (seq auto-drop) - (ds-delete-rows! this (mapv first auto-drop))) - (doseq [[_i prev-row row-set] close-safe] - (let [merged-row (merge (dissoc prev-row vf-col vt-col) - row-set)] - (append! this - (assoc merged-row vf-col close-vt-val) - tx-meta))) - (doseq [[k row-set] (unmatched-inserts opts unmatched)] - (append! this (assoc row-set (:by opts) k) - (assoc tx-meta :valid-from close-vt-val))) - (when (and (nil? (:by opts)) (empty? close-safe)) - (append! this (:set opts) (assoc tx-meta :valid-from close-vt-val))))) - this)))) + (do + (doseq [[i _prev-row _s] close-safe] + (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) + (doseq [[i _r _s] auto-truncate] + (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) + (when (seq auto-drop) + (ds-delete-rows! this (mapv first auto-drop))) + (doseq [[_i prev-row row-set] close-safe] + (let [merged-row (merge (dissoc prev-row vf-col vt-col) + row-set)] + (append! this + (assoc merged-row vf-col close-vt-val) + tx-meta))) + (doseq [[k row-set] (unmatched-inserts opts unmatched)] + (append! this (assoc row-set (:by opts) k) + (assoc tx-meta :valid-from close-vt-val))) + (when (and (nil? (:by opts)) (empty? close-safe)) + (append! this (:set opts) (assoc tx-meta :valid-from close-vt-val))))) + this)))) (retract! [this opts] (retract! this opts nil)) @@ -563,21 +563,21 @@ (transient []) (range n)))] (when (seq victims) (ds-delete-rows! this victims)) this) - (let [{vf-col :from-col vt-col :to-col vt-unit :unit} valid-cfg - close-vt-val (or (coerce-temporal-value (:valid-from tx-meta) vt-unit) - (now-in-unit vt-unit)) + (let [{vf-col :from-col vt-col :to-col vt-unit :unit} valid-cfg + close-vt-val (or (coerce-temporal-value (:valid-from tx-meta) vt-unit) + (now-in-unit vt-unit)) ;; `:valid-to` explicitly present in tx-meta selects the ;; bounded surgery path even when the value is MAX (the ;; `FOR ALL VALID_TIME` form passes `Long/MAX_VALUE` and ;; expects every matching row to be dropped). - bounded? (contains? tx-meta :valid-to) - close-vt-end (or (coerce-temporal-value (:valid-to tx-meta) vt-unit) - Long/MAX_VALUE) - _ (when bounded? + bounded? (contains? tx-meta :valid-to) + close-vt-end (or (coerce-temporal-value (:valid-to tx-meta) vt-unit) + Long/MAX_VALUE) + _ (when bounded? ;; Bounded retract has explicit [vf, vt) — reject ;; zero-width / reverse periods that would silently ;; degenerate the overlap test to a no-op. - (validate-period! close-vt-val close-vt-end :valid)) + (validate-period! close-vt-val close-vt-end :valid)) ;; Bounded retract — SQL:2011 `FOR PORTION OF VALID_TIME ;; FROM x TO y DELETE` semantic. For each matching row's ;; window [row-vf, row-vt), apply surgery against @@ -588,163 +588,163 @@ ;; straddles both → split (truncate + append right tail) ;; Bounded retract is unconditional — no `:auto-split?` flag, ;; the bounded form IS the surgical semantic. - n (long row-count-val)] - (if bounded? + n (long row-count-val)] + (if bounded? ;; ----- Bounded retract (SQL:2011 FOR PORTION OF VALID_TIME) ----- ;; Surgery is unconditional — no `:auto-split?` flag. - (let [{:keys [drops trunc-vt trunc-vf splits]} - (loop [i 0 acc {:drops (transient []) - :trunc-vt (transient []) - :trunc-vf (transient []) - :splits (transient [])}] - (if (>= i n) - {:drops (persistent! (:drops acc)) - :trunc-vt (persistent! (:trunc-vt acc)) - :trunc-vf (persistent! (:trunc-vf acc)) - :splits (persistent! (:splits acc))} - (let [row (materialize-row columns-field i)] - (if (match-fn row) - (let [row-vf (long (get row vf-col)) - row-vt (long (get row vt-col)) - klass (cond - (or (<= row-vt close-vt-val) - (>= row-vf close-vt-end)) - :no-conflict - - (and (>= row-vf close-vt-val) - (<= row-vt close-vt-end)) - :drops - - (and (< row-vf close-vt-val) - (<= row-vt close-vt-end)) - :trunc-vt - - (and (>= row-vf close-vt-val) - (> row-vt close-vt-end)) - :trunc-vf - - :else :splits)] - (recur (inc i) - (if (= :no-conflict klass) - acc - (update acc klass conj! [i row])))) - (recur (inc i) acc)))))] - (if system-cfg + (let [{:keys [drops trunc-vt trunc-vf splits]} + (loop [i 0 acc {:drops (transient []) + :trunc-vt (transient []) + :trunc-vf (transient []) + :splits (transient [])}] + (if (>= i n) + {:drops (persistent! (:drops acc)) + :trunc-vt (persistent! (:trunc-vt acc)) + :trunc-vf (persistent! (:trunc-vf acc)) + :splits (persistent! (:splits acc))} + (let [row (materialize-row columns-field i)] + (if (match-fn row) + (let [row-vf (long (get row vf-col)) + row-vt (long (get row vt-col)) + klass (cond + (or (<= row-vt close-vt-val) + (>= row-vf close-vt-end)) + :no-conflict + + (and (>= row-vf close-vt-val) + (<= row-vt close-vt-end)) + :drops + + (and (< row-vf close-vt-val) + (<= row-vt close-vt-end)) + :trunc-vt + + (and (>= row-vf close-vt-val) + (> row-vt close-vt-end)) + :trunc-vf + + :else :splits)] + (recur (inc i) + (if (= :no-conflict klass) + acc + (update acc klass conj! [i row])))) + (recur (inc i) acc)))))] + (if system-cfg ;; ----- Bitemporal bounded retract ----- ;; Each surgery class gets its system-time event: close ;; old row's system-to, append the surviving slice(s) ;; with fresh system-from. Drops have no replacements. - (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] - (doseq [[i prev-row] drops] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [] - tx-meta)) - (doseq [[i prev-row] trunc-vt] - (let [orig-vf (long (get prev-row vf-col))] + (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] + (doseq [[i prev-row] drops] (replace-row-bitemporal! this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val}] - tx-meta))) - (doseq [[i prev-row] trunc-vf] - (let [orig-vt (long (get prev-row vt-col))] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [{:vf close-vt-end :vt orig-vt}] - tx-meta))) - (doseq [[i prev-row] splits] - (let [orig-vf (long (get prev-row vf-col)) - orig-vt (long (get prev-row vt-col))] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val} - {:vf close-vt-end :vt orig-vt}] - tx-meta))) - this) + [] + tx-meta)) + (doseq [[i prev-row] trunc-vt] + (let [orig-vf (long (get prev-row vf-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val}] + tx-meta))) + (doseq [[i prev-row] trunc-vf] + (let [orig-vt (long (get prev-row vt-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf close-vt-end :vt orig-vt}] + tx-meta))) + (doseq [[i prev-row] splits] + (let [orig-vf (long (get prev-row vf-col)) + orig-vt (long (get prev-row vt-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val} + {:vf close-vt-end :vt orig-vt}] + tx-meta))) + this) ;; ----- Valid-only bounded retract (existing in-place) ----- - (do - (doseq [[i _r] trunc-vt] - (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) - (doseq [[i _r] trunc-vf] - (idx/idx-set! (:index (get columns-field vf-col)) i close-vt-end)) - (let [tails (mapv (fn [[i r]] - (idx/idx-set! (:index (get columns-field vt-col)) - i close-vt-val) - (-> r (assoc vf-col close-vt-end))) - splits)] - (when (seq drops) - (ds-delete-rows! this (mapv first drops))) - (doseq [tail tails] - (append! this tail tx-meta))) - this))) + (do + (doseq [[i _r] trunc-vt] + (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) + (doseq [[i _r] trunc-vf] + (idx/idx-set! (:index (get columns-field vf-col)) i close-vt-end)) + (let [tails (mapv (fn [[i r]] + (idx/idx-set! (:index (get columns-field vt-col)) + i close-vt-val) + (-> r (assoc vf-col close-vt-end))) + splits)] + (when (seq drops) + (ds-delete-rows! this (mapv first drops))) + (doseq [tail tails] + (append! this tail tx-meta))) + this))) ;; ----- Open-window retract (existing behavior) ----- - (let [{:keys [close-safe overlaps]} - (loop [i 0 acc {:close-safe (transient []) - :overlaps (transient [])}] - (if (>= i n) - {:close-safe (persistent! (:close-safe acc)) - :overlaps (persistent! (:overlaps acc))} - (let [row (materialize-row columns-field i)] - (if (match-fn row) - (let [row-vf (long (get row vf-col)) - row-vt (long (get row vt-col)) - open? (= row-vt Long/MAX_VALUE) - klass (cond - (and open? (< row-vf close-vt-val)) :close-safe - (and (not open?) (<= row-vt close-vt-val)) :no-conflict - :else :overlaps)] - (recur (inc i) - (if (= :no-conflict klass) - acc - (update acc klass conj! [i row])))) - (recur (inc i) acc))))) - {auto-truncate :truncate auto-drop :drop} - (if (and (seq overlaps) auto-split?) - (group-by (fn [[_i r]] - (if (< (long (get r vf-col)) close-vt-val) - :truncate - :drop)) - overlaps) - {:truncate [] :drop []})] - (when (and (seq overlaps) (not auto-split?)) - (throw (ex-info "retract! would touch rows whose vt-window doesn't cover the retract instant" - {:where (:where opts) - :by (:by opts) - :retract-at close-vt-val - :overlaps (mapv (fn [[i r]] - (assoc (select-keys r [vf-col vt-col]) :row-idx i)) - overlaps) - :hint "pass :auto-split? true to split overlapping rows, or restructure the write"}))) - (if system-cfg + (let [{:keys [close-safe overlaps]} + (loop [i 0 acc {:close-safe (transient []) + :overlaps (transient [])}] + (if (>= i n) + {:close-safe (persistent! (:close-safe acc)) + :overlaps (persistent! (:overlaps acc))} + (let [row (materialize-row columns-field i)] + (if (match-fn row) + (let [row-vf (long (get row vf-col)) + row-vt (long (get row vt-col)) + open? (= row-vt Long/MAX_VALUE) + klass (cond + (and open? (< row-vf close-vt-val)) :close-safe + (and (not open?) (<= row-vt close-vt-val)) :no-conflict + :else :overlaps)] + (recur (inc i) + (if (= :no-conflict klass) + acc + (update acc klass conj! [i row])))) + (recur (inc i) acc))))) + {auto-truncate :truncate auto-drop :drop} + (if (and (seq overlaps) auto-split?) + (group-by (fn [[_i r]] + (if (< (long (get r vf-col)) close-vt-val) + :truncate + :drop)) + overlaps) + {:truncate [] :drop []})] + (when (and (seq overlaps) (not auto-split?)) + (throw (ex-info "retract! would touch rows whose vt-window doesn't cover the retract instant" + {:where (:where opts) + :by (:by opts) + :retract-at close-vt-val + :overlaps (mapv (fn [[i r]] + (assoc (select-keys r [vf-col vt-col]) :row-idx i)) + overlaps) + :hint "pass :auto-split? true to split overlapping rows, or restructure the write"}))) + (if system-cfg ;; ----- Bitemporal open-window retract ----- - (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] - (doseq [[i prev-row] close-safe] - (let [orig-vf (long (get prev-row vf-col))] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val}] - tx-meta))) - (doseq [[i prev-row] auto-truncate] - (let [orig-vf (long (get prev-row vf-col))] + (let [system-now (system-now-from-tx-meta system-cfg tx-meta)] + (doseq [[i prev-row] close-safe] + (let [orig-vf (long (get prev-row vf-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val}] + tx-meta))) + (doseq [[i prev-row] auto-truncate] + (let [orig-vf (long (get prev-row vf-col))] + (replace-row-bitemporal! + this prev-row i system-now valid-cfg system-cfg + [{:vf orig-vf :vt close-vt-val}] + tx-meta))) + (doseq [[i prev-row] auto-drop] (replace-row-bitemporal! this prev-row i system-now valid-cfg system-cfg - [{:vf orig-vf :vt close-vt-val}] - tx-meta))) - (doseq [[i prev-row] auto-drop] - (replace-row-bitemporal! - this prev-row i system-now valid-cfg system-cfg - [] - tx-meta)) - this) + [] + tx-meta)) + this) ;; ----- Valid-only open-window retract ----- - (do - (doseq [[i _row] close-safe] - (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) - (doseq [[i _r] auto-truncate] - (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) - (when (seq auto-drop) - (ds-delete-rows! this (mapv first auto-drop))) - this)))))))) + (do + (doseq [[i _row] close-safe] + (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) + (doseq [[i _r] auto-truncate] + (idx/idx-set! (:index (get columns-field vt-col)) i close-vt-val)) + (when (seq auto-drop) + (ds-delete-rows! this (mapv first auto-drop))) + this)))))))) (ds-delete-rows! [this row-idxs] (when-not edit diff --git a/src/stratum/query.clj b/src/stratum/query.clj index b9c9e93..d2a1ce5 100644 --- a/src/stratum/query.clj +++ b/src/stratum/query.clj @@ -489,108 +489,108 @@ ;; === Original hardcoded path === (let [columns resolved-cols ;; Strip SQL table-qualifier namespaces from group/order/window keywords - group (when group (mapv #(if (keyword? %) (norm/strip-ns %) %) group)) - order (when order (mapv (fn [[c dir]] [(norm/strip-ns c) dir]) order)) - window (when window - (mapv (fn [ws] - (cond-> ws - (:col ws) (update :col norm/strip-ns) - (:partition-by ws) (update :partition-by #(mapv norm/strip-ns %)) - (:order-by ws) (update :order-by #(mapv (fn [[c d]] [(norm/strip-ns c) d]) %)))) - window)) + group (when group (mapv #(if (keyword? %) (norm/strip-ns %) %) group)) + order (when order (mapv (fn [[c dir]] [(norm/strip-ns c) dir]) order)) + window (when window + (mapv (fn [ws] + (cond-> ws + (:col ws) (update :col norm/strip-ns) + (:partition-by ws) (update :partition-by #(mapv norm/strip-ns %)) + (:order-by ws) (update :order-by #(mapv (fn [[c d]] [(norm/strip-ns c) d]) %)))) + window)) ;; BITMAP SEMI-JOIN: check first — converts eligible joins to mask predicates ;; so subsequent paths (fused, normal) operate on pre-filtered fact data. - bitmap-semi (when (and (seq join) (= 1 (count join))) - (let [used-cols (into #{} - (concat - (when group (filter keyword? group)) - (mapcat (fn [a] - (let [na (norm/normalize-agg a)] - (concat (when (:col na) [(:col na)]) - (when (:cols na) (:cols na))))) - (or agg [])) - (when select (filter keyword? select)) - (keep (fn [pred] - (try - (let [np (norm/normalize-pred pred)] - (first np)) - (catch Exception _ nil))) - (or where []))))] - (when (jn/bitmap-semi-join-eligible? (first join) used-cols (some? select)) - (let [fact-length (x/get-column-length (val (first columns))) - result (jn/execute-bitmap-semi-join columns fact-length (first join))] - result)))) + bitmap-semi (when (and (seq join) (= 1 (count join))) + (let [used-cols (into #{} + (concat + (when group (filter keyword? group)) + (mapcat (fn [a] + (let [na (norm/normalize-agg a)] + (concat (when (:col na) [(:col na)]) + (when (:cols na) (:cols na))))) + (or agg [])) + (when select (filter keyword? select)) + (keep (fn [pred] + (try + (let [np (norm/normalize-pred pred)] + (first np)) + (catch Exception _ nil))) + (or where []))))] + (when (jn/bitmap-semi-join-eligible? (first join) used-cols (some? select)) + (let [fact-length (x/get-column-length (val (first columns))) + result (jn/execute-bitmap-semi-join columns fact-length (first join))] + result)))) ;; If bitmap semi-join fired, remove join and inject mask predicate - [columns join where] (if bitmap-semi - [(:columns bitmap-semi) nil - (conj (vec (or where [])) [:= (:mask-key bitmap-semi) 1])] - [columns join where]) + [columns join where] (if bitmap-semi + [(:columns bitmap-semi) nil + (conj (vec (or where [])) [:= (:mask-key bitmap-semi) 1])] + [columns join where]) ;; FUSED JOIN+GROUP+AGG: probe+gather+group-by in single Java pass ;; Eligible when: single INNER join, GROUP BY dim cols, agg fact cols, no WHERE - fused-result - (when (and (seq join) (seq group) (seq agg) (empty? (or where [])) (nil? select)) - (let [norm-aggs (norm/auto-alias-aggs (mapv norm/normalize-agg agg))] - (when (jn/fused-join-group-agg-eligible? join group norm-aggs columns) - (let [fact-length (x/get-column-length (val (first columns))) - columnar? (= :columns result)] - (jn/execute-fused-join-group-agg - columns fact-length (first join) group norm-aggs columnar?))))) + fused-result + (when (and (seq join) (seq group) (seq agg) (empty? (or where [])) (nil? select)) + (let [norm-aggs (norm/auto-alias-aggs (mapv norm/normalize-agg agg))] + (when (jn/fused-join-group-agg-eligible? join group norm-aggs columns) + (let [fact-length (x/get-column-length (val (first columns))) + columnar? (= :columns result)] + (jn/execute-fused-join-group-agg + columns fact-length (first join) group norm-aggs columnar?))))) ;; FUSED JOIN+GLOBAL-AGG: single-pass probe+accumulate, no gather ;; For join queries without GROUP BY (H2O J1/J2/J3 pattern) - fused-global-join-result - (when (and (not fused-result) - (seq join) (not (seq group)) (seq agg) - (empty? (or where [])) (nil? select)) - (let [norm-aggs (norm/auto-alias-aggs (mapv norm/normalize-agg agg))] - (when (jn/fused-join-global-agg-eligible? join norm-aggs) - (let [fact-length (x/get-column-length (val (first columns)))] - (jn/execute-fused-join-global-agg - columns fact-length (first join) norm-aggs))))) - fused-any (or fused-result fused-global-join-result)] - (if fused-any + fused-global-join-result + (when (and (not fused-result) + (seq join) (not (seq group)) (seq agg) + (empty? (or where [])) (nil? select)) + (let [norm-aggs (norm/auto-alias-aggs (mapv norm/normalize-agg agg))] + (when (jn/fused-join-global-agg-eligible? join norm-aggs) + (let [fact-length (x/get-column-length (val (first columns)))] + (jn/execute-fused-join-global-agg + columns fact-length (first join) norm-aggs))))) + fused-any (or fused-result fused-global-join-result)] + (if fused-any ;; Post-processing for fused results - (if (= :columns result) - fused-any - (let [results (if distinct (post/apply-distinct fused-any) fused-any) - results (if (seq having) (post/apply-having results having) results) - results (if (seq _having-only-keys) - (mapv #(apply dissoc % _having-only-keys) results) - results) - results (if (seq order) (post/apply-order results order limit offset) results) - results (if (seq _order-only-keys) - (mapv #(apply dissoc % _order-only-keys) results) - results) - results (if (or limit offset) (post/apply-limit-offset results limit offset) results)] - results)) + (if (= :columns result) + fused-any + (let [results (if distinct (post/apply-distinct fused-any) fused-any) + results (if (seq having) (post/apply-having results having) results) + results (if (seq _having-only-keys) + (mapv #(apply dissoc % _having-only-keys) results) + results) + results (if (seq order) (post/apply-order results order limit offset) results) + results (if (seq _order-only-keys) + (mapv #(apply dissoc % _order-only-keys) results) + results) + results (if (or limit offset) (post/apply-limit-offset results limit offset) results)] + results)) ;; Normal path - (let [;; JOIN PHASE: bitmap semi-join already handled above; remaining joins use hash - [columns length] - (if (seq join) - (let [initial-length (x/get-column-length (val (first columns))) - result (jn/execute-joins columns initial-length join)] - [(:columns result) (long (:length result))]) - [columns nil]) + (let [;; JOIN PHASE: bitmap semi-join already handled above; remaining joins use hash + [columns length] + (if (seq join) + (let [initial-length (x/get-column-length (val (first columns))) + result (jn/execute-joins columns initial-length join)] + [(:columns result) (long (:length result))]) + [columns nil]) ;; ANOMALY RESOLUTION: score post-join so all columns (including ;; join results and index-backed data) are available - [query columns] (if (seq _anomaly-models) - (let [len (long (or length - (x/get-column-length (val (first columns)))))] - (resolve-anomaly-columns query columns len _anomaly-models)) - [query columns]) + [query columns] (if (seq _anomaly-models) + (let [len (long (or length + (x/get-column-length (val (first columns)))))] + (resolve-anomaly-columns query columns len _anomaly-models)) + [query columns]) ;; Re-destructure clauses that may have been rewritten by anomaly resolution. ;; ONLY when models were present — otherwise preserve local var modifications ;; (e.g. bitmap semi-join injects mask predicates into `where`) - where (if (seq _anomaly-models) (:where query) where) - select (if (seq _anomaly-models) (:select query) select) - having (if (seq _anomaly-models) (:having query) having) - order (if (seq _anomaly-models) (:order query) order)] + where (if (seq _anomaly-models) (:where query) where) + select (if (seq _anomaly-models) (:select query) select) + having (if (seq _anomaly-models) (:having query) having) + order (if (seq _anomaly-models) (:order query) order)] ;; Bind expr/*columns-meta*, gb/*dense-group-limit* and the ;; column-reference set so materialize-columns / check-memory-budget! ;; can prune unreferenced columns from the budget and the decode ;; pass. - (binding [expr/*columns-meta* (into {} (keep (fn [[k v]] (when (or (:dict v) (:temporal-unit v)) [k v]))) columns) - gb/*dense-group-limit* *dense-group-limit* - cols/*query-column-refs* (cols/query-references query columns)] + (binding [expr/*columns-meta* (into {} (keep (fn [[k v]] (when (or (:dict v) (:temporal-unit v)) [k v]))) columns) + gb/*dense-group-limit* *dense-group-limit* + cols/*query-column-refs* (cols/query-references query columns)] ;; Streaming SELECT DISTINCT col: dedupe at the ;; column level via ColumnOpsExt's parallel ;; BitSet/Hash distinct, no row-map allocation. @@ -598,417 +598,417 @@ ;; LIMIT N with no other clauses streams the order ;; column and fetches only the surviving N rows from ;; each output column. - (cond - (q-distinct/distinct-eligible? query columns) - (q-distinct/execute-distinct query columns) + (cond + (q-distinct/distinct-eligible? query columns) + (q-distinct/execute-distinct query columns) - (top-n/top-n-eligible? query columns) - (top-n/execute-top-n query columns) + (top-n/top-n-eligible? query columns) + (top-n/execute-top-n query columns) - :else - (let [preds (mapv norm/normalize-pred (or where [])) - aggs (norm/auto-alias-aggs (mapv norm/normalize-agg (or agg []))) + :else + (let [preds (mapv norm/normalize-pred (or where [])) + aggs (norm/auto-alias-aggs (mapv norm/normalize-agg (or agg []))) ;; Determine data length from first column (use join length if available) - length (long (or length - (x/get-column-length (val (first columns))))) + length (long (or length + (x/get-column-length (val (first columns))))) ;; Zone map pruning: compute surviving chunks for index-sourced columns. ;; This must happen before any materialization to avoid double work. ;; Only applies when ALL columns are from indices AND we have simple predicates. - surviving-chunks (when (and (x/any-index? columns) - (seq preds) - (not (some #(map? (first %)) preds))) - (try (x/compute-surviving-chunks preds columns) - (catch Exception _ nil))) + surviving-chunks (when (and (x/any-index? columns) + (seq preds) + (not (some #(map? (first %)) preds))) + (try (x/compute-surviving-chunks preds columns) + (catch Exception _ nil))) ;; Apply pruned materialization: replace index columns with shorter arrays ;; containing only the chunks that survive zone map filtering. - [columns length] (if surviving-chunks - (let [pruned (x/materialize-columns-pruned columns surviving-chunks) - first-data (:data (val (first pruned))) - new-len (cond - (expr/long-array? first-data) (alength ^longs first-data) - (expr/double-array? first-data) (alength ^doubles first-data) - :else length)] - [pruned (long new-len)]) - [columns length]) + [columns length] (if surviving-chunks + (let [pruned (x/materialize-columns-pruned columns surviving-chunks) + first-data (:data (val (first pruned))) + new-len (cond + (expr/long-array? first-data) (alength ^longs first-data) + (expr/double-array? first-data) (alength ^doubles first-data) + :else length)] + [pruned (long new-len)]) + [columns length]) ;; Pre-materialize string-producing expression predicates (e.g., LOWER(name) = 'bob') ;; into dict-encoded temp columns. Must happen before materialize-computed-preds ;; which would fail on string expressions via eval-expr-vectorized. - [preds columns] (if (some #(and (map? (first %)) - (expr/string-producing-expr? (first %))) preds) - (let [counter (atom 0)] - (reduce (fn [[ps cs] pred] - (let [col-ref (first pred)] - (if (and (map? col-ref) (expr/string-producing-expr? col-ref)) - (let [n (swap! counter inc) - col-name (keyword (str "__pred_str_" n)) - col-entry (expr/eval-string-expr col-ref cs length)] - [(conj ps (assoc pred 0 col-name)) - (assoc cs col-name col-entry)]) - [(conj ps pred) cs]))) - [[] columns] - preds)) - [preds columns]) + [preds columns] (if (some #(and (map? (first %)) + (expr/string-producing-expr? (first %))) preds) + (let [counter (atom 0)] + (reduce (fn [[ps cs] pred] + (let [col-ref (first pred)] + (if (and (map? col-ref) (expr/string-producing-expr? col-ref)) + (let [n (swap! counter inc) + col-name (keyword (str "__pred_str_" n)) + col-entry (expr/eval-string-expr col-ref cs length)] + [(conj ps (assoc pred 0 col-name)) + (assoc cs col-name col-entry)]) + [(conj ps pred) cs]))) + [[] columns] + preds)) + [preds columns]) ;; Pre-compute expression predicates (e.g., [:> [:* :a :b] 1000]) - [preds columns] (if (some #(map? (first %)) preds) - (x/materialize-computed-preds preds columns length) - [preds columns]) + [preds columns] (if (some #(map? (first %)) preds) + (x/materialize-computed-preds preds columns length) + [preds columns]) ;; Materialize string predicates (LIKE, CONTAINS, etc.) into mask columns - [preds columns] (if (some #(pred/string-pred-ops (second %)) preds) - (let [mat-cols (x/materialize-columns columns)] - (pred/materialize-string-preds preds mat-cols length)) - [preds columns]) + [preds columns] (if (some #(pred/string-pred-ops (second %)) preds) + (let [mat-cols (x/materialize-columns columns)] + (pred/materialize-string-preds preds mat-cols length)) + [preds columns]) ;; Resolve string/keyword equality/inequality on dict-encoded columns ;; Converts [:col :eq "Alice"] → [:col :eq 0] using dict codes ;; Keywords are converted to (str kw) to match dict storage - preds (let [has-dict-pred? (some #(and (pred/dict-resolvable-ops (second %)) - (>= (count %) 3) - (let [v (nth % 2)] - (or (string? v) (keyword? v)))) preds)] - (if has-dict-pred? + preds (let [has-dict-pred? (some #(and (pred/dict-resolvable-ops (second %)) + (>= (count %) 3) + (let [v (nth % 2)] + (or (string? v) (keyword? v)))) preds)] + (if has-dict-pred? ;; Convert keyword args to strings before dict resolution - (pred/resolve-dict-equality-preds - (mapv (fn [pred] - (if (and (>= (count pred) 3) (keyword? (nth pred 2))) - (assoc pred 2 (str (nth pred 2))) - pred)) - preds) - columns) - preds)) + (pred/resolve-dict-equality-preds + (mapv (fn [pred] + (if (and (>= (count pred) 3) (keyword? (nth pred 2))) + (assoc pred 2 (str (nth pred 2))) + pred)) + preds) + columns) + preds)) ;; Pre-materialize string-producing expressions (UPPER, LOWER, SUBSTR, etc.) ;; into dict-encoded temp columns. This must happen before group-by pre-computation ;; because string exprs produce dict-encoded columns, not numeric arrays. - [group aggs select columns] - (let [has-string-expr? (or (some #(and (sequential? %) - (expr/string-producing-expr? (norm/normalize-expr (vec %)))) - group) - (some #(and (:expr %) - (expr/string-producing-expr? (:expr %))) - aggs))] - (if has-string-expr? - (let [result (expr/materialize-string-exprs group aggs select columns length)] + [group aggs select columns] + (let [has-string-expr? (or (some #(and (sequential? %) + (expr/string-producing-expr? (norm/normalize-expr (vec %)))) + group) + (some #(and (:expr %) + (expr/string-producing-expr? (:expr %))) + aggs))] + (if has-string-expr? + (let [result (expr/materialize-string-exprs group aggs select columns length)] ;; set! updates the thread-local binding of *columns-meta* so that ;; subsequent expression eval within this `binding` scope sees the ;; newly materialized dict-encoded temp columns. - (set! expr/*columns-meta* (into {} (keep (fn [[k v]] (when (or (:dict v) (:temporal-unit v)) [k v]))) (nth result 3))) - result) - [group aggs select columns])) + (set! expr/*columns-meta* (into {} (keep (fn [[k v]] (when (or (:dict v) (:temporal-unit v)) [k v]))) (nth result 3))) + result) + [group aggs select columns])) ;; Compile non-SIMD predicates (OR, IN, NOT-IN) into a mask column. ;; The mask is materialized as a long[] and added as [:__mask :eq 1]. ;; This lets the SIMD/Java paths handle any predicate shape with zero changes. ;; Only materialize columns referenced by non-SIMD preds (not all columns), ;; so that unreferenced index columns stay as indices for the chunked path. - [preds columns] (let [[simd-preds non-simd-preds] (pred/split-preds preds columns)] - (if (seq non-simd-preds) - (let [;; Extract column keys referenced by non-SIMD preds - pred-col-keys (let [ks (atom #{})] - (letfn [(walk [p] - (let [op (second p)] - (case op - :or (doseq [sub (subvec p 2)] (walk sub)) - :fn (swap! ks conj (first p)) - (:in :not-in) (swap! ks conj (first p)) - (do - (when (keyword? (first p)) - (swap! ks conj (first p))) + [preds columns] (let [[simd-preds non-simd-preds] (pred/split-preds preds columns)] + (if (seq non-simd-preds) + (let [;; Extract column keys referenced by non-SIMD preds + pred-col-keys (let [ks (atom #{})] + (letfn [(walk [p] + (let [op (second p)] + (case op + :or (doseq [sub (subvec p 2)] (walk sub)) + :fn (swap! ks conj (first p)) + (:in :not-in) (swap! ks conj (first p)) + (do + (when (keyword? (first p)) + (swap! ks conj (first p))) ;; RHS keyword args (col-vs-col) ;; — round-3 agent dedupe. - (doseq [a (subvec p 2)] - (when (keyword? a) - (swap! ks conj a)))))))] - (doseq [p non-simd-preds] (walk p))) - @ks) + (doseq [a (subvec p 2)] + (when (keyword? a) + (swap! ks conj a)))))))] + (doseq [p non-simd-preds] (walk p))) + @ks) ;; Only materialize those columns, keep others as indices - partial-mat (reduce (fn [cols k] - (if-let [c (get cols k)] - (assoc cols k (x/materialize-column c)) - cols)) - columns pred-col-keys) - mask-fn (pred/compile-pred-mask non-simd-preds partial-mat) - mask-arr (mask-fn length)] - [(conj simd-preds [:__mask :eq 1]) - (assoc partial-mat :__mask {:type :int64 :data mask-arr})]) - [simd-preds columns])) + partial-mat (reduce (fn [cols k] + (if-let [c (get cols k)] + (assoc cols k (x/materialize-column c)) + cols)) + columns pred-col-keys) + mask-fn (pred/compile-pred-mask non-simd-preds partial-mat) + mask-arr (mask-fn length)] + [(conj simd-preds [:__mask :eq 1]) + (assoc partial-mat :__mask {:type :int64 :data mask-arr})]) + [simd-preds columns])) ;; Detect fused extract+count eligibility BEFORE expression pre-computation ;; Eligible: single extract expr group, all-COUNT aggs, no predicates, long[] source - orig-group group - fused-extract? (and (= 1 (count group)) - (not (keyword? (first group))) - (seq aggs) - (every? #(= :count (:op %)) aggs) - (empty? preds) - (let [expr (norm/normalize-expr (vec (first group)))] - (and (map? expr) - (#{:minute :hour :second :day-of-week} (:op expr)) - (= 1 (count (:args expr))) - (keyword? (first (:args expr))) - (let [col-info (get columns (first (:args expr)))] - (or (expr/long-array? (:data col-info)) - (and (:index col-info) - (= :int64 (:type col-info)))))))) + orig-group group + fused-extract? (and (= 1 (count group)) + (not (keyword? (first group))) + (seq aggs) + (every? #(= :count (:op %)) aggs) + (empty? preds) + (let [expr (norm/normalize-expr (vec (first group)))] + (and (map? expr) + (#{:minute :hour :second :day-of-week} (:op expr)) + (= 1 (count (:args expr))) + (keyword? (first (:args expr))) + (let [col-info (get columns (first (:args expr)))] + (or (expr/long-array? (:data col-info)) + (and (:index col-info) + (= :int64 (:type col-info)))))))) ;; Pre-compute expression group-by columns (e.g., [:minute :et] → __grp_0) ;; Skip when fused-extract will handle extraction inline - [group columns] (if (and (seq group) - (not fused-extract?) - (some #(not (keyword? %)) group)) - (let [mat-cols (x/materialize-columns columns) - col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) - cache (java.util.HashMap.)] - (loop [idx 0, new-group [], new-cols columns] - (if (>= idx (count group)) - [new-group new-cols] - (let [g (nth group idx) + [group columns] (if (and (seq group) + (not fused-extract?) + (some #(not (keyword? %)) group)) + (let [mat-cols (x/materialize-columns columns) + col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) + cache (java.util.HashMap.)] + (loop [idx 0, new-group [], new-cols columns] + (if (>= idx (count group)) + [new-group new-cols] + (let [g (nth group idx) ;; Recognize `[:as expr alias]` so users can name ;; expression group keys; fall back to a positional ;; synthetic when no alias is supplied. - [raw alias] (if (and (vector? g) (= :as (first g)) (= 3 (count g))) - [(nth g 1) (nth g 2)] - [g nil])] - (if (keyword? raw) - (recur (inc idx) (conj new-group raw) new-cols) - (let [expr (norm/normalize-expr (vec raw)) - la (expr/eval-expr-to-long expr col-arrays length cache) - col-name (or alias (keyword (str "__grp_" idx)))] - (recur (inc idx) - (conj new-group col-name) - (assoc new-cols col-name {:type :int64 :data la})))))))) - [group columns]) + [raw alias] (if (and (vector? g) (= :as (first g)) (= 3 (count g))) + [(nth g 1) (nth g 2)] + [g nil])] + (if (keyword? raw) + (recur (inc idx) (conj new-group raw) new-cols) + (let [expr (norm/normalize-expr (vec raw)) + la (expr/eval-expr-to-long expr col-arrays length cache) + col-name (or alias (keyword (str "__grp_" idx)))] + (recur (inc idx) + (conj new-group col-name) + (assoc new-cols col-name {:type :int64 :data la})))))))) + [group columns]) ;; Pre-materialize expression aggs into temp columns so SIMD routing matches. ;; E.g. [:avg [:length :url]] → compute LENGTH into __agg_expr_0, then [:avg :__agg_expr_0] ;; Only for global aggs (no group-by) — group-by does this after materialization ;; to preserve chunked index streaming eligibility. - [aggs columns] (if (and (seq aggs) (not (seq group)) - (some :expr aggs)) - (let [mat-cols (x/materialize-columns columns) - col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) - cache (java.util.HashMap.)] - (loop [idx 0, new-aggs [], new-cols columns] - (if (>= idx (count aggs)) - [new-aggs new-cols] - (let [agg (nth aggs idx)] - (if-let [expr (:expr agg)] - (let [result-arr (expr/eval-expr-vectorized expr col-arrays length cache) - col-name (keyword (str "__agg_expr_" idx))] - (recur (inc idx) - (conj new-aggs (-> agg - (dissoc :expr) - (assoc :col col-name))) - (assoc new-cols col-name {:type :float64 :data result-arr}))) - (recur (inc idx) (conj new-aggs agg) new-cols)))))) - [aggs columns]) + [aggs columns] (if (and (seq aggs) (not (seq group)) + (some :expr aggs)) + (let [mat-cols (x/materialize-columns columns) + col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) + cache (java.util.HashMap.)] + (loop [idx 0, new-aggs [], new-cols columns] + (if (>= idx (count aggs)) + [new-aggs new-cols] + (let [agg (nth aggs idx)] + (if-let [expr (:expr agg)] + (let [result-arr (expr/eval-expr-vectorized expr col-arrays length cache) + col-name (keyword (str "__agg_expr_" idx))] + (recur (inc idx) + (conj new-aggs (-> agg + (dissoc :expr) + (assoc :col col-name))) + (assoc new-cols col-name {:type :float64 :data result-arr}))) + (recur (inc idx) (conj new-aggs agg) new-cols)))))) + [aggs columns]) ;; Normalize select items if present - select-items (when (seq select) - (vec (map-indexed #(x/normalize-select-item %2 %1) select))) + select-items (when (seq select) + (vec (map-indexed #(x/normalize-select-item %2 %1) select))) ;; Pre-materialize string-producing expressions in SELECT into temp columns - [select-items columns] - (if (and (seq select-items) - (some #(and (:expr %) (expr/string-producing-expr? (:expr %))) select-items)) - (let [counter (atom 0)] - (reduce (fn [[items cols] sel] - (if (and (:expr sel) (expr/string-producing-expr? (:expr sel))) - (let [n (swap! counter inc) - col-name (keyword (str "__sel_str_" n)) - col-entry (expr/eval-string-expr (:expr sel) cols length)] - [(conj items (-> sel (dissoc :expr) (assoc :ref col-name))) - (assoc cols col-name col-entry)]) - [(conj items sel) cols])) - [[] columns] - select-items)) - [select-items columns]) + [select-items columns] + (if (and (seq select-items) + (some #(and (:expr %) (expr/string-producing-expr? (:expr %))) select-items)) + (let [counter (atom 0)] + (reduce (fn [[items cols] sel] + (if (and (:expr sel) (expr/string-producing-expr? (:expr sel))) + (let [n (swap! counter inc) + col-name (keyword (str "__sel_str_" n)) + col-entry (expr/eval-string-expr (:expr sel) cols length)] + [(conj items (-> sel (dissoc :expr) (assoc :ref col-name))) + (assoc cols col-name col-entry)]) + [(conj items sel) cols])) + [[] columns] + select-items)) + [select-items columns]) ;; Window functions: pre-routing only when no GROUP BY ;; When GROUP BY is present, windows run post-GROUP-BY on aggregated results - columns (if (and (seq window) (not (seq group))) - (win/execute-window-functions columns length window) - columns) + columns (if (and (seq window) (not (seq group))) + (win/execute-window-functions columns length window) + columns) ;; Window having pushdown: evaluate having on raw arrays before projection. ;; Avoids materializing ~6M row maps when only ~120K survive (e.g. ROW_NUMBER <= 2). - [columns length having] - (if (and (seq window) (not (seq group)) (seq having)) - (let [normalized-havings (mapv norm/normalize-pred having) - win-cols (set (map :as window)) + [columns length having] + (if (and (seq window) (not (seq group)) (seq having)) + (let [normalized-havings (mapv norm/normalize-pred having) + win-cols (set (map :as window)) ;; Only push down if all having cols come from window results - all-window-cols? (every? #(contains? win-cols (first %)) normalized-havings)] - (if all-window-cols? - (let [matching (java.util.ArrayList.)] - (dotimes [i (int length)] - (when (every? - (fn [pred] - (let [col-key (first pred) - col-entry (get columns col-key) - col-data (if (map? col-entry) (:data col-entry) col-entry) - v (if (expr/double-array? col-data) - (aget ^doubles col-data i) - (double (aget ^longs col-data i))) - op (second pred) - arg (double (nth pred 2))] - (case op - :lt (< v arg) - :gt (> v arg) - :lte (<= v arg) - :gte (>= v arg) - :eq (== v arg) - :neq (not (== v arg)) - true))) - normalized-havings) - (.add matching (int i)))) - (let [n (.size matching)] - (if (< n (long length)) - (let [indices (int-array n) - _ (dotimes [i n] (aset indices i (int (.get matching i)))) - new-cols (reduce-kv - (fn [m k v] - (let [data (if (map? v) (:data v) v)] - (assoc m k - (cond - (expr/long-array? data) - (let [g (ColumnOps/gatherLong ^longs data indices (int n))] - (if (map? v) (assoc v :data g) g)) - (expr/double-array? data) - (let [g (ColumnOps/gatherDouble ^doubles data indices (int n))] - (if (map? v) (assoc v :data g) g)) - (expr/string-array? data) - (let [out (make-array String n)] - (dotimes [i n] - (aset ^"[Ljava.lang.String;" out i - (aget ^"[Ljava.lang.String;" data (aget indices i)))) - (if (map? v) (assoc v :data out) out)) - :else v)))) - {} - columns)] - [new-cols n nil]) ;; nil clears having — already applied - [columns length having]))) - [columns length having])) - [columns length having]) + all-window-cols? (every? #(contains? win-cols (first %)) normalized-havings)] + (if all-window-cols? + (let [matching (java.util.ArrayList.)] + (dotimes [i (int length)] + (when (every? + (fn [pred] + (let [col-key (first pred) + col-entry (get columns col-key) + col-data (if (map? col-entry) (:data col-entry) col-entry) + v (if (expr/double-array? col-data) + (aget ^doubles col-data i) + (double (aget ^longs col-data i))) + op (second pred) + arg (double (nth pred 2))] + (case op + :lt (< v arg) + :gt (> v arg) + :lte (<= v arg) + :gte (>= v arg) + :eq (== v arg) + :neq (not (== v arg)) + true))) + normalized-havings) + (.add matching (int i)))) + (let [n (.size matching)] + (if (< n (long length)) + (let [indices (int-array n) + _ (dotimes [i n] (aset indices i (int (.get matching i)))) + new-cols (reduce-kv + (fn [m k v] + (let [data (if (map? v) (:data v) v)] + (assoc m k + (cond + (expr/long-array? data) + (let [g (ColumnOps/gatherLong ^longs data indices (int n))] + (if (map? v) (assoc v :data g) g)) + (expr/double-array? data) + (let [g (ColumnOps/gatherDouble ^doubles data indices (int n))] + (if (map? v) (assoc v :data g) g)) + (expr/string-array? data) + (let [out (make-array String n)] + (dotimes [i n] + (aset ^"[Ljava.lang.String;" out i + (aget ^"[Ljava.lang.String;" data (aget indices i)))) + (if (map? v) (assoc v :data out) out)) + :else v)))) + {} + columns)] + [new-cols n nil]) ;; nil clears having — already applied + [columns length having]))) + [columns length having])) + [columns length having]) ;; If we have window functions, ensure they appear in the select projection - select-items (if (and (seq window) (not (seq group)) (seq select-items)) + select-items (if (and (seq window) (not (seq group)) (seq select-items)) ;; Add window result columns to select items if not already present - (let [win-names (set (map :as window)) - existing-names (set (map :name select-items))] - (into select-items - (keep (fn [ws] - (when-not (contains? existing-names (:as ws)) - {:name (:as ws) :ref (:as ws)}))) - window)) - select-items) + (let [win-names (set (map :as window)) + existing-names (set (map :name select-items))] + (into select-items + (keep (fn [ws] + (when-not (contains? existing-names (:as ws)) + {:name (:as ws) :ref (:as ws)}))) + window)) + select-items) ;; If we have window functions but no explicit select, build a projection ;; that includes all original columns + window columns - select-items (if (and (seq window) (not (seq select-items)) (not (seq aggs)) (not (seq group))) - (let [base-cols (vec (remove #{:__mask} (keys columns))) - items (mapv (fn [k] {:name k :ref k}) base-cols)] - items) - select-items) + select-items (if (and (seq window) (not (seq select-items)) (not (seq aggs)) (not (seq group))) + (let [base-cols (vec (remove #{:__mask} (keys columns))) + items (mapv (fn [k] {:name k :ref k}) base-cols)] + items) + select-items) ;; Route to execution strategy - results - (cond + results + (cond ;; Projection (SELECT without aggregation) - (and (seq select-items) (empty? aggs) (not (seq group))) - (x/execute-projection preds select-items columns length (= :columns result)) + (and (seq select-items) (empty? aggs) (not (seq group))) + (x/execute-projection preds select-items columns length (= :columns result)) ;; Fused extract + COUNT group-by: extract inline, no intermediate array - fused-extract? - (let [expr (norm/normalize-expr (vec (first orig-group))) - col-key (first (:args expr)) - col-info (get columns col-key) - ^longs raw-col (or (:data col-info) - (index/idx-materialize-to-array (:index col-info))) - expr-type (case (:op expr) - :minute (int ColumnOpsExt/EXTRACT_MINUTE) - :hour (int ColumnOpsExt/EXTRACT_HOUR) - :second (int ColumnOpsExt/EXTRACT_SECOND) - :day-of-week (int ColumnOpsExt/EXTRACT_DAY_OF_WEEK)) - n-aggs (int (count aggs)) - ^"[[D" result-array (ColumnOpsExt/fusedExtractCountDenseParallel - raw-col expr-type n-aggs (int length)) - max-key (int (ColumnOpsExt/extractMaxKey expr-type)) + fused-extract? + (let [expr (norm/normalize-expr (vec (first orig-group))) + col-key (first (:args expr)) + col-info (get columns col-key) + ^longs raw-col (or (:data col-info) + (index/idx-materialize-to-array (:index col-info))) + expr-type (case (:op expr) + :minute (int ColumnOpsExt/EXTRACT_MINUTE) + :hour (int ColumnOpsExt/EXTRACT_HOUR) + :second (int ColumnOpsExt/EXTRACT_SECOND) + :day-of-week (int ColumnOpsExt/EXTRACT_DAY_OF_WEEK)) + n-aggs (int (count aggs)) + ^"[[D" result-array (ColumnOpsExt/fusedExtractCountDenseParallel + raw-col expr-type n-aggs (int length)) + max-key (int (ColumnOpsExt/extractMaxKey expr-type)) ;; Build group-col name from original expression - group-col-name (keyword (str (name (:op expr)))) - agg-aliases (mapv #(or (:as %) (:op %)) aggs) - columnar? (= :columns result)] - (if columnar? + group-col-name (keyword (str (name (:op expr)))) + agg-aliases (mapv #(or (:as %) (:op %)) aggs) + columnar? (= :columns result)] + (if columnar? ;; Columnar output - (let [key-list (java.util.ArrayList.) - _ (dotimes [k max-key] + (let [key-list (java.util.ArrayList.) + _ (dotimes [k max-key] + (let [^doubles accs (aget result-array k)] + (when (and accs (> (aget accs 1) 0.0)) + (.add key-list (long k))))) + n (.size key-list) + keys-arr (long-array n) + _ (dotimes [i n] (aset keys-arr i (long (.get key-list i)))) + agg-arrs (mapv (fn [a-idx] + (let [la (long-array n)] + (dotimes [i n] + (let [k (long (.get key-list i)) + ^doubles accs (aget result-array (int k))] + (aset la i (long (aget accs (inc (* a-idx 2))))))) + la)) + (range n-aggs))] + (-> {group-col-name keys-arr :n-rows n} + (into (map vector agg-aliases agg-arrs)))) + ;; Row-oriented output + (let [results (transient [])] + (dotimes [k max-key] (let [^doubles accs (aget result-array k)] (when (and accs (> (aget accs 1) 0.0)) - (.add key-list (long k))))) - n (.size key-list) - keys-arr (long-array n) - _ (dotimes [i n] (aset keys-arr i (long (.get key-list i)))) - agg-arrs (mapv (fn [a-idx] - (let [la (long-array n)] - (dotimes [i n] - (let [k (long (.get key-list i)) - ^doubles accs (aget result-array (int k))] - (aset la i (long (aget accs (inc (* a-idx 2))))))) - la)) - (range n-aggs))] - (-> {group-col-name keys-arr :n-rows n} - (into (map vector agg-aliases agg-arrs)))) - ;; Row-oriented output - (let [results (transient [])] - (dotimes [k max-key] - (let [^doubles accs (aget result-array k)] - (when (and accs (> (aget accs 1) 0.0)) - (let [cnt (long (aget accs 1)) - row (-> {group-col-name (long k)} - (into (map (fn [alias] [alias cnt]) agg-aliases)) - (assoc :_count cnt))] - (conj! results row))))) - (persistent! results)))) + (let [cnt (long (aget accs 1)) + row (-> {group-col-name (long k)} + (into (map (fn [alias] [alias cnt]) agg-aliases)) + (assoc :_count cnt))] + (conj! results row))))) + (persistent! results)))) ;; Group-by query — try chunk-streaming first, fallback to materialization - (seq group) - (let [columnar? (= :columns result)] - (or (gb/execute-chunked-group-by preds aggs group columns length columnar?) - (let [mat-cols (x/materialize-columns columns) + (seq group) + (let [columnar? (= :columns result)] + (or (gb/execute-chunked-group-by preds aggs group columns length columnar?) + (let [mat-cols (x/materialize-columns columns) ;; Pre-compute expression aggs after materialization so ;; chunked streaming is tried first with raw indices. - [aggs mat-cols] - (if (some :expr aggs) - (let [col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) - cache (java.util.HashMap.)] - (loop [idx 0, new-aggs [], new-cols mat-cols] - (if (>= idx (count aggs)) - [new-aggs new-cols] - (let [agg (nth aggs idx)] - (if-let [expr (:expr agg)] - (let [result-arr (expr/eval-expr-vectorized expr col-arrays length cache) - col-name (keyword (str "__agg_expr_" idx))] - (recur (inc idx) - (conj new-aggs (-> agg - (dissoc :expr) - (assoc :col col-name))) - (assoc new-cols col-name {:type :float64 :data result-arr}))) - (recur (inc idx) (conj new-aggs agg) new-cols)))))) - [aggs mat-cols])] - (x/execute-group-by preds aggs group mat-cols length columnar?)))) + [aggs mat-cols] + (if (some :expr aggs) + (let [col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) + cache (java.util.HashMap.)] + (loop [idx 0, new-aggs [], new-cols mat-cols] + (if (>= idx (count aggs)) + [new-aggs new-cols] + (let [agg (nth aggs idx)] + (if-let [expr (:expr agg)] + (let [result-arr (expr/eval-expr-vectorized expr col-arrays length cache) + col-name (keyword (str "__agg_expr_" idx))] + (recur (inc idx) + (conj new-aggs (-> agg + (dissoc :expr) + (assoc :col col-name))) + (assoc new-cols col-name {:type :float64 :data result-arr}))) + (recur (inc idx) (conj new-aggs agg) new-cols)))))) + [aggs mat-cols])] + (x/execute-group-by preds aggs group mat-cols length columnar?)))) ;; Unfiltered COUNT — short-circuit to avoid JIT poisoning. ;; fusedSimdCountRange compiled with numPreds=0 marks predicate branches ;; as UnreachedCode. When a filtered COUNT later hits those branches, ;; the JIT deoptimizes (2ms → 30ms). Returning length directly avoids ;; ever calling fusedSimdCountRange with zero predicates. - (and (= 1 (count aggs)) - (= :count (:op (first aggs))) - (empty? preds)) - (post/format-fused-result {:result 0.0 :count length} (first aggs)) + (and (= 1 (count aggs)) + (= :count (:op (first aggs))) + (empty? preds)) + (post/format-fused-result {:result 0.0 :count length} (first aggs)) ;; Unfiltered SUM/MIN/MAX/AVG/COUNT on index inputs — stats-only O(chunks). ;; Walks chunk-level statistics instead of materializing + SIMD. @@ -1017,240 +1017,240 @@ ;; metadata has count/min/max/null-count but not sum) advertise ;; :stats-sum-incomplete? on the column map; in that case SUM ;; and AVG must fall through to the SIMD path. - (and (seq aggs) - (empty? preds) - (not (seq group)) - (x/all-indices? columns) - (every? (fn [a] - (and (#{:sum :min :max :avg :count} (:op a)) - (nil? (:expr a)) - (or (not (#{:sum :avg} (:op a))) - (not (:stats-sum-incomplete? - (get columns (:col a))))))) - aggs)) - (let [result-row - (reduce - (fn [row agg] - (let [alias (keyword (or (:as agg) (:op agg)))] - (if (= :count (:op agg)) - (assoc row alias (long length) :_count length) - (let [entries (gb/collect-chunk-entries (:index (get columns (:col agg)))) - n-chunks (count entries) - col-long? (= :int64 (:type (get columns (:col agg)))) - [sum cnt mn mx] - (loop [i 0 sum 0.0 cnt (long 0) mn Double/MAX_VALUE mx (- Double/MAX_VALUE)] - (if (>= i n-chunks) - [sum cnt mn mx] - (let [^ChunkEntry entry (nth entries i) - ^stratum.stats.ChunkStats cs (.stats entry)] - (recur (inc i) - (+ sum (double (:sum cs))) - (+ cnt (long (:count cs))) - (Math/min mn (double (:min-val cs))) - (Math/max mx (double (:max-val cs)))))))] - (assoc row alias - (case (:op agg) + (and (seq aggs) + (empty? preds) + (not (seq group)) + (x/all-indices? columns) + (every? (fn [a] + (and (#{:sum :min :max :avg :count} (:op a)) + (nil? (:expr a)) + (or (not (#{:sum :avg} (:op a))) + (not (:stats-sum-incomplete? + (get columns (:col a))))))) + aggs)) + (let [result-row + (reduce + (fn [row agg] + (let [alias (keyword (or (:as agg) (:op agg)))] + (if (= :count (:op agg)) + (assoc row alias (long length) :_count length) + (let [entries (gb/collect-chunk-entries (:index (get columns (:col agg)))) + n-chunks (count entries) + col-long? (= :int64 (:type (get columns (:col agg)))) + [sum cnt mn mx] + (loop [i 0 sum 0.0 cnt (long 0) mn Double/MAX_VALUE mx (- Double/MAX_VALUE)] + (if (>= i n-chunks) + [sum cnt mn mx] + (let [^ChunkEntry entry (nth entries i) + ^stratum.stats.ChunkStats cs (.stats entry)] + (recur (inc i) + (+ sum (double (:sum cs))) + (+ cnt (long (:count cs))) + (Math/min mn (double (:min-val cs))) + (Math/max mx (double (:max-val cs)))))))] + (assoc row alias + (case (:op agg) ;; For int64 columns, cast min/max back to Long ;; (lossless since original values were longs) - :sum (if (zero? cnt) nil (if col-long? (long sum) sum)) - :min (if (zero? cnt) nil (if col-long? (long mn) mn)) - :max (if (zero? cnt) nil (if col-long? (long mx) mx)) - :avg (if (zero? cnt) nil (/ sum (double cnt)))) - :_count cnt))))) - {} - aggs)] - [result-row]) + :sum (if (zero? cnt) nil (if col-long? (long sum) sum)) + :min (if (zero? cnt) nil (if col-long? (long mn) mn)) + :max (if (zero? cnt) nil (if col-long? (long mx) mx)) + :avg (if (zero? cnt) nil (/ sum (double cnt)))) + :_count cnt))))) + {} + aggs)] + [result-row]) ;; Chunk-streaming SIMD: all inputs are indices + simd-eligible ;; COUNT uses JIT-isolated fusedSimdChunkedCountParallel in ColumnOpsExt ;; to avoid fusedSimdChunkBatch aggType interference (B5 idx: 2ms→27ms). - (and (x/all-indices? columns) - (= 1 (count aggs)) - (pred/simd-eligible? preds aggs columns length) - (nil? (:expr (first aggs)))) - (let [agg (first aggs)] - (if (= :count (:op agg)) + (and (x/all-indices? columns) + (= 1 (count aggs)) + (pred/simd-eligible? preds aggs columns length) + (nil? (:expr (first aggs)))) + (let [agg (first aggs)] + (if (= :count (:op agg)) ;; COUNT: JIT-isolated chunked count (no agg columns, no aggType switch) - (let [result (x/execute-chunked-fused-count preds columns length)] - (post/format-fused-result result agg)) - (if (= :avg (:op agg)) + (let [result (x/execute-chunked-fused-count preds columns length)] + (post/format-fused-result result agg)) + (if (= :avg (:op agg)) ;; avg = sum / count via chunked path - (let [sum-result (x/execute-chunked-fused - preds {:op :sum :col (:col agg) :as nil} - columns length) - cnt (:count sum-result)] - (post/format-fused-result - (if (zero? cnt) {:result Double/NaN :count 0} - {:result (/ (:result sum-result) (double cnt)) :count cnt}) - agg)) - (post/format-fused-result - (x/execute-chunked-fused preds agg columns length) - agg)))) + (let [sum-result (x/execute-chunked-fused + preds {:op :sum :col (:col agg) :as nil} + columns length) + cnt (:count sum-result)] + (post/format-fused-result + (if (zero? cnt) {:result Double/NaN :count 0} + {:result (/ (:result sum-result) (double cnt)) :count cnt}) + agg)) + (post/format-fused-result + (x/execute-chunked-fused preds agg columns length) + agg)))) ;; Block-skip COUNT on arrays: skip blocks via min/max statistics - (and (= 1 (count aggs)) - (= :count (:op (first aggs))) - (seq preds) - (not (x/all-indices? columns)) - (pred/simd-eligible? preds aggs columns length) - (nil? (:expr (first aggs)))) - (let [mat-cols (x/materialize-columns columns) - pp (gb/prepare-pred-arrays preds mat-cols) - ^doubles r (ColumnOpsExt/fusedSimdCountBlockSkipParallel - (int (:n-long pp)) - ^ints (:long-pred-types pp) - ^"[[J" (:long-cols pp) - ^longs (:long-lo pp) - ^longs (:long-hi pp) - (int (:n-dbl pp)) - ^ints (:dbl-pred-types pp) - ^"[[D" (:dbl-cols pp) - ^doubles (:dbl-lo pp) - ^doubles (:dbl-hi pp) - (int length))] - (post/format-fused-result {:result 0.0 :count (long (aget r 1))} (first aggs))) + (and (= 1 (count aggs)) + (= :count (:op (first aggs))) + (seq preds) + (not (x/all-indices? columns)) + (pred/simd-eligible? preds aggs columns length) + (nil? (:expr (first aggs)))) + (let [mat-cols (x/materialize-columns columns) + pp (gb/prepare-pred-arrays preds mat-cols) + ^doubles r (ColumnOpsExt/fusedSimdCountBlockSkipParallel + (int (:n-long pp)) + ^ints (:long-pred-types pp) + ^"[[J" (:long-cols pp) + ^longs (:long-lo pp) + ^longs (:long-hi pp) + (int (:n-dbl pp)) + ^ints (:dbl-pred-types pp) + ^"[[D" (:dbl-cols pp) + ^doubles (:dbl-lo pp) + ^doubles (:dbl-hi pp) + (int length))] + (post/format-fused-result {:result 0.0 :count (long (aget r 1))} (first aggs))) ;; Single fused SIMD aggregate on arrays (fastest path for arrays) - (and (= 1 (count aggs)) - (pred/simd-eligible? preds aggs columns length) - (nil? (:expr (first aggs)))) - (let [mat-cols (x/materialize-columns columns)] - (post/format-fused-result - (x/execute-fused preds (first aggs) mat-cols length) - (first aggs))) + (and (= 1 (count aggs)) + (pred/simd-eligible? preds aggs columns length) + (nil? (:expr (first aggs)))) + (let [mat-cols (x/materialize-columns columns)] + (post/format-fused-result + (x/execute-fused preds (first aggs) mat-cols length) + (first aggs))) ;; Multiple SUM-like aggs — single-pass via fusedSimdMultiSum (1 pass vs N) - (and (seq aggs) - (not (seq group)) - (pred/multi-agg-simd-eligible? preds aggs columns length) - (every? #(#{:sum :sum-product :count :avg} (:op %)) aggs) - (<= (count (filterv #(not= :count (:op %)) aggs)) 4)) - (let [mat-cols (x/materialize-columns columns)] - (x/execute-fused-multi-sum preds aggs mat-cols length)) + (and (seq aggs) + (not (seq group)) + (pred/multi-agg-simd-eligible? preds aggs columns length) + (every? #(#{:sum :sum-product :count :avg} (:op %)) aggs) + (<= (count (filterv #(not= :count (:op %)) aggs)) 4)) + (let [mat-cols (x/materialize-columns columns)] + (x/execute-fused-multi-sum preds aggs mat-cols length)) ;; Multiple aggs without GROUP BY — single-pass global aggregate via ;; group-by with 0 group columns (materializes, then one-pass Java SIMD). - (and (seq aggs) - (not (seq group)) - (pred/multi-agg-simd-eligible? preds aggs columns length)) - (let [mat-cols (x/materialize-columns columns)] - (x/execute-group-by preds aggs [] mat-cols length false)) + (and (seq aggs) + (not (seq group)) + (pred/multi-agg-simd-eligible? preds aggs columns length)) + (let [mat-cols (x/materialize-columns columns)] + (x/execute-group-by preds aggs [] mat-cols length false)) ;; Fast path for ungrouped percentile/median/approx-quantile — bypass Clojure scalar loop - (and (seq aggs) - (not (seq group)) - (every? #(#{:median :percentile :approx-quantile} (:op %)) aggs)) - (let [mat-cols (x/materialize-columns columns) - col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) + (and (seq aggs) + (not (seq group)) + (every? #(#{:median :percentile :approx-quantile} (:op %)) aggs)) + (let [mat-cols (x/materialize-columns columns) + col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols) ;; Build predicate mask (long[]) if predicates exist - mask (when (seq preds) - (let [m (long-array length)] - (dotimes [i length] - (when (every? #(gb/eval-pred-scalar col-arrays i %) preds) - (aset m i 1))) - m)) - cnt (if mask - (areduce ^longs mask i s (long 0) (+ s (aget ^longs mask i))) - (long length))] - [(into {:_count cnt} - (map (fn [agg] - (let [alias (or (:as agg) (:op agg)) - col-data (get col-arrays (:col agg)) - is-long? (expr/long-array? col-data) - pct (double (case (:op agg) - :median 0.5 - (:percentile :approx-quantile) (:param agg))) - result (case (:op agg) - (:median :percentile) - (if mask - (if is-long? - (ColumnOps/percentileFilteredLong ^longs col-data ^longs mask (int length) pct) - (ColumnOps/percentileFiltered ^doubles col-data ^longs mask (int length) pct)) - (ColumnOps/percentile (if is-long? - (let [la ^longs col-data - da (double-array (alength la))] - (dotimes [i (alength la)] (aset da i (double (aget la i)))) - da) - ^doubles col-data) - (int length) pct)) - :approx-quantile - (if mask - (let [work (double-array cnt) - pos (int-array 1)] - (dotimes [i length] - (when (== 1 (aget ^longs mask i)) - (aset work (aget pos 0) - (if is-long? - (double (aget ^longs col-data i)) - (aget ^doubles col-data i))) - (aset pos 0 (inc (aget pos 0))))) - (ColumnOpsAnalytics/tdigestApproxQuantileParallel work (int cnt) pct 200.0)) - (let [da (if is-long? - (let [la ^longs col-data - d (double-array (alength la))] - (dotimes [i (alength la)] (aset d i (double (aget la i)))) - d) - ^doubles col-data)] - (ColumnOpsAnalytics/tdigestApproxQuantileParallel da (int length) pct 200.0))))] - [alias result])) - aggs))]) + mask (when (seq preds) + (let [m (long-array length)] + (dotimes [i length] + (when (every? #(gb/eval-pred-scalar col-arrays i %) preds) + (aset m i 1))) + m)) + cnt (if mask + (areduce ^longs mask i s (long 0) (+ s (aget ^longs mask i))) + (long length))] + [(into {:_count cnt} + (map (fn [agg] + (let [alias (or (:as agg) (:op agg)) + col-data (get col-arrays (:col agg)) + is-long? (expr/long-array? col-data) + pct (double (case (:op agg) + :median 0.5 + (:percentile :approx-quantile) (:param agg))) + result (case (:op agg) + (:median :percentile) + (if mask + (if is-long? + (ColumnOps/percentileFilteredLong ^longs col-data ^longs mask (int length) pct) + (ColumnOps/percentileFiltered ^doubles col-data ^longs mask (int length) pct)) + (ColumnOps/percentile (if is-long? + (let [la ^longs col-data + da (double-array (alength la))] + (dotimes [i (alength la)] (aset da i (double (aget la i)))) + da) + ^doubles col-data) + (int length) pct)) + :approx-quantile + (if mask + (let [work (double-array cnt) + pos (int-array 1)] + (dotimes [i length] + (when (== 1 (aget ^longs mask i)) + (aset work (aget pos 0) + (if is-long? + (double (aget ^longs col-data i)) + (aget ^doubles col-data i))) + (aset pos 0 (inc (aget pos 0))))) + (ColumnOpsAnalytics/tdigestApproxQuantileParallel work (int cnt) pct 200.0)) + (let [da (if is-long? + (let [la ^longs col-data + d (double-array (alength la))] + (dotimes [i (alength la)] (aset d i (double (aget la i)))) + d) + ^doubles col-data)] + (ColumnOpsAnalytics/tdigestApproxQuantileParallel da (int length) pct 200.0))))] + [alias result])) + aggs))]) ;; Multiple or complex aggregations - scalar path - (seq aggs) - (let [mat-cols (x/materialize-columns columns)] - [(gb/execute-scalar-aggs preds aggs mat-cols length)]) + (seq aggs) + (let [mat-cols (x/materialize-columns columns)] + [(gb/execute-scalar-aggs preds aggs mat-cols length)]) ;; Filter only (no aggregation) - just count - :else - (let [mat-cols (x/materialize-columns columns) - col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols)] - [{:_count (loop [i 0 cnt 0] - (if (>= i length) - cnt - (if (every? #(gb/eval-pred-scalar col-arrays i %) preds) - (recur (inc i) (inc cnt)) - (recur (inc i) cnt))))}]))] + :else + (let [mat-cols (x/materialize-columns columns) + col-arrays (into {} (map (fn [[k v]] [k (:data v)])) mat-cols)] + [{:_count (loop [i 0 cnt 0] + (if (>= i length) + cnt + (if (every? #(gb/eval-pred-scalar col-arrays i %) preds) + (recur (inc i) (inc cnt)) + (recur (inc i) cnt))))}]))] ;; Post-processing pipeline (skip for columnar results — raw arrays) - (if (= :columns result) - results - (let [;; Post-GROUP-BY window execution: run windows on aggregated results - results (if (and (seq window) (seq group) (seq results)) - (let [grouped-cols (x/results->columns results) - n-grouped (count results) - with-windows (win/execute-window-functions grouped-cols n-grouped window) - all-ks (keys with-windows)] - (mapv (fn [i] - (reduce (fn [m k] - (let [v (get with-windows k)] - (assoc m k - (cond - (expr/long-array? v) (aget ^longs v i) - (expr/double-array? v) (aget ^doubles v i) - (expr/string-array? v) - (aget ^"[Ljava.lang.String;" v i) + (if (= :columns result) + results + (let [;; Post-GROUP-BY window execution: run windows on aggregated results + results (if (and (seq window) (seq group) (seq results)) + (let [grouped-cols (x/results->columns results) + n-grouped (count results) + with-windows (win/execute-window-functions grouped-cols n-grouped window) + all-ks (keys with-windows)] + (mapv (fn [i] + (reduce (fn [m k] + (let [v (get with-windows k)] + (assoc m k + (cond + (expr/long-array? v) (aget ^longs v i) + (expr/double-array? v) (aget ^doubles v i) + (expr/string-array? v) + (aget ^"[Ljava.lang.String;" v i) ;; Wrapped in {:type :data} map - (map? v) - (let [d (:data v)] - (cond - (expr/long-array? d) (aget ^longs d i) - (expr/double-array? d) (aget ^doubles d i) - :else (aget ^"[Ljava.lang.String;" d i))) - :else (get m k))))) - {} all-ks)) - (range n-grouped))) - results) - results (if distinct (post/apply-distinct results) results) - results (if (seq having) (post/apply-having results having) results) - results (if (seq _having-only-keys) - (mapv #(apply dissoc % _having-only-keys) results) - results) - results (if (seq order) (post/apply-order results order limit offset) results) - results (if (seq _order-only-keys) - (mapv #(apply dissoc % _order-only-keys) results) - results) - results (if (or limit offset) (post/apply-limit-offset results limit offset) results)] - results)))))))))] + (map? v) + (let [d (:data v)] + (cond + (expr/long-array? d) (aget ^longs d i) + (expr/double-array? d) (aget ^doubles d i) + :else (aget ^"[Ljava.lang.String;" d i))) + :else (get m k))))) + {} all-ks)) + (range n-grouped))) + results) + results (if distinct (post/apply-distinct results) results) + results (if (seq having) (post/apply-having results having) results) + results (if (seq _having-only-keys) + (mapv #(apply dissoc % _having-only-keys) results) + results) + results (if (seq order) (post/apply-order results order limit offset) results) + results (if (seq _order-only-keys) + (mapv #(apply dissoc % _order-only-keys) results) + results) + results (if (or limit offset) (post/apply-limit-offset results limit offset) results)] + results)))))))))] ;; MIN/MAX over a dict-encoded string column arrive here as codes; ;; this is the one place aggregate results are decoded back to ;; values. No-op (identity) when nothing string-valued was @@ -1292,52 +1292,52 @@ ;; Original compiled paths (let [columns resolved-cols preds (mapv norm/normalize-pred (or where [])) - aggs (mapv norm/normalize-agg (or agg [])) - length (x/get-column-length (val (first columns)))] - (cond + aggs (mapv norm/normalize-agg (or agg [])) + length (x/get-column-length (val (first columns)))] + (cond ;; Chunk-streaming path: all indices + simd-eligible ;; COUNT uses JIT-isolated fusedSimdChunkedCountParallel - (and (x/all-indices? columns) - (= 1 (count aggs)) - (pred/simd-eligible? preds aggs columns length) - (nil? (:expr (first aggs))) - (nil? (:group query))) - (let [agg (first aggs)] - (fn [] - (if (= :count (:op agg)) - (post/format-fused-result (x/execute-chunked-fused-count preds columns length) agg) - (let [result (x/execute-chunked-fused preds agg columns length)] - (if (= :avg (:op agg)) - (let [cnt (:count result)] - [{(keyword (or (:as agg) :avg)) - (if (zero? cnt) Double/NaN (/ (:result result) (double cnt))) - :_count cnt}]) - (post/format-fused-result result agg)))))) + (and (x/all-indices? columns) + (= 1 (count aggs)) + (pred/simd-eligible? preds aggs columns length) + (nil? (:expr (first aggs))) + (nil? (:group query))) + (let [agg (first aggs)] + (fn [] + (if (= :count (:op agg)) + (post/format-fused-result (x/execute-chunked-fused-count preds columns length) agg) + (let [result (x/execute-chunked-fused preds agg columns length)] + (if (= :avg (:op agg)) + (let [cnt (:count result)] + [{(keyword (or (:as agg) :avg)) + (if (zero? cnt) Double/NaN (/ (:result result) (double cnt))) + :_count cnt}]) + (post/format-fused-result result agg)))))) ;; Array-based SIMD path - (and (= 1 (count aggs)) - (pred/simd-eligible? preds aggs columns length) - (nil? (:expr (first aggs))) - (nil? (:group query))) - (let [mat-cols (x/materialize-columns columns) - agg (first aggs) - compiler-agg (x/agg->compiler-spec agg) - compiled (qc/compile-query - {:columns mat-cols - :predicates preds - :aggregate compiler-agg - :length length})] - (fn [] - (let [result (compiled)] - (if (= :avg (:op agg)) - (let [cnt (:count result)] - [{(keyword (or (:as agg) :avg)) - (if (zero? cnt) Double/NaN (/ (:result result) (double cnt))) - :_count cnt}]) - (post/format-fused-result result agg))))) + (and (= 1 (count aggs)) + (pred/simd-eligible? preds aggs columns length) + (nil? (:expr (first aggs))) + (nil? (:group query))) + (let [mat-cols (x/materialize-columns columns) + agg (first aggs) + compiler-agg (x/agg->compiler-spec agg) + compiled (qc/compile-query + {:columns mat-cols + :predicates preds + :aggregate compiler-agg + :length length})] + (fn [] + (let [result (compiled)] + (if (= :avg (:op agg)) + (let [cnt (:count result)] + [{(keyword (or (:as agg) :avg)) + (if (zero? cnt) Double/NaN (/ (:result result) (double cnt))) + :_count cnt}]) + (post/format-fused-result result agg))))) ;; Fall back to interpreted execution - :else - (fn [] (q query)))))] + :else + (fn [] (q query)))))] (if (empty? decoders) run (fn [] (post/decode-string-aggs (run) decoders))))) diff --git a/src/stratum/storage.clj b/src/stratum/storage.clj index 2da0912..ba03b3a 100644 --- a/src/stratum/storage.clj +++ b/src/stratum/storage.clj @@ -14,6 +14,7 @@ [:indices :commits ] → index-snapshot → serialized PSS node (Leaf or Branch)" (:require [konserve.core :as k] + [konserve.protocols :as kp] [hasch.core :as hasch] [clojure.set :as set] [org.replikativ.persistent-sorted-set :as pss]) @@ -26,35 +27,54 @@ ;; ============================================================================ (def ^:private store-locks - "Per-store GC/sync coordination locks, keyed by the store - reference itself. Map identity follows the store's `.equals` - semantics: - - - Konserve memory store / typical deftypes: default Object - equality (identity) → distinct instances get distinct locks. - - File-store / S3-store records with content-based equality: - two instances configured against the same backing storage - share a lock — which is what we want (they protect the same - physical data). - - Worst case for an oddly-implemented backend is two locks - collapsed into one (over-locking, safe). Under-locking is - impossible: distinct keys always get distinct locks. - - Stratum apps typically run with a small bounded set of stores - per JVM (often one), so we don't worry about cleanup. Inspect - the current set via `(locked-stores)`." + "Per-store GC/sync coordination locks, keyed by the store's konserve `:id`. + + KEYED BY ID, NOT BY THE STORE OBJECT, and the distinction is the whole + point. This used to key on the store reference and reason that + \"file-store / S3-store records with content-based equality: two instances + configured against the same backing storage share a lock\". They do not — + two `connect-fs-store` calls on one path are neither `identical?` nor `=`, + and hash differently. So two connections to one store got two independent + monitors and a `gc!` on one could run inside a `sync!` on the other, + sweeping chunks the other was about to reference. + + The old note that \"under-locking is impossible\" had it backwards: two + distinct keys for ONE physical store is exactly under-locking. The safe + direction is the opposite one, and it is the same rule + `konserve.gc-guard` follows for the safe point — the key may be COARSER + than the physical store (two stores collapsing onto one lock merely + over-locks) but never FINER. + + konserve's `:id` is coarser: it is a logical identity, shared by replicas + across machines. Within one JVM — which is all a monitor can reach — one + id names one store. + + A store built through a backend constructor rather than + `konserve.store/connect-store` carries no id, and falls back to the object. + That is the old behaviour, under-locking included, so such callers should + still avoid connecting twice. + + Stratum apps typically run with a small bounded set of stores per JVM + (often one), so we don't worry about cleanup. Inspect the current set via + `(locked-stores)`." (java.util.concurrent.ConcurrentHashMap.)) +(defn- lock-key + "What identifies the physical store for locking: its konserve `:id` when it + has one, otherwise the store itself. See `store-locks`." + [store] + (or (kp/store-id store) store)) + (defn- store-lock "Lazily allocate (or return) the Object monitor associated with `store`. Same store across calls → same Object. The `putIfAbsent` cycle resolves the racy double-allocation." [store] - (or (.get ^java.util.concurrent.ConcurrentHashMap store-locks store) - (let [o (Object.) - existing (.putIfAbsent ^java.util.concurrent.ConcurrentHashMap store-locks store o)] - (or existing o)))) + (let [k (lock-key store)] + (or (.get ^java.util.concurrent.ConcurrentHashMap store-locks k) + (let [o (Object.) + existing (.putIfAbsent ^java.util.concurrent.ConcurrentHashMap store-locks k o)] + (or existing o))))) (defn with-storage-lock "Execute f while holding the per-store GC/sync coordination diff --git a/test/stratum/storage_test.clj b/test/stratum/storage_test.clj index 3741782..0e5494b 100644 --- a/test/stratum/storage_test.clj +++ b/test/stratum/storage_test.clj @@ -7,6 +7,8 @@ [konserve.memory :refer [new-mem-store]] [konserve.filestore :refer [connect-fs-store]] [konserve.core :as k] + [konserve.store :as ks] + [konserve.protocols :as kp] [org.replikativ.persistent-sorted-set :as pss])) ;; ============================================================================ @@ -324,3 +326,47 @@ (is (= 3.14 (chunk/chunk-constant-val cc))) (is (= 3.14 (chunk/read-double cc 0))) (is (= 1000 (chunk/chunk-length cc)))))) + +;; ============================================================================ +;; GC / sync coordination lock +;; ============================================================================ + +(deftest storage-lock-follows-the-store-not-the-object + (testing "REGRESSION: the lock keyed on the store REFERENCE, on the reasoning + that record equality would make two connections to one path share a + monitor. They do not — two `connect-fs-store` calls on one path are + neither `identical?` nor `=`, and hash differently — so a process + that connected twice got two independent monitors, and a `gc!` on + one could run inside a `sync!` on the other and sweep chunks it was + about to reference. + + Keying on konserve's `:id` fixes it, and follows the same rule the + GC safe point does: the key may be COARSER than the physical store, + never finer." + (let [path (str "/tmp/stratum-lock-test-" (random-uuid)) + id (random-uuid) + cfg {:backend :file :path path :id id} + _ (ks/create-store cfg {:sync? true}) + a (ks/connect-store cfg {:sync? true}) + b (ks/connect-store cfg {:sync? true})] + (is (not= a b) + "the premise: two connections to one store are NOT equal") + (is (= (kp/store-id a) (kp/store-id b)) + "but they agree on the store they name") + ;; The monitor itself is private, so observe it through the behaviour it + ;; exists for: a lock held on one connection must exclude the other. + (let [entered (promise) + blocked (atom true) + t (Thread. (fn [] + (storage/with-storage-lock a + (fn [] (deliver entered true) (Thread/sleep 300)))))] + (.start t) + @entered + (let [waited (let [t0 (System/nanoTime)] + (storage/with-storage-lock b (fn [] (reset! blocked false))) + (/ (- (System/nanoTime) t0) 1e6))] + (.join t) + (is (false? @blocked)) + (is (> waited 100.0) + (str "a lock held on one connection must exclude the other; " + "waited only " waited "ms")))))))