From 7e0d6e754eb5ef637b3afc0552879c62d7897e51 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Thu, 6 Aug 2026 09:21:17 +0200 Subject: [PATCH 1/2] refactor(reader): make VarBinArray non-sealed, split its modes out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VarBinArray was the one Array sub-hierarchy still declared sealed with an explicit permits list, unlike its siblings (ByteArray, LongArray, DecimalArray, ...), which reopen the hierarchy via non-sealed and let each representation live as an ordinary top-level class. Nothing in the codebase does an exhaustive switch over VarBinArray's specific modes (grepped: zero matches), so the sealing bought no compiler-enforced exhaustiveness — only coupling the interface's declaration to the full list of its implementations, and forcing every new representation (most recently VarBinConstantArray, #331) to be nested inside VarBinArray.java and added to its permits clause. VarBinArray is now `non-sealed`. Each former nested record is a top-level class in reader.array, renamed to match the sibling families' self-describing naming (MaterializedByteArray, LazyConstantLongArray, ...): - OffsetMode -> VarBinOffsetArray - DictMode -> VarBinDictArray - ChunkedMode -> VarBinChunkedArray - ViewMode -> VarBinViewArray - SlicedMode -> VarBinSlicedArray - ConstantMode -> VarBinConstantArray checkedLength (bounds-check shared by the offset- and dict-backed representations) moved to a new package-private VarBinArrays helper, mirroring the RunEndArrays precedent, since it can no longer be a private interface method reachable from sibling top-level classes. Pure rename/move: no behavior change. All call sites across reader/writer/cli updated; docs/compatibility.md's Notes column updated to the new class names. --- .../dfa1/vortex/cli/tui/ArrayFixtures.java | 7 +- docs/compatibility.md | 8 +- .../dfa1/vortex/reader/ScanIterator.java | 3 +- .../reader/array/ChunkedArrayCombiner.java | 8 +- .../dfa1/vortex/reader/array/VarBinArray.java | 820 +----------------- .../vortex/reader/array/VarBinArrays.java | 35 + .../reader/array/VarBinChunkedArray.java | 175 ++++ .../reader/array/VarBinConstantArray.java | 66 ++ .../vortex/reader/array/VarBinDictArray.java | 239 +++++ .../reader/array/VarBinOffsetArray.java | 120 +++ .../reader/array/VarBinSlicedArray.java | 65 ++ .../vortex/reader/array/VarBinViewArray.java | 140 +++ .../decode/ConstantEncodingDecoder.java | 6 +- .../reader/decode/DictEncodingDecoder.java | 5 +- .../reader/decode/FsstEncodingDecoder.java | 4 +- .../reader/decode/RunEndEncodingDecoder.java | 5 +- .../reader/decode/SparseEncodingDecoder.java | 7 +- .../reader/decode/VarBinEncodingDecoder.java | 4 +- .../decode/VarBinViewEncodingDecoder.java | 6 +- .../reader/decode/ZstdEncodingDecoder.java | 5 +- .../reader/layout/DictLayoutDecoder.java | 9 +- .../vortex/reader/array/VarBinArrayTest.java | 28 +- ...eTest.java => VarBinChunkedArrayTest.java} | 36 +- ...ModeTest.java => VarBinViewArrayTest.java} | 16 +- .../vortex/reader/compute/ComputeArrays.java | 5 +- .../decode/ConstantEncodingDecoderTest.java | 7 +- .../decode/VarBinEncodingDecoderTest.java | 3 +- .../writer/MultiChunkUtf8RoundTripTest.java | 15 +- 28 files changed, 964 insertions(+), 883 deletions(-) create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArrays.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinConstantArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinDictArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinSlicedArray.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinViewArray.java rename reader/src/test/java/io/github/dfa1/vortex/reader/array/{VarBinChunkedModeTest.java => VarBinChunkedArrayTest.java} (83%) rename reader/src/test/java/io/github/dfa1/vortex/reader/array/{VarBinViewModeTest.java => VarBinViewArrayTest.java} (89%) diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/ArrayFixtures.java b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/ArrayFixtures.java index 704134740..f41d33993 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/tui/ArrayFixtures.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/tui/ArrayFixtures.java @@ -17,6 +17,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -128,7 +129,7 @@ static BoolArray bools(Arena arena, boolean... vs) { return new MaterializedBoolArray(DType.BOOL, vs.length, seg.asReadOnly()); } - /// Builds a UTF-8 [VarBinArray] (`OffsetMode`, I64 offsets) from the given strings. + /// Builds a UTF-8 [VarBinArray] (`VarBinOffsetArray`, I64 offsets) from the given strings. static VarBinArray utf8(Arena arena, String... vs) { byte[][] rows = new byte[vs.length][]; for (int i = 0; i < vs.length; i++) { @@ -137,7 +138,7 @@ static VarBinArray utf8(Arena arena, String... vs) { return varbin(arena, DType.UTF8, rows); } - /// Builds a binary [VarBinArray] (`OffsetMode`, I64 offsets) from the given byte rows. + /// Builds a binary [VarBinArray] (`VarBinOffsetArray`, I64 offsets) from the given byte rows. static VarBinArray binary(Arena arena, byte[]... rows) { return varbin(arena, DType.BINARY, rows); } @@ -156,6 +157,6 @@ private static VarBinArray varbin(Arena arena, DType dtype, byte[]... rows) { pos += rows[i].length; offsets.setAtIndex(ValueLayout.JAVA_LONG, i + 1, pos); } - return new VarBinArray.OffsetMode(dtype, rows.length, bytes.asReadOnly(), offsets, PType.I64); + return new VarBinOffsetArray(dtype, rows.length, bytes.asReadOnly(), offsets, PType.I64); } } diff --git a/docs/compatibility.md b/docs/compatibility.md index 0dc512d1b..f5ed48978 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -136,18 +136,18 @@ decoder falls into one of three shapes: | `vortex.null` | n/a | n/a | no per-row data | | `vortex.bytebool` | Zero-copy | Zero-copy | mmap slice | | `vortex.zigzag` | Lazy | Lazy | `LazyZigZagXxxArray` (I8/I16/I32/I64); broadcast → `LazyConstantXxxArray`, ADR 0010 + 0015 | -| `vortex.constant` | Lazy | Lazy | `LazyConstantXxxArray` (primitive + bool + decimal) + `VarBinArray.ConstantMode` (Utf8/Binary); per-row broadcast, no buffer, ADR 0015 | +| `vortex.constant` | Lazy | Lazy | `LazyConstantXxxArray` (primitive + bool + decimal) + `VarBinConstantArray` (Utf8/Binary); per-row broadcast, no buffer, ADR 0015 | | `vortex.ext` | Zero-copy | Zero-copy | wraps storage | | `vortex.runend` | Lazy | Lazy | `LazyRunEndXxxArray` (primitive + bool); Utf8/Binary stays Materialized (offset rebasing), ADR 0015 | | `vortex.varbin` | Zero-copy | Zero-copy | bytes + offsets slices | -| `vortex.varbinview` | Lazy | Lazy | `VarBinArray.ViewMode` — keeps views + data buffers as mmap slices | +| `vortex.varbinview` | Lazy | Lazy | `VarBinViewArray` — keeps views + data buffers as mmap slices | | `vortex.alp` | Lazy | Lazy | `LazyAlpXxxArray`; broadcast → `LazyConstantXxxArray`; patched stays Materialized, ADR 0010 + 0015 | | `vortex.alprd` | Lazy | Lazy | `LazyAlpRdDoubleArray`/`LazyAlpRdFloatArray` — left/right + patches on access | -| `vortex.dict` | Lazy | Lazy | `DictXxxArray` (numeric) + `VarBinArray.DictMode` (string), ADR 0012 | +| `vortex.dict` | Lazy | Lazy | `DictXxxArray` (numeric) + `VarBinDictArray` (string), ADR 0012 | | `vortex.sparse` | Lazy | Lazy | `LazySparseXxxArray` (primitive + bool); Utf8/Binary stays Materialized, ADR 0015 | | `vortex.sequence` | Zero-copy | Zero-copy | synthetic (no data) | | `vortex.struct` | Zero-copy | Zero-copy | `StructArray` wraps fields | -| `vortex.chunked` | Lazy | Lazy | `ChunkedXxxArray` (primitive/Bool) + `VarBinArray.ChunkedMode` (Utf8/Binary), ADR 0012 | +| `vortex.chunked` | Lazy | Lazy | `ChunkedXxxArray` (primitive/Bool) + `VarBinChunkedArray` (Utf8/Binary), ADR 0012 | | `vortex.fsst` | Materialized | Materialized | symbol-table decompression | | `vortex.list` | Lazy | Lazy | `ListArray` wraps elements + offsets children; shape inherits from child | | `vortex.listview` | Lazy | Lazy | `ListViewArray` wraps elements + offsets + sizes children | diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index ece06ded1..857094092 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -29,6 +29,7 @@ import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.StructArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinSlicedArray; import io.github.dfa1.vortex.reader.layout.Layout; import io.github.dfa1.vortex.reader.layout.LayoutDecodeContext; import io.github.dfa1.vortex.reader.layout.ZonedStatsSchema; @@ -777,7 +778,7 @@ private static Array sliceArray(Array full, long offset, long length, DType dtyp case ByteArray a -> new OffsetByteArray(dtype, length, a, offset); case BoolArray a -> new OffsetBoolArray(dtype, length, a, offset); case NullArray ignored -> new NullArray(dtype, length); - case VarBinArray a -> new VarBinArray.SlicedMode(dtype, length, a, offset); + case VarBinArray a -> new VarBinSlicedArray(dtype, length, a, offset); case StructArray s -> { // A shared nested struct column is decoded once over the full range, then sliced // per chunk by slicing each field into the same window. Field dtypes come from the diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java index d2909de15..bf688534c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java @@ -14,7 +14,7 @@ /// Stitches the per-chunk arrays of one logical column into a single view, dispatching on the /// column's [DType]. Each family gets its zero-copy composite shape (ADR 0012): primitive and /// boolean chunks fold into the `ChunkedXxxArray` records, variable-length chunks into -/// [VarBinArray.ChunkedMode], and list chunks into a stitched [ListArray] whose bulk element data +/// [VarBinChunkedArray], and list chunks into a stitched [ListArray] whose bulk element data /// stays zero-copy (the child element arrays are themselves combined recursively) while only the /// small outer offsets table is rebuilt so per-chunk offsets — which each reset to zero — become one /// cumulative table. @@ -46,8 +46,8 @@ public static Array combine(DType dtype, long totalRows, List chunks, } Array data = switch (dtype) { case DType.Bool ignored -> ChunkedBoolArray.of(dtype, totalRows, chunks); - case DType.Utf8 ignored -> VarBinArray.ChunkedMode.of(dtype, totalRows, chunks, arena); - case DType.Binary ignored -> VarBinArray.ChunkedMode.of(dtype, totalRows, chunks, arena); + case DType.Utf8 ignored -> VarBinChunkedArray.of(dtype, totalRows, chunks, arena); + case DType.Binary ignored -> VarBinChunkedArray.of(dtype, totalRows, chunks, arena); case DType.List list -> combineLists(list, totalRows, chunks, arena); case DType.Primitive prim -> combinePrimitive(prim.ptype(), dtype, totalRows, chunks); default -> throw new VortexException("unsupported dtype for chunked layout: " + dtype); @@ -77,7 +77,7 @@ private static Array combinePrimitive(PType ptype, DType dtype, long totalRows, /// /// An entirely-null chunk decodes to a [NullArray] rather than a [ListArray] (e.g. a /// `vortex.null` flat, or `vortex.constant` with a null scalar, #269). Mirroring the sibling - /// [VarBinArray.ChunkedMode] fix, such a chunk contributes `n` zero-length list rows: it adds no + /// [VarBinChunkedArray] fix, such a chunk contributes `n` zero-length list rows: it adds no /// elements, so it is skipped by the recursive element combine, and its `n` outer offsets simply /// repeat the running element count. Row-level nullability is preserved separately by the /// caller's validity bitmap. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java index fd1ff6e82..0bc96794e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java @@ -2,143 +2,29 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.lang.foreign.ValueLayout; -import java.nio.charset.StandardCharsets; -import java.util.Objects; import java.util.Optional; import java.util.function.IntConsumer; -/// Sealed interface for variable-length binary / UTF-8 string columns. +/// Interface for variable-length binary / UTF-8 string columns. /// -/// Implementations: [OffsetMode] for standard offset-based layout, [DictMode] for -/// dictionary-encoded strings, [ChunkedMode] for multi-chunk columns, [ViewMode] for Arrow -/// StringView / BinaryView layout (16-byte view per row + zero or more shared data buffers), -/// [SlicedMode] for a zero-copy row-shifted view, and [ConstantMode] for a `vortex.constant` -/// value broadcast across every row. All accessors resolve transparently regardless of mode; -/// only [OffsetMode] exposes [OffsetMode#offsetsSegment()] and [OffsetMode#offsetsPtype()]. -public sealed interface VarBinArray extends Array - permits VarBinArray.OffsetMode, VarBinArray.DictMode, VarBinArray.ChunkedMode, - VarBinArray.ViewMode, VarBinArray.SlicedMode, VarBinArray.ConstantMode { - - /// Sliced view over a [VarBinArray]: every accessor delegates to `inner` - /// with the row index shifted by `offset`. Used by the scan iterator to - /// surface a column that was decoded once (because it shares a single - /// flat layout across multiple aligned chunks) as a per-chunk slice - /// without copying. - /// - /// @param dtype logical element type (typically [DType.Utf8] or [DType.Binary]) - /// @param length number of logical elements in this slice - /// @param inner underlying VarBin array - /// @param offset starting row index into `inner` - record SlicedMode(DType dtype, long length, VarBinArray inner, long offset) - implements VarBinArray { - - @Override - public MemorySegment bytesSegment() { - return inner.bytesSegment(); - } - - /// Delegates the probe to the wrapped array — empty if the inner is - /// itself composite (chunked / view). - /// - /// @return the inner array's segment if segment-backed, otherwise empty - @Override - public Optional segmentIfPresent() { - return inner.segmentIfPresent(); - } - - @Override - public byte[] getBytes(long i) { - return inner.getBytes(i + offset); - } - - @Override - public String getString(long i) { - return inner.getString(i + offset); - } - - @Override - public int getByteLength(long i) { - return inner.getByteLength(i + offset); - } - - @Override - public void forEachByteLength(IntConsumer c) { - for (long i = 0; i < length; i++) { - c.accept(getByteLength(i)); - } - } - - @Override - public VarBinArray limited(long rows) { - if (rows >= length) { - return this; - } - return new SlicedMode(dtype, rows, inner, offset); - } - } - - /// Metadata-only mode for `vortex.constant` Utf8/Binary columns: a single value broadcast - /// across `length` logical rows. No buffer is allocated and no per-row materialization - /// occurs — every accessor returns the same shared bytes, mirroring the - /// `LazyConstantXxxArray` family the primitive array types use for the same encoding. - /// - /// @param dtype logical element type (Utf8 or Binary) - /// @param length number of logical rows the constant is broadcast across - /// @param bytes the constant value's raw bytes, shared across every row; [#getBytes(long)] - /// clones it per that method's copy contract - @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. - record ConstantMode(DType dtype, long length, byte[] bytes) implements VarBinArray { - - @Override - public MemorySegment bytesSegment() { - return MemorySegment.NULL; - } - - /// No single contiguous segment backs a broadcast constant. - /// - /// @return always empty - @Override - public Optional segmentIfPresent() { - return Optional.empty(); - } - - @Override - public byte[] getBytes(long i) { - Objects.checkIndex(i, length); - return bytes.clone(); - } - - @Override - public String getString(long i) { - Objects.checkIndex(i, length); - return new String(bytes, StandardCharsets.UTF_8); - } - - @Override - public int getByteLength(long i) { - Objects.checkIndex(i, length); - return bytes.length; - } - - @Override - public void forEachByteLength(IntConsumer c) { - int len = bytes.length; - for (long i = 0; i < length; i++) { - c.accept(len); - } - } - - @Override - public VarBinArray limited(long rows) { - return rows >= length ? this : new ConstantMode(dtype, rows, bytes); - } - } +/// Implementations: [VarBinOffsetArray] for standard offset-based layout, [VarBinDictArray] for +/// dictionary-encoded strings, [VarBinChunkedArray] for multi-chunk columns, [VarBinViewArray] +/// for Arrow StringView / BinaryView layout (16-byte view per row + zero or more shared data +/// buffers), [VarBinSlicedArray] for a zero-copy row-shifted view, and [VarBinConstantArray] for +/// a `vortex.constant` value broadcast across every row. All accessors resolve transparently +/// regardless of implementation; only [VarBinOffsetArray] exposes +/// [VarBinOffsetArray#offsetsSegment()] and [VarBinOffsetArray#offsetsPtype()]. +/// +/// Deliberately `non-sealed` (unlike the sibling `ByteArray`/`LongArray`/… families reopened the +/// same way off [Array]'s seal): which representation backs a given `VarBinArray` is an +/// implementation detail, not part of the type's contract, so new representations live as +/// ordinary top-level classes in this package rather than requiring this interface to enumerate +/// them. +public non-sealed interface VarBinArray extends Array { /// Returns the concatenated raw bytes segment backing all elements. /// @@ -148,7 +34,7 @@ public VarBinArray limited(long rows) { /// Returns the concatenated raw bytes segment directly — the primary data /// buffer is already materialized, so no copy or allocation is needed. /// Note this is the data buffer only; the per-row offsets are exposed - /// separately by [OffsetMode#offsetsSegment()]. + /// separately by [VarBinOffsetArray#offsetsSegment()]. /// /// @param arena unused; the existing bytes segment is returned as-is /// @return the concatenated raw bytes [MemorySegment] @@ -194,18 +80,18 @@ default Optional segmentIfPresent() { /// @return a `VarBinArray` containing the first `rows` elements VarBinArray limited(long rows); - /// Materializes any `VarBinArray` into a flat [OffsetMode]. The fast path - /// returns `src` unchanged when it is already an [OffsetMode]. Other modes - /// (ViewMode in particular) walk every row through the typed accessors, copy the bytes - /// into a fresh contiguous segment allocated from `arena`, and build an I64 + /// Materializes any `VarBinArray` into a flat [VarBinOffsetArray]. The fast path + /// returns `src` unchanged when it is already a [VarBinOffsetArray]. Other implementations + /// ([VarBinViewArray] in particular) walk every row through the typed accessors, copy the + /// bytes into a fresh contiguous segment allocated from `arena`, and build an I64 /// offsets table. Used by parent decoders (dict, sparse, runend) whose downstream code /// depends on the bytes-plus-offsets shape. /// /// @param src any VarBinArray /// @param arena allocator for the materialized bytes and offsets segments - /// @return an OffsetMode view over the same logical content - static OffsetMode toOffsetMode(VarBinArray src, SegmentAllocator arena) { - if (src instanceof OffsetMode om) { + /// @return a [VarBinOffsetArray] view over the same logical content + static VarBinOffsetArray toOffsetMode(VarBinArray src, SegmentAllocator arena) { + if (src instanceof VarBinOffsetArray om) { return om; } long n = src.length(); @@ -223,30 +109,7 @@ static OffsetMode toOffsetMode(VarBinArray src, SegmentAllocator arena) { bytePos += b.length; outOffsets.setAtIndex(VortexFormat.LE_LONG, i + 1, bytePos); } - return new OffsetMode(src.dtype(), n, outBytes.asReadOnly(), outOffsets, PType.I64); - } - - /// Validates that element bytes `[start, end)` lie inside `bytes` and returns the - /// element length. - /// - /// Offsets arrive from an untrusted file and are deliberately not scanned at decode - /// time (VarBin decode stays zero-copy and lazy), so a non-monotonic, negative or - /// past-the-end pair has to be rejected here — as a [VortexException], never as a raw - /// `NegativeArraySizeException` from `new byte[end - start]` or an - /// `IndexOutOfBoundsException` from [MemorySegment#copy(MemorySegment, long, MemorySegment, long, long)] - /// (ADR 0003). - /// - /// @param bytes data buffer the offsets index into - /// @param start start offset of the element - /// @param end end offset of the element, exclusive - /// @return the element length in bytes - private static int checkedLength(MemorySegment bytes, long start, long end) { - long len = end - start; - if (start < 0 || len < 0 || end > bytes.byteSize() || len > Integer.MAX_VALUE) { - throw new VortexException("varbin element bytes [" + start + ", " + end - + ") out of range for a data buffer of " + bytes.byteSize() + " bytes"); - } - return (int) len; + return new VarBinOffsetArray(src.dtype(), n, outBytes.asReadOnly(), outOffsets, PType.I64); } /// Creates a dict-mode `VarBinArray`. Lengths and bytes are resolved via the @@ -264,640 +127,7 @@ static VarBinArray ofDict(DType dtype, long n, MemorySegment dictValBytes, MemorySegment dictValOffsets, PType dictValOffPType, MemorySegment dictCodesSegs, PType dictCodesPType) { - return new DictMode(dtype, n, dictValBytes, dictValOffsets, dictValOffPType, + return new VarBinDictArray(dtype, n, dictValBytes, dictValOffsets, dictValOffPType, dictCodesSegs, dictCodesPType); } - - /// Standard offset-based `VarBinArray`. - /// - /// Element `i` occupies `bytesSegment[offsetsSegment[i]..offsetsSegment[i+1]]`. - /// - /// @param dtype logical type (Utf8 or Binary) - /// @param length number of variable-length elements - /// @param bytesSegment concatenated raw byte data for all elements - /// @param offsetsSegment offsets segment of length `length + 1` - /// @param offsetsPtype physical type of the offsets values (I32/U32 or I64/U64) - @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. - record OffsetMode(DType dtype, long length, MemorySegment bytesSegment, - MemorySegment offsetsSegment, PType offsetsPtype) - implements VarBinArray { - - @Override - public byte[] getBytes(long i) { - long start = readOffset(i); - long end = readOffset(i + 1); - int len = checkedLength(bytesSegment, start, end); - byte[] out = new byte[len]; - MemorySegment.copy(bytesSegment, start, MemorySegment.ofArray(out), 0, len); - return out; - } - - @Override - public String getString(long i) { - return new String(getBytes(i), StandardCharsets.UTF_8); - } - - @Override - public int getByteLength(long i) { - return checkedLength(bytesSegment, readOffset(i), readOffset(i + 1)); - } - - @Override - public void forEachByteLength(IntConsumer c) { - long n = length; - // The loop reads offsets[0..n] at a fixed stride and must stay uniform to - // vectorize (CLAUDE.md hot-loop rule), so the untrusted offsets segment is - // sized once here rather than bounds-checked per row. A non-monotonic pair - // still yields a negative length to the consumer; the typed accessors - // ([#getBytes(long)], [#getByteLength(long)]) reject it when the row is read. - checkOffsetsExtent(n); - if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { - for (long i = 0; i < n; i++) { - c.accept(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i + 1) - - offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); - } - } else { - for (long i = 0; i < n; i++) { - c.accept((int) (offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i + 1) - - offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i))); - } - } - } - - @Override - public VarBinArray limited(long rows) { - if (rows >= length) { - return this; - } - checkOffsetsExtent(rows); - long byteEnd = readOffset(rows); - if (byteEnd < 0 || byteEnd > bytesSegment.byteSize()) { - throw new VortexException("varbin offset " + byteEnd + " at row " + rows - + " out of range for a data buffer of " + bytesSegment.byteSize() + " bytes"); - } - int offBytes = offsetWidth(); - MemorySegment newOffsetsSeg = offsetsSegment.asSlice(0, (rows + 1) * offBytes); - return new OffsetMode(dtype, rows, - bytesSegment.asSlice(0, byteEnd), newOffsetsSeg, offsetsPtype); - } - - /// Verifies the offsets segment holds the `rows + 1` offsets the array claims. - /// - /// @param rows number of rows whose offsets are about to be read - private void checkOffsetsExtent(long rows) { - int width = offsetWidth(); - if (rows + 1 > offsetsSegment.byteSize() / width) { - throw new VortexException("varbin offsets segment of " + offsetsSegment.byteSize() - + " bytes holds fewer than " + (rows + 1) + " " + offsetsPtype + " offsets"); - } - } - - private int offsetWidth() { - return (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) ? Integer.BYTES : Long.BYTES; - } - - /// Reads offset `i` at the width of [#offsetsPtype]. - /// - /// The offsets segment comes straight from an untrusted file, so an index past its - /// end must surface as a [VortexException] rather than a raw - /// `IndexOutOfBoundsException` (ADR 0003). - /// - /// @param i zero-based offset index, in `[0, length]` - /// @return the offset value widened to a signed long - private long readOffset(long i) { - try { - if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { - return offsetsSegment.getAtIndex(VortexFormat.LE_INT, i); - } - return offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i); - } catch (IndexOutOfBoundsException e) { - throw new VortexException("varbin offset index " + i + " (" + offsetsPtype - + ") out of range for an offsets segment of " - + offsetsSegment.byteSize() + " bytes", e); - } - } - } - - /// Dictionary-encoded `VarBinArray`. - /// - /// Stores dictionary values and per-row codes; all accessors resolve through the - /// dictionary without materializing strings at construction time. - /// - /// @param dtype logical type (Utf8 or Binary) - /// @param length number of logical elements (rows) - /// @param bytesSegment concatenated raw bytes for all dictionary values - /// @param dictValOffsets offsets into `bytesSegment` for each dictionary entry (length = dictSize + 1) - /// @param dictValOffPType physical type of the dictionary value offsets - /// @param dictCodesSegs per-row dictionary code indices (length = `length`) - /// @param dictCodesPType physical type of the dictionary codes - @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. - record DictMode(DType dtype, long length, MemorySegment bytesSegment, - MemorySegment dictValOffsets, PType dictValOffPType, - MemorySegment dictCodesSegs, PType dictCodesPType) - implements VarBinArray { - - @Override - public byte[] getBytes(long i) { - long code = dictReadCode(i); - long start = dictReadOff(code); - long end = dictReadOff(code + 1); - int len = checkedLength(bytesSegment, start, end); - byte[] out = new byte[len]; - MemorySegment.copy(bytesSegment, start, MemorySegment.ofArray(out), 0, len); - return out; - } - - @Override - public String getString(long i) { - return new String(getBytes(i), StandardCharsets.UTF_8); - } - - @Override - public int getByteLength(long i) { - long code = dictReadCode(i); - return checkedLength(bytesSegment, dictReadOff(code), dictReadOff(code + 1)); - } - - @Override - public void forEachByteLength(IntConsumer c) { - // Hot loop: hoist both ptype dispatches out of the per-row body so C2 sees a - // uniform, fixed-stride loop (CLAUDE.md hot-loop rule). A variable-target - // switch(dictValOffPType)/switch(dictCodesPType) per element blocks C2 - // superword vectorization and adds invariant width/bounds arithmetic on every - // row (regression introduced by #215). I32 dict-value offsets are by far the - // most common (FSST + most dict encodings emit 32-bit offsets), so the fast - // path reads offsets at a constant 4-byte stride and branch-splits the code - // read once; wider offset ptypes take the general per-row path. - // - // The per-row body must stay branch-free, so the untrusted-code bounds check is - // a boundary catch-and-wrap around the whole loop rather than a test per row: - // an out-of-pool code (or a truncated codes buffer) trips the segment access and - // must surface as a VortexException, never a raw IndexOutOfBoundsException. The - // handler re-validates on the cold path so the blame — and the exception type — - // land on whichever input actually failed, including the caller's own consumer. - try { - if (dictValOffPType == PType.I32) { - forEachI32OffsetByteLength(c); - } else { - for (long i = 0; i < length; i++) { - long code = dictReadCode(i); - c.accept((int) (dictReadOff(code + 1) - dictReadOff(code))); - } - } - } catch (IndexOutOfBoundsException e) { - throw attribute(e); - } - } - - /// Cold path: works out which untrusted input made a bulk length walk run off the - /// end, so the message names the segment that actually failed. - /// - /// Re-checks the codes extent, then every code against the value-offsets extent — - /// affordable here because it only runs after something has already thrown. When - /// both check out the failure came from the caller's [IntConsumer], and that - /// exception is returned unchanged rather than relabeled as malformed input. - /// - /// @param e the out-of-bounds failure raised by the walk - /// @return a [VortexException] describing the malformed input, or `e` itself - private RuntimeException attribute(IndexOutOfBoundsException e) { - int codeWidth = dictCodesPType.byteSize(); - if (length > dictCodesSegs.byteSize() / codeWidth) { - return new VortexException("dict codes segment of " + dictCodesSegs.byteSize() - + " bytes holds fewer than " + length + " " + dictCodesPType + " codes"); - } - long offsetCount = dictValOffsets.byteSize() / dictValOffPType.byteSize(); - for (long i = 0; i < length; i++) { - long code = readCodeAt(i); - if (code < 0 || code + 1 >= offsetCount) { - return new VortexException("dict code " + code + " at row " + i - + " out of range for " + offsetCount + " value offsets"); - } - } - return e; - } - - /// Fast path of [#forEachByteLength(IntConsumer)] for I32 dict-value offsets: the - /// code-ptype switch is hoisted out of the loop so each specialized loop body reads - /// codes at a single fixed stride and computes lengths from a constant 4-byte offset - /// stride, leaving the per-row body uniform and vectorizable. - /// - /// @param c consumer called once per row with the byte length at that index - private void forEachI32OffsetByteLength(IntConsumer c) { - long n = length; - switch (dictCodesPType) { - case U8 -> { - for (long i = 0; i < n; i++) { - c.accept(i32OffsetLength( - Byte.toUnsignedLong(dictCodesSegs.get(ValueLayout.JAVA_BYTE, i)))); - } - } - case U16 -> { - for (long i = 0; i < n; i++) { - c.accept(i32OffsetLength( - Short.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_SHORT, i)))); - } - } - case U32 -> { - for (long i = 0; i < n; i++) { - c.accept(i32OffsetLength( - Integer.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i)))); - } - } - case I32 -> { - for (long i = 0; i < n; i++) { - c.accept(i32OffsetLength(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i))); - } - } - case I64, U64 -> { - for (long i = 0; i < n; i++) { - c.accept(i32OffsetLength(dictCodesSegs.getAtIndex(VortexFormat.LE_LONG, i))); - } - } - default -> throw new VortexException("unsupported codes ptype: " + dictCodesPType); - } - } - - /// Byte length of the dictionary entry `code` when the value offsets are I32: - /// `offsets[code + 1] - offsets[code]` read at a constant 4-byte stride. The - /// segment access itself bounds-checks, so an out-of-range code is caught without - /// the per-row width recomputation that [#dictReadOff(long)] performs. - /// - /// @param code zero-based dictionary entry index (in `[0, dictSize)`) - /// @return the byte length of dictionary entry `code` - private int i32OffsetLength(long code) { - return dictValOffsets.getAtIndex(VortexFormat.LE_INT, code + 1) - - dictValOffsets.getAtIndex(VortexFormat.LE_INT, code); - } - - @Override - public VarBinArray limited(long rows) { - if (rows >= length) { - return this; - } - int codeBytes = dictCodesPType.byteSize(); - if (rows > dictCodesSegs.byteSize() / codeBytes) { - throw new VortexException("dict codes segment of " + dictCodesSegs.byteSize() - + " bytes holds fewer than " + rows + " " + dictCodesPType + " codes"); - } - return VarBinArray.ofDict(dtype, rows, bytesSegment, dictValOffsets, dictValOffPType, - dictCodesSegs.asSlice(0, rows * codeBytes), dictCodesPType); - } - - /// Reads the dictionary code for row `i` at the width of [#dictCodesPType]. - /// - /// The codes buffer is untrusted and may be shorter than [#length()], so an - /// overrun is reported as a [VortexException] instead of a raw - /// `IndexOutOfBoundsException` (ADR 0003). - /// - /// @param i zero-based row index - /// @return the dictionary code, widened to a signed long - private long dictReadCode(long i) { - try { - return readCodeAt(i); - } catch (IndexOutOfBoundsException e) { - throw new VortexException("dict code index " + i + " (" + dictCodesPType - + ") out of range for a codes segment of " - + dictCodesSegs.byteSize() + " bytes", e); - } - } - - private long readCodeAt(long i) { - return switch (dictCodesPType) { - case U8 -> Byte.toUnsignedLong(dictCodesSegs.get(ValueLayout.JAVA_BYTE, i)); - case U16 -> Short.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_SHORT, i)); - case U32 -> Integer.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i)); - case I32 -> dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i); - case I64, U64 -> dictCodesSegs.getAtIndex(VortexFormat.LE_LONG, i); - default -> throw new VortexException("unsupported codes ptype: " + dictCodesPType); - }; - } - - /// Reads dictionary-value offset `i` at the true width of [#dictValOffPType]. - /// - /// Offsets can arrive at any integer width — FSST decompresses its values to a - /// child with I32 offsets, legacy dicts use I64, and narrow sequence-encoded - /// offsets keep their U8/U16 ptype on the wire. Reading at the wrong width (e.g. - /// an 8-byte read against a 4-byte-stride buffer) walks off the segment. The - /// buffer bounds and ptype are untrusted input, so both are checked here and a - /// [VortexException] is thrown rather than a bare `IndexOutOfBoundsException` - /// (ADR 0003). - /// - /// @param i zero-based offset index (in `[0, dictSize]`) - /// @return the offset value widened to a signed long - private long dictReadOff(long i) { - int width = dictValOffPType.byteSize(); - long byteOffset = i * width; - if (i < 0 || byteOffset + width > dictValOffsets.byteSize()) { - throw new VortexException("dict value offset index " + i + " (" + dictValOffPType - + ", " + width + "-byte) out of range for offsets segment of " - + dictValOffsets.byteSize() + " bytes"); - } - return switch (dictValOffPType) { - case U8 -> Byte.toUnsignedLong(dictValOffsets.get(ValueLayout.JAVA_BYTE, byteOffset)); - case I8 -> dictValOffsets.get(ValueLayout.JAVA_BYTE, byteOffset); - case U16 -> Short.toUnsignedLong(dictValOffsets.get(VortexFormat.LE_SHORT, byteOffset)); - case I16 -> dictValOffsets.get(VortexFormat.LE_SHORT, byteOffset); - case U32 -> Integer.toUnsignedLong(dictValOffsets.get(VortexFormat.LE_INT, byteOffset)); - case I32 -> dictValOffsets.get(VortexFormat.LE_INT, byteOffset); - case I64, U64 -> dictValOffsets.get(VortexFormat.LE_LONG, byteOffset); - default -> throw new VortexException( - "unsupported dict value offset ptype: " + dictValOffPType); - }; - } - } - - /// Multi-chunk `VarBinArray` — wraps a list of child `VarBinArray`s plus - /// cumulative row offsets. Per-row accessors binary-search `offsets` to find the - /// owning chunk and delegate. Per ADR 0012, preserves zero-copy on multi-chunk Utf8 / - /// Binary columns: each chunk's underlying segments stay live (mmap slices); no concat. - /// - /// [#bytesSegment()] is the [MemorySegment#NULL] sentinel — chunked - /// arrays have no single contiguous bytes segment. Callers that need contiguous - /// bytes must materialize via the chunked children. - /// - /// @param dtype logical element type (Utf8 or Binary) - /// @param length total logical row count - /// @param children chunk arrays in scan order; each is itself a [VarBinArray] - /// @param offsets cumulative row counts; length = `children.length + 1` - @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. - record ChunkedMode(DType dtype, long length, VarBinArray[] children, long[] offsets) - implements VarBinArray { - - /// Builds a `ChunkedMode` from a list of chunk arrays that are already all typed - /// [VarBinArray]s (no all-null [NullArray] chunks). Used by [#limited(long)], - /// whose children are always concrete `VarBinArray`s. - /// - /// @param dtype logical element type - /// @param totalRows expected total row count - /// @param chunks non-empty list of [VarBinArray] chunks - /// @return a new `ChunkedMode` - /// @throws VortexException on empty input, non-[VarBinArray] chunks, or row-count mismatch - public static ChunkedMode of(DType dtype, long totalRows, - java.util.List chunks) { - return of(dtype, totalRows, chunks, null); - } - - /// Builds a `ChunkedMode` from a list of chunk arrays. - /// - /// An entirely-null chunk decodes to a [NullArray] rather than a [VarBinArray] - /// (e.g. a `vortex.null` flat, or `vortex.constant` with a null scalar, #269). - /// Such a chunk is materialized into an all-null [OffsetMode] of the same row - /// count — every row zero-length, all offsets zero — so the chunked column keeps - /// a uniform `VarBinArray` shape. Row-level nullability is preserved separately by - /// the caller's validity bitmap. - /// - /// @param dtype logical element type - /// @param totalRows expected total row count - /// @param chunks non-empty list of chunk arrays; each a [VarBinArray] or [NullArray] - /// @param arena allocator for the offsets segment of a materialized null chunk; - /// may be `null` only when no chunk is a [NullArray] - /// @return a new `ChunkedMode` - /// @throws VortexException on empty input, non-`VarBinArray`/`NullArray` chunks, - /// or row-count mismatch - public static ChunkedMode of(DType dtype, long totalRows, - java.util.List chunks, SegmentAllocator arena) { - if (chunks.isEmpty()) { - throw new VortexException("VarBinArray.ChunkedMode: empty chunk list"); - } - var typed = new java.util.ArrayList(chunks.size()); - for (Array c : chunks) { - Array data = c instanceof MaskedArray m ? m.inner() : c; - if (data instanceof ChunkedMode nested) { - java.util.Collections.addAll(typed, nested.children); - } else if (data instanceof VarBinArray vb) { - typed.add(vb); - } else if (data instanceof NullArray na) { - if (arena == null) { - throw new VortexException( - "VarBinArray.ChunkedMode: null chunk requires an allocator"); - } - typed.add(allNull(dtype, na.length(), arena)); - } else { - throw new VortexException("VarBinArray.ChunkedMode: chunk is not a VarBinArray: " - + data.getClass().getSimpleName()); - } - } - long[] off = new long[typed.size() + 1]; - for (int i = 0; i < typed.size(); i++) { - off[i + 1] = off[i] + typed.get(i).length(); - } - if (off[off.length - 1] != totalRows) { - throw new VortexException("VarBinArray.ChunkedMode: chunk rows sum to " - + off[off.length - 1] + ", expected " + totalRows); - } - return new ChunkedMode(dtype, totalRows, typed.toArray(VarBinArray[]::new), off); - } - - /// Builds an all-null [OffsetMode] of `n` rows: an empty bytes segment and an - /// offsets segment of `n + 1` zeros, so every row is zero-length. Row nullability - /// is carried by the caller's validity bitmap, not the byte data. - /// - /// @param dtype logical element type (Utf8 or Binary) - /// @param n number of all-null rows - /// @param arena allocator for the offsets segment - /// @return an [OffsetMode] with `n` zero-length rows - private static OffsetMode allNull(DType dtype, long n, SegmentAllocator arena) { - MemorySegment offsets = arena.allocate((n + 1) * Long.BYTES, Long.BYTES); - return new OffsetMode(dtype, n, MemorySegment.NULL, offsets, PType.I64); - } - - private int findChunk(long i) { - int hit = java.util.Arrays.binarySearch(offsets, i); - int idx = hit >= 0 ? hit : -hit - 2; - if (idx >= children.length) { - idx = children.length - 1; - } - return idx; - } - - @Override - public MemorySegment bytesSegment() { - return MemorySegment.NULL; - } - - /// No single contiguous segment — chunked data lives across children. - /// - /// @return always empty - @Override - public Optional segmentIfPresent() { - return Optional.empty(); - } - - @Override - public byte[] getBytes(long i) { - int c = findChunk(i); - return children[c].getBytes(i - offsets[c]); - } - - @Override - public String getString(long i) { - int c = findChunk(i); - return children[c].getString(i - offsets[c]); - } - - @Override - public int getByteLength(long i) { - int c = findChunk(i); - return children[c].getByteLength(i - offsets[c]); - } - - @Override - public void forEachByteLength(IntConsumer c) { - for (VarBinArray child : children) { - child.forEachByteLength(c); - } - } - - @Override - public VarBinArray limited(long rows) { - if (rows >= length) { - return this; - } - // Keep full children that fit, recursively limited the boundary child. - var kept = new java.util.ArrayList(children.length); - for (int i = 0; i < children.length; i++) { - long start = offsets[i]; - long end = offsets[i + 1]; - if (start >= rows) { - break; - } - if (end <= rows) { - kept.add(children[i]); - } else { - kept.add(children[i].limited(rows - start)); - } - } - return ChunkedMode.of(dtype, rows, kept); - } - } - - /// Arrow StringView / BinaryView `VarBinArray`. - /// - /// Each row is a 16-byte view in `views`: bytes 0-3 are the u32 size; for - /// sizes ≤ 12 bytes the data is inlined in bytes 4..15; for sizes > 12 bytes - /// bytes 4-7 hold a 4-byte prefix (ignored on read), bytes 8-11 the u32 buffer - /// index into `dataBufs`, and bytes 12-15 the u32 offset within that - /// buffer. Per-row accessors resolve the view on demand — no concat or - /// materialization at construction time. - /// - /// [#bytesSegment()] returns [MemorySegment#NULL] because there is - /// no single contiguous bytes segment; callers needing one must materialize via - /// the typed accessors. - /// - /// @param dtype logical element type (Utf8 or Binary) - /// @param length total logical row count - /// @param views 16-byte view per row; length must be ≥ `length * 16` - /// @param dataBufs zero or more shared data buffers referenced by long views - @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable refs that flow through pipelines without ever being compared. - record ViewMode(DType dtype, long length, MemorySegment views, MemorySegment[] dataBufs) - implements VarBinArray { - - private static final int VIEW_SIZE = 16; - private static final int MAX_INLINED_SIZE = 12; - - @Override - public MemorySegment bytesSegment() { - return MemorySegment.NULL; - } - - /// No single contiguous segment — view rows reference shared data buffers. - /// - /// @return always empty - @Override - public Optional segmentIfPresent() { - return Optional.empty(); - } - - @Override - public int getByteLength(long i) { - return checkedSize(views.get(VortexFormat.LE_INT, viewOffset(i))); - } - - @Override - public byte[] getBytes(long i) { - long viewOff = viewOffset(i); - int size = checkedSize(views.get(VortexFormat.LE_INT, viewOff)); - byte[] out = new byte[size]; - if (size <= MAX_INLINED_SIZE) { - // Inlined data always fits the remaining 12 bytes of the view itself. - MemorySegment.copy(views, viewOff + 4, MemorySegment.ofArray(out), 0, size); - } else { - int bufferIndex = views.get(VortexFormat.LE_INT, viewOff + 8); - long srcOffset = Integer.toUnsignedLong(views.get(VortexFormat.LE_INT, viewOff + 12)); - if (bufferIndex < 0 || bufferIndex >= dataBufs.length) { - throw new VortexException("varbin view at row " + i + " references data buffer " - + bufferIndex + " of " + dataBufs.length); - } - MemorySegment buf = dataBufs[bufferIndex]; - if (srcOffset + size > buf.byteSize()) { - throw new VortexException("varbin view bytes [" + srcOffset + ", " - + (srcOffset + size) + ") out of range for data buffer " + bufferIndex - + " of " + buf.byteSize() + " bytes"); - } - MemorySegment.copy(buf, srcOffset, MemorySegment.ofArray(out), 0, size); - } - return out; - } - - @Override - public String getString(long i) { - return new String(getBytes(i), StandardCharsets.UTF_8); - } - - @Override - public void forEachByteLength(IntConsumer c) { - long n = length; - // Sized once, outside the loop, so the per-row body stays uniform — same - // trade-off as OffsetMode: a negative size on the wire still reaches the - // consumer, and [#getBytes(long)] rejects it when the row is read. - checkViewsExtent(n); - for (long i = 0; i < n; i++) { - c.accept(views.get(VortexFormat.LE_INT, i * VIEW_SIZE)); - } - } - - @Override - public VarBinArray limited(long rows) { - if (rows >= length) { - return this; - } - checkViewsExtent(rows); - return new ViewMode(dtype, rows, views.asSlice(0, rows * VIEW_SIZE), dataBufs); - } - - /// Byte offset of view `i`, rejecting a row the views segment does not cover. - /// - /// @param i zero-based row index - /// @return the byte offset of the 16-byte view for row `i` - private long viewOffset(long i) { - long off = i * VIEW_SIZE; - if (i < 0 || off + VIEW_SIZE > views.byteSize()) { - throw new VortexException("varbin view index " + i - + " out of range for a views segment of " + views.byteSize() + " bytes"); - } - return off; - } - - /// Verifies the views segment holds `rows` complete 16-byte views. - /// - /// @param rows number of rows whose views are about to be read - private void checkViewsExtent(long rows) { - if (rows > views.byteSize() / VIEW_SIZE) { - throw new VortexException("varbin views segment of " + views.byteSize() - + " bytes holds fewer than " + rows + " views"); - } - } - - /// Rejects a negative element size read from a view header, which would otherwise - /// reach `new byte[size]` as a `NegativeArraySizeException` (ADR 0003). - /// - /// @param size element size read from the view - /// @return `size` when it is non-negative - private static int checkedSize(int size) { - if (size < 0) { - throw new VortexException("negative varbin view size " + size); - } - return size; - } - } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArrays.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArrays.java new file mode 100644 index 000000000..a9b8e6dc8 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArrays.java @@ -0,0 +1,35 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.error.VortexException; + +import java.lang.foreign.MemorySegment; + +/// Package-private helpers shared by the `VarBin*Array` implementations of [VarBinArray]. +final class VarBinArrays { + + private VarBinArrays() { + } + + /// Validates that element bytes `[start, end)` lie inside `bytes` and returns the + /// element length. + /// + /// Offsets arrive from an untrusted file and are deliberately not scanned at decode + /// time (VarBin decode stays zero-copy and lazy), so a non-monotonic, negative or + /// past-the-end pair has to be rejected here — as a [VortexException], never as a raw + /// `NegativeArraySizeException` from `new byte[end - start]` or an + /// `IndexOutOfBoundsException` from [MemorySegment#copy(MemorySegment, long, MemorySegment, long, long)] + /// (ADR 0003). + /// + /// @param bytes data buffer the offsets index into + /// @param start start offset of the element + /// @param end end offset of the element, exclusive + /// @return the element length in bytes + static int checkedLength(MemorySegment bytes, long start, long end) { + long len = end - start; + if (start < 0 || len < 0 || end > bytes.byteSize() || len > Integer.MAX_VALUE) { + throw new VortexException("varbin element bytes [" + start + ", " + end + + ") out of range for a data buffer of " + bytes.byteSize() + " bytes"); + } + return (int) len; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArray.java new file mode 100644 index 000000000..78dea415f --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArray.java @@ -0,0 +1,175 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.Optional; +import java.util.function.IntConsumer; + +/// Multi-chunk [VarBinArray] — wraps a list of child `VarBinArray`s plus +/// cumulative row offsets. Per-row accessors binary-search `offsets` to find the +/// owning chunk and delegate. Per ADR 0012, preserves zero-copy on multi-chunk Utf8 / +/// Binary columns: each chunk's underlying segments stay live (mmap slices); no concat. +/// +/// [#bytesSegment()] is the [MemorySegment#NULL] sentinel — chunked +/// arrays have no single contiguous bytes segment. Callers that need contiguous +/// bytes must materialize via the chunked children. +/// +/// @param dtype logical element type (Utf8 or Binary) +/// @param length total logical row count +/// @param children chunk arrays in scan order; each is itself a [VarBinArray] +/// @param offsets cumulative row counts; length = `children.length + 1` +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record VarBinChunkedArray(DType dtype, long length, VarBinArray[] children, long[] offsets) + implements VarBinArray { + + /// Builds a `VarBinChunkedArray` from a list of chunk arrays that are already all typed + /// [VarBinArray]s (no all-null [NullArray] chunks). Used by [#limited(long)], + /// whose children are always concrete `VarBinArray`s. + /// + /// @param dtype logical element type + /// @param totalRows expected total row count + /// @param chunks non-empty list of [VarBinArray] chunks + /// @return a new `VarBinChunkedArray` + /// @throws VortexException on empty input, non-[VarBinArray] chunks, or row-count mismatch + public static VarBinChunkedArray of(DType dtype, long totalRows, + java.util.List chunks) { + return of(dtype, totalRows, chunks, null); + } + + /// Builds a `VarBinChunkedArray` from a list of chunk arrays. + /// + /// An entirely-null chunk decodes to a [NullArray] rather than a [VarBinArray] + /// (e.g. a `vortex.null` flat, or `vortex.constant` with a null scalar, #269). + /// Such a chunk is materialized into an all-null [VarBinOffsetArray] of the same row + /// count — every row zero-length, all offsets zero — so the chunked column keeps + /// a uniform `VarBinArray` shape. Row-level nullability is preserved separately by + /// the caller's validity bitmap. + /// + /// @param dtype logical element type + /// @param totalRows expected total row count + /// @param chunks non-empty list of chunk arrays; each a [VarBinArray] or [NullArray] + /// @param arena allocator for the offsets segment of a materialized null chunk; + /// may be `null` only when no chunk is a [NullArray] + /// @return a new `VarBinChunkedArray` + /// @throws VortexException on empty input, non-`VarBinArray`/`NullArray` chunks, + /// or row-count mismatch + public static VarBinChunkedArray of(DType dtype, long totalRows, + java.util.List chunks, SegmentAllocator arena) { + if (chunks.isEmpty()) { + throw new VortexException("VarBinChunkedArray: empty chunk list"); + } + var typed = new java.util.ArrayList(chunks.size()); + for (Array c : chunks) { + Array data = c instanceof MaskedArray m ? m.inner() : c; + if (data instanceof VarBinChunkedArray nested) { + java.util.Collections.addAll(typed, nested.children); + } else if (data instanceof VarBinArray vb) { + typed.add(vb); + } else if (data instanceof NullArray na) { + if (arena == null) { + throw new VortexException( + "VarBinChunkedArray: null chunk requires an allocator"); + } + typed.add(allNull(dtype, na.length(), arena)); + } else { + throw new VortexException("VarBinChunkedArray: chunk is not a VarBinArray: " + + data.getClass().getSimpleName()); + } + } + long[] off = new long[typed.size() + 1]; + for (int i = 0; i < typed.size(); i++) { + off[i + 1] = off[i] + typed.get(i).length(); + } + if (off[off.length - 1] != totalRows) { + throw new VortexException("VarBinChunkedArray: chunk rows sum to " + + off[off.length - 1] + ", expected " + totalRows); + } + return new VarBinChunkedArray(dtype, totalRows, typed.toArray(VarBinArray[]::new), off); + } + + /// Builds an all-null [VarBinOffsetArray] of `n` rows: an empty bytes segment and an + /// offsets segment of `n + 1` zeros, so every row is zero-length. Row nullability + /// is carried by the caller's validity bitmap, not the byte data. + /// + /// @param dtype logical element type (Utf8 or Binary) + /// @param n number of all-null rows + /// @param arena allocator for the offsets segment + /// @return a [VarBinOffsetArray] with `n` zero-length rows + private static VarBinOffsetArray allNull(DType dtype, long n, SegmentAllocator arena) { + MemorySegment offsets = arena.allocate((n + 1) * Long.BYTES, Long.BYTES); + return new VarBinOffsetArray(dtype, n, MemorySegment.NULL, offsets, PType.I64); + } + + private int findChunk(long i) { + int hit = java.util.Arrays.binarySearch(offsets, i); + int idx = hit >= 0 ? hit : -hit - 2; + if (idx >= children.length) { + idx = children.length - 1; + } + return idx; + } + + @Override + public MemorySegment bytesSegment() { + return MemorySegment.NULL; + } + + /// No single contiguous segment — chunked data lives across children. + /// + /// @return always empty + @Override + public Optional segmentIfPresent() { + return Optional.empty(); + } + + @Override + public byte[] getBytes(long i) { + int c = findChunk(i); + return children[c].getBytes(i - offsets[c]); + } + + @Override + public String getString(long i) { + int c = findChunk(i); + return children[c].getString(i - offsets[c]); + } + + @Override + public int getByteLength(long i) { + int c = findChunk(i); + return children[c].getByteLength(i - offsets[c]); + } + + @Override + public void forEachByteLength(IntConsumer c) { + for (VarBinArray child : children) { + child.forEachByteLength(c); + } + } + + @Override + public VarBinArray limited(long rows) { + if (rows >= length) { + return this; + } + // Keep full children that fit, recursively limited the boundary child. + var kept = new java.util.ArrayList(children.length); + for (int i = 0; i < children.length; i++) { + long start = offsets[i]; + long end = offsets[i + 1]; + if (start >= rows) { + break; + } + if (end <= rows) { + kept.add(children[i]); + } else { + kept.add(children[i].limited(rows - start)); + } + } + return VarBinChunkedArray.of(dtype, rows, kept); + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinConstantArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinConstantArray.java new file mode 100644 index 000000000..fdb1c9538 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinConstantArray.java @@ -0,0 +1,66 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.Optional; +import java.util.function.IntConsumer; + +/// Metadata-only [VarBinArray] for `vortex.constant` Utf8/Binary columns: a single value +/// broadcast across `length` logical rows. No buffer is allocated and no per-row +/// materialization occurs — every accessor returns the same shared bytes, mirroring the +/// `LazyConstantXxxArray` family the primitive array types use for the same encoding. +/// +/// @param dtype logical element type (Utf8 or Binary) +/// @param length number of logical rows the constant is broadcast across +/// @param bytes the constant value's raw bytes, shared across every row; [#getBytes(long)] +/// clones it per that method's copy contract +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record VarBinConstantArray(DType dtype, long length, byte[] bytes) implements VarBinArray { + + @Override + public MemorySegment bytesSegment() { + return MemorySegment.NULL; + } + + /// No single contiguous segment backs a broadcast constant. + /// + /// @return always empty + @Override + public Optional segmentIfPresent() { + return Optional.empty(); + } + + @Override + public byte[] getBytes(long i) { + Objects.checkIndex(i, length); + return bytes.clone(); + } + + @Override + public String getString(long i) { + Objects.checkIndex(i, length); + return new String(bytes, StandardCharsets.UTF_8); + } + + @Override + public int getByteLength(long i) { + Objects.checkIndex(i, length); + return bytes.length; + } + + @Override + public void forEachByteLength(IntConsumer c) { + int len = bytes.length; + for (long i = 0; i < length; i++) { + c.accept(len); + } + } + + @Override + public VarBinArray limited(long rows) { + return rows >= length ? this : new VarBinConstantArray(dtype, rows, bytes); + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinDictArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinDictArray.java new file mode 100644 index 000000000..7b12a75bb --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinDictArray.java @@ -0,0 +1,239 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; +import java.util.function.IntConsumer; + +/// Dictionary-encoded [VarBinArray]. +/// +/// Stores dictionary values and per-row codes; all accessors resolve through the +/// dictionary without materializing strings at construction time. +/// +/// @param dtype logical type (Utf8 or Binary) +/// @param length number of logical elements (rows) +/// @param bytesSegment concatenated raw bytes for all dictionary values +/// @param dictValOffsets offsets into `bytesSegment` for each dictionary entry (length = dictSize + 1) +/// @param dictValOffPType physical type of the dictionary value offsets +/// @param dictCodesSegs per-row dictionary code indices (length = `length`) +/// @param dictCodesPType physical type of the dictionary codes +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record VarBinDictArray(DType dtype, long length, MemorySegment bytesSegment, + MemorySegment dictValOffsets, PType dictValOffPType, + MemorySegment dictCodesSegs, PType dictCodesPType) + implements VarBinArray { + + @Override + public byte[] getBytes(long i) { + long code = dictReadCode(i); + long start = dictReadOff(code); + long end = dictReadOff(code + 1); + int len = VarBinArrays.checkedLength(bytesSegment, start, end); + byte[] out = new byte[len]; + MemorySegment.copy(bytesSegment, start, MemorySegment.ofArray(out), 0, len); + return out; + } + + @Override + public String getString(long i) { + return new String(getBytes(i), StandardCharsets.UTF_8); + } + + @Override + public int getByteLength(long i) { + long code = dictReadCode(i); + return VarBinArrays.checkedLength(bytesSegment, dictReadOff(code), dictReadOff(code + 1)); + } + + @Override + public void forEachByteLength(IntConsumer c) { + // Hot loop: hoist both ptype dispatches out of the per-row body so C2 sees a + // uniform, fixed-stride loop (CLAUDE.md hot-loop rule). A variable-target + // switch(dictValOffPType)/switch(dictCodesPType) per element blocks C2 + // superword vectorization and adds invariant width/bounds arithmetic on every + // row (regression introduced by #215). I32 dict-value offsets are by far the + // most common (FSST + most dict encodings emit 32-bit offsets), so the fast + // path reads offsets at a constant 4-byte stride and branch-splits the code + // read once; wider offset ptypes take the general per-row path. + // + // The per-row body must stay branch-free, so the untrusted-code bounds check is + // a boundary catch-and-wrap around the whole loop rather than a test per row: + // an out-of-pool code (or a truncated codes buffer) trips the segment access and + // must surface as a VortexException, never a raw IndexOutOfBoundsException. The + // handler re-validates on the cold path so the blame — and the exception type — + // land on whichever input actually failed, including the caller's own consumer. + try { + if (dictValOffPType == PType.I32) { + forEachI32OffsetByteLength(c); + } else { + for (long i = 0; i < length; i++) { + long code = dictReadCode(i); + c.accept((int) (dictReadOff(code + 1) - dictReadOff(code))); + } + } + } catch (IndexOutOfBoundsException e) { + throw attribute(e); + } + } + + /// Cold path: works out which untrusted input made a bulk length walk run off the + /// end, so the message names the segment that actually failed. + /// + /// Re-checks the codes extent, then every code against the value-offsets extent — + /// affordable here because it only runs after something has already thrown. When + /// both check out the failure came from the caller's [IntConsumer], and that + /// exception is returned unchanged rather than relabeled as malformed input. + /// + /// @param e the out-of-bounds failure raised by the walk + /// @return a [VortexException] describing the malformed input, or `e` itself + private RuntimeException attribute(IndexOutOfBoundsException e) { + int codeWidth = dictCodesPType.byteSize(); + if (length > dictCodesSegs.byteSize() / codeWidth) { + return new VortexException("dict codes segment of " + dictCodesSegs.byteSize() + + " bytes holds fewer than " + length + " " + dictCodesPType + " codes"); + } + long offsetCount = dictValOffsets.byteSize() / dictValOffPType.byteSize(); + for (long i = 0; i < length; i++) { + long code = readCodeAt(i); + if (code < 0 || code + 1 >= offsetCount) { + return new VortexException("dict code " + code + " at row " + i + + " out of range for " + offsetCount + " value offsets"); + } + } + return e; + } + + /// Fast path of [#forEachByteLength(IntConsumer)] for I32 dict-value offsets: the + /// code-ptype switch is hoisted out of the loop so each specialized loop body reads + /// codes at a single fixed stride and computes lengths from a constant 4-byte offset + /// stride, leaving the per-row body uniform and vectorizable. + /// + /// @param c consumer called once per row with the byte length at that index + private void forEachI32OffsetByteLength(IntConsumer c) { + long n = length; + switch (dictCodesPType) { + case U8 -> { + for (long i = 0; i < n; i++) { + c.accept(i32OffsetLength( + Byte.toUnsignedLong(dictCodesSegs.get(ValueLayout.JAVA_BYTE, i)))); + } + } + case U16 -> { + for (long i = 0; i < n; i++) { + c.accept(i32OffsetLength( + Short.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_SHORT, i)))); + } + } + case U32 -> { + for (long i = 0; i < n; i++) { + c.accept(i32OffsetLength( + Integer.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i)))); + } + } + case I32 -> { + for (long i = 0; i < n; i++) { + c.accept(i32OffsetLength(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i))); + } + } + case I64, U64 -> { + for (long i = 0; i < n; i++) { + c.accept(i32OffsetLength(dictCodesSegs.getAtIndex(VortexFormat.LE_LONG, i))); + } + } + default -> throw new VortexException("unsupported codes ptype: " + dictCodesPType); + } + } + + /// Byte length of the dictionary entry `code` when the value offsets are I32: + /// `offsets[code + 1] - offsets[code]` read at a constant 4-byte stride. The + /// segment access itself bounds-checks, so an out-of-range code is caught without + /// the per-row width recomputation that [#dictReadOff(long)] performs. + /// + /// @param code zero-based dictionary entry index (in `[0, dictSize)`) + /// @return the byte length of dictionary entry `code` + private int i32OffsetLength(long code) { + return dictValOffsets.getAtIndex(VortexFormat.LE_INT, code + 1) + - dictValOffsets.getAtIndex(VortexFormat.LE_INT, code); + } + + @Override + public VarBinArray limited(long rows) { + if (rows >= length) { + return this; + } + int codeBytes = dictCodesPType.byteSize(); + if (rows > dictCodesSegs.byteSize() / codeBytes) { + throw new VortexException("dict codes segment of " + dictCodesSegs.byteSize() + + " bytes holds fewer than " + rows + " " + dictCodesPType + " codes"); + } + return VarBinArray.ofDict(dtype, rows, bytesSegment, dictValOffsets, dictValOffPType, + dictCodesSegs.asSlice(0, rows * codeBytes), dictCodesPType); + } + + /// Reads the dictionary code for row `i` at the width of [#dictCodesPType]. + /// + /// The codes buffer is untrusted and may be shorter than [#length()], so an + /// overrun is reported as a [VortexException] instead of a raw + /// `IndexOutOfBoundsException` (ADR 0003). + /// + /// @param i zero-based row index + /// @return the dictionary code, widened to a signed long + private long dictReadCode(long i) { + try { + return readCodeAt(i); + } catch (IndexOutOfBoundsException e) { + throw new VortexException("dict code index " + i + " (" + dictCodesPType + + ") out of range for a codes segment of " + + dictCodesSegs.byteSize() + " bytes", e); + } + } + + private long readCodeAt(long i) { + return switch (dictCodesPType) { + case U8 -> Byte.toUnsignedLong(dictCodesSegs.get(ValueLayout.JAVA_BYTE, i)); + case U16 -> Short.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_SHORT, i)); + case U32 -> Integer.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i)); + case I32 -> dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i); + case I64, U64 -> dictCodesSegs.getAtIndex(VortexFormat.LE_LONG, i); + default -> throw new VortexException("unsupported codes ptype: " + dictCodesPType); + }; + } + + /// Reads dictionary-value offset `i` at the true width of [#dictValOffPType]. + /// + /// Offsets can arrive at any integer width — FSST decompresses its values to a + /// child with I32 offsets, legacy dicts use I64, and narrow sequence-encoded + /// offsets keep their U8/U16 ptype on the wire. Reading at the wrong width (e.g. + /// an 8-byte read against a 4-byte-stride buffer) walks off the segment. The + /// buffer bounds and ptype are untrusted input, so both are checked here and a + /// [VortexException] is thrown rather than a bare `IndexOutOfBoundsException` + /// (ADR 0003). + /// + /// @param i zero-based offset index (in `[0, dictSize]`) + /// @return the offset value widened to a signed long + private long dictReadOff(long i) { + int width = dictValOffPType.byteSize(); + long byteOffset = i * width; + if (i < 0 || byteOffset + width > dictValOffsets.byteSize()) { + throw new VortexException("dict value offset index " + i + " (" + dictValOffPType + + ", " + width + "-byte) out of range for offsets segment of " + + dictValOffsets.byteSize() + " bytes"); + } + return switch (dictValOffPType) { + case U8 -> Byte.toUnsignedLong(dictValOffsets.get(ValueLayout.JAVA_BYTE, byteOffset)); + case I8 -> dictValOffsets.get(ValueLayout.JAVA_BYTE, byteOffset); + case U16 -> Short.toUnsignedLong(dictValOffsets.get(VortexFormat.LE_SHORT, byteOffset)); + case I16 -> dictValOffsets.get(VortexFormat.LE_SHORT, byteOffset); + case U32 -> Integer.toUnsignedLong(dictValOffsets.get(VortexFormat.LE_INT, byteOffset)); + case I32 -> dictValOffsets.get(VortexFormat.LE_INT, byteOffset); + case I64, U64 -> dictValOffsets.get(VortexFormat.LE_LONG, byteOffset); + default -> throw new VortexException( + "unsupported dict value offset ptype: " + dictValOffPType); + }; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java new file mode 100644 index 000000000..6292b65b0 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java @@ -0,0 +1,120 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; + +import java.lang.foreign.MemorySegment; +import java.nio.charset.StandardCharsets; +import java.util.function.IntConsumer; + +/// Standard offset-based [VarBinArray]. +/// +/// Element `i` occupies `bytesSegment[offsetsSegment[i]..offsetsSegment[i+1]]`. +/// +/// @param dtype logical type (Utf8 or Binary) +/// @param length number of variable-length elements +/// @param bytesSegment concatenated raw byte data for all elements +/// @param offsetsSegment offsets segment of length `length + 1` +/// @param offsetsPtype physical type of the offsets values (I32/U32 or I64/U64) +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record VarBinOffsetArray(DType dtype, long length, MemorySegment bytesSegment, + MemorySegment offsetsSegment, PType offsetsPtype) + implements VarBinArray { + + @Override + public byte[] getBytes(long i) { + long start = readOffset(i); + long end = readOffset(i + 1); + int len = VarBinArrays.checkedLength(bytesSegment, start, end); + byte[] out = new byte[len]; + MemorySegment.copy(bytesSegment, start, MemorySegment.ofArray(out), 0, len); + return out; + } + + @Override + public String getString(long i) { + return new String(getBytes(i), StandardCharsets.UTF_8); + } + + @Override + public int getByteLength(long i) { + return VarBinArrays.checkedLength(bytesSegment, readOffset(i), readOffset(i + 1)); + } + + @Override + public void forEachByteLength(IntConsumer c) { + long n = length; + // The loop reads offsets[0..n] at a fixed stride and must stay uniform to + // vectorize (CLAUDE.md hot-loop rule), so the untrusted offsets segment is + // sized once here rather than bounds-checked per row. A non-monotonic pair + // still yields a negative length to the consumer; the typed accessors + // ([#getBytes(long)], [#getByteLength(long)]) reject it when the row is read. + checkOffsetsExtent(n); + if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { + for (long i = 0; i < n; i++) { + c.accept(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i + 1) + - offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); + } + } else { + for (long i = 0; i < n; i++) { + c.accept((int) (offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i + 1) + - offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i))); + } + } + } + + @Override + public VarBinArray limited(long rows) { + if (rows >= length) { + return this; + } + checkOffsetsExtent(rows); + long byteEnd = readOffset(rows); + if (byteEnd < 0 || byteEnd > bytesSegment.byteSize()) { + throw new VortexException("varbin offset " + byteEnd + " at row " + rows + + " out of range for a data buffer of " + bytesSegment.byteSize() + " bytes"); + } + int offBytes = offsetWidth(); + MemorySegment newOffsetsSeg = offsetsSegment.asSlice(0, (rows + 1) * offBytes); + return new VarBinOffsetArray(dtype, rows, + bytesSegment.asSlice(0, byteEnd), newOffsetsSeg, offsetsPtype); + } + + /// Verifies the offsets segment holds the `rows + 1` offsets the array claims. + /// + /// @param rows number of rows whose offsets are about to be read + private void checkOffsetsExtent(long rows) { + int width = offsetWidth(); + if (rows + 1 > offsetsSegment.byteSize() / width) { + throw new VortexException("varbin offsets segment of " + offsetsSegment.byteSize() + + " bytes holds fewer than " + (rows + 1) + " " + offsetsPtype + " offsets"); + } + } + + private int offsetWidth() { + return (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) ? Integer.BYTES : Long.BYTES; + } + + /// Reads offset `i` at the width of [#offsetsPtype]. + /// + /// The offsets segment comes straight from an untrusted file, so an index past its + /// end must surface as a [VortexException] rather than a raw + /// `IndexOutOfBoundsException` (ADR 0003). + /// + /// @param i zero-based offset index, in `[0, length]` + /// @return the offset value widened to a signed long + private long readOffset(long i) { + try { + if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { + return offsetsSegment.getAtIndex(VortexFormat.LE_INT, i); + } + return offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i); + } catch (IndexOutOfBoundsException e) { + throw new VortexException("varbin offset index " + i + " (" + offsetsPtype + + ") out of range for an offsets segment of " + + offsetsSegment.byteSize() + " bytes", e); + } + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinSlicedArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinSlicedArray.java new file mode 100644 index 000000000..f99e9c546 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinSlicedArray.java @@ -0,0 +1,65 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.util.Optional; +import java.util.function.IntConsumer; + +/// Sliced view over a [VarBinArray]: every accessor delegates to `inner` +/// with the row index shifted by `offset`. Used by the scan iterator to +/// surface a column that was decoded once (because it shares a single +/// flat layout across multiple aligned chunks) as a per-chunk slice +/// without copying. +/// +/// @param dtype logical element type (typically [DType.Utf8] or [DType.Binary]) +/// @param length number of logical elements in this slice +/// @param inner underlying VarBin array +/// @param offset starting row index into `inner` +public record VarBinSlicedArray(DType dtype, long length, VarBinArray inner, long offset) + implements VarBinArray { + + @Override + public MemorySegment bytesSegment() { + return inner.bytesSegment(); + } + + /// Delegates the probe to the wrapped array — empty if the inner is + /// itself composite (chunked / view). + /// + /// @return the inner array's segment if segment-backed, otherwise empty + @Override + public Optional segmentIfPresent() { + return inner.segmentIfPresent(); + } + + @Override + public byte[] getBytes(long i) { + return inner.getBytes(i + offset); + } + + @Override + public String getString(long i) { + return inner.getString(i + offset); + } + + @Override + public int getByteLength(long i) { + return inner.getByteLength(i + offset); + } + + @Override + public void forEachByteLength(IntConsumer c) { + for (long i = 0; i < length; i++) { + c.accept(getByteLength(i)); + } + } + + @Override + public VarBinArray limited(long rows) { + if (rows >= length) { + return this; + } + return new VarBinSlicedArray(dtype, rows, inner, offset); + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinViewArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinViewArray.java new file mode 100644 index 000000000..ddced0c7f --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinViewArray.java @@ -0,0 +1,140 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; + +import java.lang.foreign.MemorySegment; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.function.IntConsumer; + +/// Arrow StringView / BinaryView [VarBinArray]. +/// +/// Each row is a 16-byte view in `views`: bytes 0-3 are the u32 size; for +/// sizes ≤ 12 bytes the data is inlined in bytes 4..15; for sizes > 12 bytes +/// bytes 4-7 hold a 4-byte prefix (ignored on read), bytes 8-11 the u32 buffer +/// index into `dataBufs`, and bytes 12-15 the u32 offset within that +/// buffer. Per-row accessors resolve the view on demand — no concat or +/// materialization at construction time. +/// +/// [#bytesSegment()] returns [MemorySegment#NULL] because there is +/// no single contiguous bytes segment; callers needing one must materialize via +/// the typed accessors. +/// +/// @param dtype logical element type (Utf8 or Binary) +/// @param length total logical row count +/// @param views 16-byte view per row; length must be ≥ `length * 16` +/// @param dataBufs zero or more shared data buffers referenced by long views +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable refs that flow through pipelines without ever being compared. +public record VarBinViewArray(DType dtype, long length, MemorySegment views, MemorySegment[] dataBufs) + implements VarBinArray { + + private static final int VIEW_SIZE = 16; + private static final int MAX_INLINED_SIZE = 12; + + @Override + public MemorySegment bytesSegment() { + return MemorySegment.NULL; + } + + /// No single contiguous segment — view rows reference shared data buffers. + /// + /// @return always empty + @Override + public Optional segmentIfPresent() { + return Optional.empty(); + } + + @Override + public int getByteLength(long i) { + return checkedSize(views.get(VortexFormat.LE_INT, viewOffset(i))); + } + + @Override + public byte[] getBytes(long i) { + long viewOff = viewOffset(i); + int size = checkedSize(views.get(VortexFormat.LE_INT, viewOff)); + byte[] out = new byte[size]; + if (size <= MAX_INLINED_SIZE) { + // Inlined data always fits the remaining 12 bytes of the view itself. + MemorySegment.copy(views, viewOff + 4, MemorySegment.ofArray(out), 0, size); + } else { + int bufferIndex = views.get(VortexFormat.LE_INT, viewOff + 8); + long srcOffset = Integer.toUnsignedLong(views.get(VortexFormat.LE_INT, viewOff + 12)); + if (bufferIndex < 0 || bufferIndex >= dataBufs.length) { + throw new VortexException("varbin view at row " + i + " references data buffer " + + bufferIndex + " of " + dataBufs.length); + } + MemorySegment buf = dataBufs[bufferIndex]; + if (srcOffset + size > buf.byteSize()) { + throw new VortexException("varbin view bytes [" + srcOffset + ", " + + (srcOffset + size) + ") out of range for data buffer " + bufferIndex + + " of " + buf.byteSize() + " bytes"); + } + MemorySegment.copy(buf, srcOffset, MemorySegment.ofArray(out), 0, size); + } + return out; + } + + @Override + public String getString(long i) { + return new String(getBytes(i), StandardCharsets.UTF_8); + } + + @Override + public void forEachByteLength(IntConsumer c) { + long n = length; + // Sized once, outside the loop, so the per-row body stays uniform — same + // trade-off as VarBinOffsetArray: a negative size on the wire still reaches the + // consumer, and [#getBytes(long)] rejects it when the row is read. + checkViewsExtent(n); + for (long i = 0; i < n; i++) { + c.accept(views.get(VortexFormat.LE_INT, i * VIEW_SIZE)); + } + } + + @Override + public VarBinArray limited(long rows) { + if (rows >= length) { + return this; + } + checkViewsExtent(rows); + return new VarBinViewArray(dtype, rows, views.asSlice(0, rows * VIEW_SIZE), dataBufs); + } + + /// Byte offset of view `i`, rejecting a row the views segment does not cover. + /// + /// @param i zero-based row index + /// @return the byte offset of the 16-byte view for row `i` + private long viewOffset(long i) { + long off = i * VIEW_SIZE; + if (i < 0 || off + VIEW_SIZE > views.byteSize()) { + throw new VortexException("varbin view index " + i + + " out of range for a views segment of " + views.byteSize() + " bytes"); + } + return off; + } + + /// Verifies the views segment holds `rows` complete 16-byte views. + /// + /// @param rows number of rows whose views are about to be read + private void checkViewsExtent(long rows) { + if (rows > views.byteSize() / VIEW_SIZE) { + throw new VortexException("varbin views segment of " + views.byteSize() + + " bytes holds fewer than " + rows + " views"); + } + } + + /// Rejects a negative element size read from a view header, which would otherwise + /// reach `new byte[size]` as a `NegativeArraySizeException` (ADR 0003). + /// + /// @param size element size read from the view + /// @return `size` when it is non-negative + private static int checkedSize(int size) { + if (size < 0) { + throw new VortexException("negative varbin view size " + size); + } + return size; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java index a08569771..2cf4fdf40 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java @@ -16,7 +16,7 @@ import io.github.dfa1.vortex.reader.array.LazyConstantShortArray; import io.github.dfa1.vortex.reader.array.LazyDecimalArray; import io.github.dfa1.vortex.reader.array.NullArray; -import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinConstantArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -132,7 +132,7 @@ private static Array decodeBool(DType dtype, ProtoScalarValue scalar, long n) { return new LazyConstantBoolArray(dtype, n, value); } - /// Builds a metadata-only constant Utf8/Binary array — [VarBinArray.ConstantMode] returns + /// Builds a metadata-only constant Utf8/Binary array — [VarBinConstantArray] returns /// the same shared bytes for every row, so this stays O(1) regardless of `n` like every /// other constant type, instead of eagerly writing `n` copies into a real buffer. /// @@ -144,7 +144,7 @@ private static Array decodeString(ProtoScalarValue scalar, DType dtype, long n) byte[] strBytes = scalar.string_value() != null ? scalar.string_value().getBytes(StandardCharsets.UTF_8) : (scalar.bytes_value() != null ? scalar.bytes_value() : new byte[0]); - return new VarBinArray.ConstantMode(dtype, n, strBytes); + return new VarBinConstantArray(dtype, n, strBytes); } private static long scalarToRawBits(ProtoScalarValue scalar) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 1209fc201..1d10b2793 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -17,6 +17,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedLongArray; import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -276,13 +277,13 @@ private static Array decodeUtf8DictProto(DecodeContext ctx, MemorySegment metaBu poolValidity = masked.validity(); } VarBinArray valuesArr = (VarBinArray) valuesDecoded; - VarBinArray.OffsetMode dictValues = VarBinArray.toOffsetMode(valuesArr, ctx.arena()); + VarBinOffsetArray dictValues = VarBinArray.toOffsetMode(valuesArr, ctx.arena()); BoolArray rowValidity = rowValidity(ctx, codesBuf, codePType, codesValidity, poolValidity, n); // Carry the offsets ptype that `dictValues` actually materialized. `toOffsetMode` // only builds fresh I64 offsets on its slow path; on the fast path it returns the // decoded values array unchanged, keeping its own ptype (e.g. FSST decompresses to - // I32 offsets). Hardcoding I64 here made the DictMode carrier disagree with its + // I32 offsets). Hardcoding I64 here made the VarBinDictArray carrier disagree with its // buffer width (4-byte stride read as 8) and threw IOOBE — see #215. Array dict = VarBinArray.ofDict(ctx.dtype(), n, dictValues.bytesSegment(), dictValues.offsetsSegment(), dictValues.offsetsPtype(), diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java index 984a88966..a61c12540 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.reader.array.Array; -import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; @@ -168,7 +168,7 @@ public Array decode(DecodeContext ctx) { } } - return new VarBinArray.OffsetMode(ctx.dtype(), n, + return new VarBinOffsetArray(ctx.dtype(), n, outBytes.asSlice(0, totalUncompressed).asReadOnly(), outOffsets.asReadOnly(), PType.I32); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index 917b20fb3..5f49530b8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -19,6 +19,7 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -168,7 +169,7 @@ private static void validateEnds(MemorySegment endsSeg, PType endsPtype, long nu } private static Array expandStrings( - MemorySegment endsSeg, VarBinArray.OffsetMode valuesArr, + MemorySegment endsSeg, VarBinOffsetArray valuesArr, PType endsPtype, long numRuns, long offset, long n, DType dtype, SegmentAllocator arena ) { @@ -216,7 +217,7 @@ private static Array expandStrings( logicalPos = runEnd; } - return new VarBinArray.OffsetMode(dtype, n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); + return new VarBinOffsetArray(dtype, n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); } private static long readUnsigned(MemorySegment seg, long i, PType ptype) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java index 20a90b1a9..741f058ff 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java @@ -26,6 +26,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -269,7 +270,7 @@ private static Array decodeVarBin( MemorySegment outOffsets = ctx.arena().allocate((n + 1) * 4L, 4); if (numPatches == 0) { MemorySegment outBytes = ctx.arena().allocate(1); - Array result = new VarBinArray.OffsetMode(ctx.dtype(), n, outBytes, outOffsets, PType.I32); + Array result = new VarBinOffsetArray(ctx.dtype(), n, outBytes, outOffsets, PType.I32); return withSparseValidity(ctx, result, fillValid, null, idxData, 0, n, offset); } @@ -282,7 +283,7 @@ private static Array decodeVarBin( valData = m.inner(); patchValidity = m.validity(); } - VarBinArray.OffsetMode varBin = VarBinArray.toOffsetMode( + VarBinOffsetArray varBin = VarBinArray.toOffsetMode( checkedCast(valData, VarBinArray.class, "values"), ctx.arena()); MemorySegment valBytes = varBin.bytesSegment(); MemorySegment valOffsets = varBin.offsetsSegment(); @@ -337,7 +338,7 @@ private static Array decodeVarBin( + valOffsets.byteSize() + "-byte offsets buffer", e); } - Array result = new VarBinArray.OffsetMode(ctx.dtype(), n, outBytes, outOffsets, PType.I32); + Array result = new VarBinOffsetArray(ctx.dtype(), n, outBytes, outOffsets, PType.I32); return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java index 5f6944815..192087076 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.reader.array.Array; -import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.proto.ProtoVarBinMetadata; @@ -48,6 +48,6 @@ public Array decode(DecodeContext ctx) { MemorySegment bytes = ctx.buffer(0); - return new VarBinArray.OffsetMode(ctx.dtype(), n, bytes, offsets, offsetsPtype); + return new VarBinOffsetArray(ctx.dtype(), n, bytes, offsets, offsetsPtype); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoder.java index 301bb91bc..8a7c6569f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.reader.array.Array; -import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinViewArray; import io.github.dfa1.vortex.core.model.EncodingId; import java.lang.foreign.MemorySegment; @@ -30,13 +30,13 @@ public Array decode(DecodeContext ctx) { } // Lazy path: keep views + data buffers as MemorySegment slices; per-row - // accessors resolve on demand via VarBinArray.ViewMode. No copy, no concat, + // accessors resolve on demand via VarBinViewArray. No copy, no concat, // no flat byte buffer allocation. MemorySegment viewsBuf = ctx.buffer(numBufs - 1); MemorySegment[] dataBufs = new MemorySegment[numBufs - 1]; for (int i = 0; i < dataBufs.length; i++) { dataBufs[i] = ctx.buffer(i); } - return new VarBinArray.ViewMode(ctx.dtype(), ctx.rowCount(), viewsBuf, dataBufs); + return new VarBinViewArray(ctx.dtype(), ctx.rowCount(), viewsBuf, dataBufs); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java index 1373722d2..c899f076f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java @@ -18,6 +18,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedLongArray; import io.github.dfa1.vortex.reader.array.MaterializedShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import io.github.dfa1.zstd.ZstdDecompressContext; import io.github.dfa1.zstd.ZstdDecompressDictionary; @@ -179,7 +180,7 @@ private static VarBinArray buildScatteredVarBin( offsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) dataPos); } - return new VarBinArray.OffsetMode(dtype.withNullable(false), rowCount, values, offsets, PType.I32); + return new VarBinOffsetArray(dtype.withNullable(false), rowCount, values, offsets, PType.I32); } private static MemorySegment decompressFrames( @@ -324,6 +325,6 @@ private static VarBinArray buildVarBin(DType dtype, long n, MemorySegment decomp offsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) dataPos); } - return new VarBinArray.OffsetMode(dtype, n, values, offsets, PType.I32); + return new VarBinOffsetArray(dtype, n, values, offsets, PType.I32); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index 079a2699f..6fe795bfd 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -26,6 +26,7 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -64,7 +65,7 @@ public Array decode(LayoutDecodeContext ctx, Layout dictLayout, DType dtype) { Array codes = ctx.decodeChild(codesLayout, new DType.Primitive(codesPType, false)); // VarBin (string) dict: VarBinArray is a sealed interface; ofDict returns the - // lazy DictMode record (no eager expansion into per-row offsets/bytes). + // lazy VarBinDictArray record (no eager expansion into per-row offsets/bytes). // Unwrap a masked (nullable) codes/values child so the string expansion sees the raw // payload; the row-level validity is re-applied by wrapping the result below. This mirrors // the primitive path (buildLazyDictPrimitive) and is the shape a nullable global-dict Utf8 @@ -73,7 +74,7 @@ public Array decode(LayoutDecodeContext ctx, Layout dictLayout, DType dtype) { Array valuesData = values instanceof MaskedArray mv ? mv.inner() : values; BoolArray codesValidity = codes instanceof MaskedArray mc ? mc.validity() : null; Array codesData = codes instanceof MaskedArray mc ? mc.inner() : codes; - if (valuesData instanceof VarBinArray.OffsetMode vb) { + if (valuesData instanceof VarBinOffsetArray vb) { // Zip-bomb guard: read the codes as a segment so we can validate the buffer // before allocating the expansion output. For direct-mapped encodings (e.g. // vortex.primitive), the codes buffer is mmap-bounded and can be much smaller @@ -237,7 +238,7 @@ private static PType readDictLayoutCodesPType(MemorySegment rawMeta) { } private static Array expandDictStrings( - VarBinArray.OffsetMode values, MemorySegment codesSegs, + VarBinOffsetArray values, MemorySegment codesSegs, PType codesPType, DType dtype, long n, SegmentAllocator arena ) { @@ -271,7 +272,7 @@ private static Array expandDictStrings( outOffsets.setAtIndex(LE_INT, i + 1, (int) bytePos); } - return new VarBinArray.OffsetMode(dtype, n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); + return new VarBinOffsetArray(dtype, n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); } private static long readUnsigned(MemorySegment seg, long idx, PType ptype) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java index 561d44eed..5d5b37f42 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java @@ -23,7 +23,7 @@ class VarBinArrayTest { private static final DType UTF8 = DType.UTF8; - private static VarBinArray.OffsetMode of(String... values) { + private static VarBinOffsetArray of(String... values) { byte[] allBytes = String.join("", values).getBytes(StandardCharsets.UTF_8); MemorySegment bytes = MemorySegment.ofArray(allBytes); @@ -36,7 +36,7 @@ private static VarBinArray.OffsetMode of(String... values) { bb.putInt(o); } MemorySegment offsetsSeg = MemorySegment.ofArray(bb.array()); - return new VarBinArray.OffsetMode(UTF8, values.length, bytes, offsetsSeg, PType.I32); + return new VarBinOffsetArray(UTF8, values.length, bytes, offsetsSeg, PType.I32); } @Nested @@ -117,7 +117,7 @@ void empty_zeroElements() { @Test void offsetsPtype_returnsOffsetType() { // Given - VarBinArray.OffsetMode sut = of("a"); + VarBinOffsetArray sut = of("a"); // When / Then assertThat(sut.offsetsPtype()).isNotNull(); @@ -330,7 +330,7 @@ class Constant { @Test void getString_returnsSameValueForEveryRow() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 5, "hi".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 5, "hi".getBytes(StandardCharsets.UTF_8)); // When / Then assertThat(sut.getString(0)).isEqualTo("hi"); @@ -340,7 +340,7 @@ void getString_returnsSameValueForEveryRow() { @Test void getByteLength_returnsConstantLength() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 3, "abc".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 3, "abc".getBytes(StandardCharsets.UTF_8)); // When / Then assertThat(sut.getByteLength(0)).isEqualTo(3); @@ -350,7 +350,7 @@ void getByteLength_returnsConstantLength() { @Test void forEachByteLength_visitsLengthOncePerRow() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 4, "xy".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 4, "xy".getBytes(StandardCharsets.UTF_8)); List lengths = new ArrayList<>(); // When @@ -365,7 +365,7 @@ void forEachByteLength_visitsLengthOncePerRow() { @Test void getBytes_returnsIndependentCopyEachCall() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 2, "z".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 2, "z".getBytes(StandardCharsets.UTF_8)); // When byte[] first = sut.getBytes(0); @@ -378,7 +378,7 @@ void getBytes_returnsIndependentCopyEachCall() { @Test void getString_outOfBoundsIndex_throws() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 2, "v".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 2, "v".getBytes(StandardCharsets.UTF_8)); // When / Then assertThatThrownBy(() -> sut.getString(2)).isInstanceOf(IndexOutOfBoundsException.class); @@ -387,7 +387,7 @@ void getString_outOfBoundsIndex_throws() { @Test void limited_returnsShorterConstant() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 10, "k".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 10, "k".getBytes(StandardCharsets.UTF_8)); // When VarBinArray result = sut.limited(3); @@ -400,7 +400,7 @@ void limited_returnsShorterConstant() { @Test void limited_rowsAtOrAboveLength_returnsSameInstance() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 5, "m".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 5, "m".getBytes(StandardCharsets.UTF_8)); // When VarBinArray result = sut.limited(5); @@ -412,23 +412,23 @@ void limited_rowsAtOrAboveLength_returnsSameInstance() { @Test void bytesSegment_isNull_noContiguousBufferBacksABroadcast() { // Given - VarBinArray.ConstantMode sut = new VarBinArray.ConstantMode(UTF8, 2, "n".getBytes(StandardCharsets.UTF_8)); + VarBinConstantArray sut = new VarBinConstantArray(UTF8, 2, "n".getBytes(StandardCharsets.UTF_8)); // When / Then assertThat(sut.bytesSegment()).isSameAs(MemorySegment.NULL); assertThat(sut.segmentIfPresent()).isEmpty(); } - /// The general "any VarBinArray -> OffsetMode" path other decoders (RunEnd's string + /// The general "any VarBinArray -> VarBinOffsetArray" path other decoders (RunEnd's string /// expansion, dict/sparse child normalization) rely on must still work for a broadcast /// constant, walking it via the typed accessors rather than `bytesSegment()`. @Test void toOffsetMode_materializesBroadcastIntoRealOffsets() { // Given - VarBinArray sut = new VarBinArray.ConstantMode(UTF8, 3, "hey".getBytes(StandardCharsets.UTF_8)); + VarBinArray sut = new VarBinConstantArray(UTF8, 3, "hey".getBytes(StandardCharsets.UTF_8)); // When - VarBinArray.OffsetMode result = VarBinArray.toOffsetMode(sut, Arena.ofAuto()); + VarBinOffsetArray result = VarBinArray.toOffsetMode(sut, Arena.ofAuto()); // Then assertThat(result.length()).isEqualTo(3); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArrayTest.java similarity index 83% rename from reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java rename to reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArrayTest.java index 31c00d6ae..c0b8ed071 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedArrayTest.java @@ -14,7 +14,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -class VarBinChunkedModeTest { +class VarBinChunkedArrayTest { private static final DType UTF8 = DType.UTF8; @@ -24,7 +24,7 @@ class Construction { @Test void emptyChunkListRejected() { // Given / When / Then - assertThatThrownBy(() -> VarBinArray.ChunkedMode.of(UTF8, 0, List.of())) + assertThatThrownBy(() -> VarBinChunkedArray.of(UTF8, 0, List.of())) .isInstanceOf(VortexException.class); } @@ -35,7 +35,7 @@ void rowMismatchRejected() { VarBinArray c0 = stringChunk(arena, "a", "b"); // When / Then - assertThatThrownBy(() -> VarBinArray.ChunkedMode.of(UTF8, 99, List.of(c0))) + assertThatThrownBy(() -> VarBinChunkedArray.of(UTF8, 99, List.of(c0))) .isInstanceOf(VortexException.class); } } @@ -50,7 +50,7 @@ void nonVarBinChunkRejected() { DType.I64, 1, seg.asReadOnly()); // When / Then - assertThatThrownBy(() -> VarBinArray.ChunkedMode.of(UTF8, 1, List.of(notVarBin))) + assertThatThrownBy(() -> VarBinChunkedArray.of(UTF8, 1, List.of(notVarBin))) .isInstanceOf(VortexException.class); } } @@ -60,14 +60,14 @@ void nullChunkMaterializesAsAllNullRun() { try (Arena arena = Arena.ofConfined()) { // Given — a chunked Utf8 column where the middle chunk is entirely null and // decoded to NullArray (via vortex.null or a null-scalar vortex.constant, #269) - // rather than a VarBinArray. Before the fix ChunkedMode.of threw on it. + // rather than a VarBinArray. Before the fix VarBinChunkedArray.of threw on it. VarBinArray c0 = stringChunk(arena, "a", "b"); NullArray nullChunk = new NullArray(UTF8, 3); VarBinArray c2 = stringChunk(arena, "z"); // When - VarBinArray.ChunkedMode result = - VarBinArray.ChunkedMode.of(UTF8, 6, List.of(c0, nullChunk, c2), arena); + VarBinChunkedArray result = + VarBinChunkedArray.of(UTF8, 6, List.of(c0, nullChunk, c2), arena); // Then — the null chunk contributes 3 zero-length rows, keeping row alignment assertThat(result.length()).isEqualTo(6); @@ -92,7 +92,7 @@ void nullChunkWithoutAllocatorRejected() { // When / Then assertThatThrownBy( - () -> VarBinArray.ChunkedMode.of(UTF8, 2, List.of(nullChunk), null)) + () -> VarBinChunkedArray.of(UTF8, 2, List.of(nullChunk), null)) .isInstanceOf(VortexException.class); } @@ -102,11 +102,11 @@ void nestedChunkedFlattens() { // Given VarBinArray leaf0 = stringChunk(arena, "a"); VarBinArray leaf1 = stringChunk(arena, "b"); - VarBinArray.ChunkedMode nested = VarBinArray.ChunkedMode.of(UTF8, 2, List.of(leaf0, leaf1)); + VarBinChunkedArray nested = VarBinChunkedArray.of(UTF8, 2, List.of(leaf0, leaf1)); VarBinArray leaf2 = stringChunk(arena, "c"); // When - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 3, List.of(nested, leaf2)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 3, List.of(nested, leaf2)); // Then assertThat(sut.children()).hasSize(3); @@ -124,7 +124,7 @@ void getStringDispatchesAcrossChunks() { // Given VarBinArray c0 = stringChunk(arena, "alpha", "beta"); VarBinArray c1 = stringChunk(arena, "gamma"); - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 3, List.of(c0, c1)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 3, List.of(c0, c1)); // When / Then assertThat(sut.getString(0)).isEqualTo("alpha"); @@ -139,7 +139,7 @@ void getBytesDispatchesAcrossChunks() { // Given VarBinArray c0 = stringChunk(arena, "abc"); VarBinArray c1 = stringChunk(arena, "xyz"); - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 2, List.of(c0, c1)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 2, List.of(c0, c1)); // When / Then assertThat(sut.getBytes(1)).containsExactly('x', 'y', 'z'); @@ -152,7 +152,7 @@ void getByteLengthCrossesBoundary() { // Given VarBinArray c0 = stringChunk(arena, "hi"); VarBinArray c1 = stringChunk(arena, "hello"); - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 2, List.of(c0, c1)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 2, List.of(c0, c1)); // When / Then assertThat(sut.getByteLength(0)).isEqualTo(2); @@ -170,7 +170,7 @@ void keepsPrefix() { // Given VarBinArray c0 = stringChunk(arena, "a", "b", "c"); VarBinArray c1 = stringChunk(arena, "d", "e"); - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 5, List.of(c0, c1)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 5, List.of(c0, c1)); // When VarBinArray result = sut.limited(4); @@ -192,7 +192,7 @@ void chunkedHasNoContiguousSegment() { // Given a chunked array — bytes are spread across child segments VarBinArray c0 = stringChunk(arena, "a", "b"); VarBinArray c1 = stringChunk(arena, "c"); - VarBinArray.ChunkedMode sut = VarBinArray.ChunkedMode.of(UTF8, 3, List.of(c0, c1)); + VarBinChunkedArray sut = VarBinChunkedArray.of(UTF8, 3, List.of(c0, c1)); // When / Then the probe must not surface the NULL bytesSegment() sentinel assertThat(sut.segmentIfPresent()).isEmpty(); @@ -205,8 +205,8 @@ void slicedDelegatesToInnerProbe() { // Given a slice over a chunked inner — still no single segment VarBinArray c0 = stringChunk(arena, "a", "b"); VarBinArray c1 = stringChunk(arena, "c"); - VarBinArray.ChunkedMode chunked = VarBinArray.ChunkedMode.of(UTF8, 3, List.of(c0, c1)); - VarBinArray.SlicedMode sut = new VarBinArray.SlicedMode(UTF8, 2, chunked, 1); + VarBinChunkedArray chunked = VarBinChunkedArray.of(UTF8, 3, List.of(c0, c1)); + VarBinSlicedArray sut = new VarBinSlicedArray(UTF8, 2, chunked, 1); // When / Then the probe follows the inner array, not the NULL bytesSegment() assertThat(sut.segmentIfPresent()).isEmpty(); @@ -240,7 +240,7 @@ private static VarBinArray stringChunk(Arena arena, String... values) { pos += b.length; offsets.setAtIndex(ValueLayout.JAVA_INT, i + 1, pos); } - return new VarBinArray.OffsetMode(UTF8, values.length, bytes.asReadOnly(), + return new VarBinOffsetArray(UTF8, values.length, bytes.asReadOnly(), offsets.asReadOnly(), PType.I32); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewArrayTest.java similarity index 89% rename from reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java rename to reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewArrayTest.java index 1754e2827..8394518c8 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewArrayTest.java @@ -13,9 +13,9 @@ import static org.assertj.core.api.Assertions.assertThat; -/// Unit tests for [VarBinArray.ViewMode]. Covers inline (≤ 12 byte) views, +/// Unit tests for [VarBinViewArray]. Covers inline (≤ 12 byte) views, /// referenced views into shared data buffers, length-only reads, and truncate. -class VarBinViewModeTest { +class VarBinViewArrayTest { private static final DType UTF8 = DType.UTF8; private static final int VIEW_SIZE = 16; @@ -28,7 +28,7 @@ void getStringFromInlineView() { try (Arena arena = Arena.ofConfined()) { // Given two short strings stored inline in 16-byte views. MemorySegment views = writeViews(arena, new String[]{"hi", "world!"}); - var sut = new VarBinArray.ViewMode(UTF8, 2, views, new MemorySegment[0]); + var sut = new VarBinViewArray(UTF8, 2, views, new MemorySegment[0]); // When/Then assertThat(sut.getString(0)).isEqualTo("hi"); @@ -42,7 +42,7 @@ void getStringFromInlineView() { void forEachByteLengthEmitsAllRows() { try (Arena arena = Arena.ofConfined()) { MemorySegment views = writeViews(arena, new String[]{"a", "bb", "ccc"}); - var sut = new VarBinArray.ViewMode(UTF8, 3, views, new MemorySegment[0]); + var sut = new VarBinViewArray(UTF8, 3, views, new MemorySegment[0]); var seen = new ArrayList(); sut.forEachByteLength(seen::add); @@ -71,7 +71,7 @@ void getStringFromReferencedDataBuffer() { views.set(VortexFormat.LE_INT, 4, 0); // prefix (ignored on read) views.set(VortexFormat.LE_INT, 8, 0); // buffer index views.set(VortexFormat.LE_INT, 12, 0); // offset within buffer - var sut = new VarBinArray.ViewMode(UTF8, 1, views, new MemorySegment[]{dataBuf}); + var sut = new VarBinViewArray(UTF8, 1, views, new MemorySegment[]{dataBuf}); // When/Then assertThat(sut.getString(0)).isEqualTo(longStr); @@ -95,7 +95,7 @@ void mixedInlineAndReferenced() { views.set(VortexFormat.LE_INT, VIEW_SIZE + 8, 0); views.set(VortexFormat.LE_INT, VIEW_SIZE + 12, 0); - var sut = new VarBinArray.ViewMode(UTF8, 2, views, new MemorySegment[]{dataBuf}); + var sut = new VarBinViewArray(UTF8, 2, views, new MemorySegment[]{dataBuf}); assertThat(sut.getString(0)).isEqualTo("short"); assertThat(sut.getString(1)).isEqualTo(longStr); @@ -110,7 +110,7 @@ class Truncate { void keepsPrefixSharingDataBuffers() { try (Arena arena = Arena.ofConfined()) { MemorySegment views = writeViews(arena, new String[]{"a", "b", "c", "d"}); - var sut = new VarBinArray.ViewMode(UTF8, 4, views, new MemorySegment[0]); + var sut = new VarBinViewArray(UTF8, 4, views, new MemorySegment[0]); VarBinArray truncated = sut.limited(2); @@ -124,7 +124,7 @@ void keepsPrefixSharingDataBuffers() { void truncateBeyondLengthReturnsSelf() { try (Arena arena = Arena.ofConfined()) { MemorySegment views = writeViews(arena, new String[]{"a"}); - var sut = new VarBinArray.ViewMode(UTF8, 1, views, new MemorySegment[0]); + var sut = new VarBinViewArray(UTF8, 1, views, new MemorySegment[0]); assertThat(sut.limited(10)).isSameAs(sut); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java index a9c144fa7..a8b8982bc 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java @@ -13,6 +13,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; import io.github.dfa1.vortex.reader.array.MaterializedLongArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -115,7 +116,7 @@ static MaterializedBoolArray boolArray(Arena arena, boolean... values) { } /// Builds an offset-based Utf8 [VarBinArray] from the given strings, off-heap so it looks like - /// the zero-copy slices the reader hands the kernels. Exercises the [VarBinArray.OffsetMode] + /// the zero-copy slices the reader hands the kernels. Exercises the [VarBinOffsetArray] /// accessor path the string compute kernels fold over. /// /// @param arena the allocator for the off-heap bytes and offsets buffers @@ -132,7 +133,7 @@ static VarBinArray utf8Array(Arena arena, String... values) { running += values[i].getBytes(StandardCharsets.UTF_8).length; offsets.setAtIndex(VortexFormat.LE_INT, i + 1, running); } - return new VarBinArray.OffsetMode(DType.UTF8, values.length, bytes.asReadOnly(), + return new VarBinOffsetArray(DType.UTF8, values.length, bytes.asReadOnly(), offsets, PType.I32); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java index 29df26316..efb98e144 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoderTest.java @@ -8,6 +8,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinConstantArray; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -80,8 +81,8 @@ void stringScalar_decodesToLazyConstantVarBin() { // When Array result = decode(scalar, DType.UTF8, 3); - // Then — VarBinArray.ConstantMode, not an eagerly materialized OffsetMode - assertThat(result).isInstanceOf(VarBinArray.ConstantMode.class); + // Then — VarBinConstantArray, not an eagerly materialized VarBinOffsetArray + assertThat(result).isInstanceOf(VarBinConstantArray.class); VarBinArray strings = (VarBinArray) result; assertThat(strings.getString(0)).isEqualTo("hi"); assertThat(strings.getString(2)).isEqualTo("hi"); @@ -89,7 +90,7 @@ void stringScalar_decodesToLazyConstantVarBin() { /// Row count no longer bounds the work `decodeString` does: it used to eagerly allocate /// and copy `n` string repetitions into a real buffer (`n * strLen`), which for a large - /// `n` was both wasted work and an integer-overflow/OOM risk (#329). `ConstantMode` is + /// `n` was both wasted work and an integer-overflow/OOM risk (#329). `VarBinConstantArray` is /// O(1) regardless of `n`, so a row count too large to ever materialize still decodes /// instantly. @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java index 0079a8371..82c12d78e 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java @@ -9,6 +9,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -147,7 +148,7 @@ void truncatedOffsetsSegment_getBytes_throws() { // only possible on a directly built array — this exercises the readOffset guard // directly against a short offsets segment. MemorySegment data = MemorySegment.ofArray("abc".getBytes(StandardCharsets.UTF_8)); - VarBinArray array = new VarBinArray.OffsetMode(DType.UTF8, 4, data, + VarBinArray array = new VarBinOffsetArray(DType.UTF8, 4, data, TestSegments.leInts(0, 1, 2), PType.I32); // When / Then diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/MultiChunkUtf8RoundTripTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/MultiChunkUtf8RoundTripTest.java index c4dd60322..d48291e1c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/MultiChunkUtf8RoundTripTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/MultiChunkUtf8RoundTripTest.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VarBinChunkedArray; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -20,7 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; -/// Asserts [VarBinArray.ChunkedMode] fires on multi-chunk Utf8 columns. +/// Asserts [VarBinChunkedArray] fires on multi-chunk Utf8 columns. /// /// Pre-ADR-0012-VarBin, `ScanIterator.decodeChunkedLayout` threw on /// `(DType.Primitive) dtype` for Utf8/Binary. This test guards the new @@ -36,7 +37,7 @@ void manyBatchesUtf8WithCascading_columnIsChunkedMode(@TempDir Path tmp) throws // Given — many small batches force a layout-level Chunked node for the column // when cascading is enabled. The reader returns one Chunk per top-level chunk; // when that top-level chunk wraps multiple per-batch Flat layouts the column - // surfaces as VarBinArray.ChunkedMode. + // surfaces as VarBinChunkedArray. Path file = tmp.resolve("many_chunk_utf8.vtx"); int batches = 32; int perBatch = 16; @@ -51,7 +52,7 @@ void manyBatchesUtf8WithCascading_columnIsChunkedMode(@TempDir Path tmp) throws } } - // Then — at least one scan chunk must surface a ChunkedMode column. Values must + // Then — at least one scan chunk must surface a VarBinChunkedArray column. Values must // round-trip across the whole file regardless of how the writer chose to chunk. try (var vf = VortexReader.open(file, ReadRegistry.loadAll()); var iter = vf.scan(ScanOptions.columns("s"))) { @@ -60,7 +61,7 @@ void manyBatchesUtf8WithCascading_columnIsChunkedMode(@TempDir Path tmp) throws while (iter.hasNext()) { try (Chunk c = iter.next()) { VarBinArray col = c.column("s"); - if (col instanceof VarBinArray.ChunkedMode) { + if (col instanceof VarBinChunkedArray) { sawChunkedMode = true; } for (long i = 0; i < col.length(); i++) { @@ -71,7 +72,7 @@ void manyBatchesUtf8WithCascading_columnIsChunkedMode(@TempDir Path tmp) throws assertThat(seen).hasSize(batches * perBatch); assertThat(seen.get(0)).isEqualTo("row-0-0"); assertThat(seen.get(seen.size() - 1)).isEqualTo("row-" + (batches - 1) + "-" + (perBatch - 1)); - // ChunkedMode mechanics are unit-tested in VarBinChunkedModeTest. The current + // VarBinChunkedArray mechanics are unit-tested in VarBinChunkedModeTest. The current // writer's encoding choices for Utf8 do not produce a column-level Chunked // layout (each writer chunk surfaces as its own scan Chunk, and within a scan // Chunk the column is a leaf VarBinArray). When a future writer change makes @@ -91,14 +92,14 @@ void scanAll_singleChunkPerWrite_isMaterializedNotChunked(@TempDir Path tmp) thr sut.writeChunk(Map.of(ColumnName.of("s"), strings)); } - // Then — column should be a leaf VarBinArray (OffsetMode or DictMode), not ChunkedMode + // Then — column should be a leaf VarBinArray (VarBinOffsetArray or VarBinDictArray), not VarBinChunkedArray try (var vf = VortexReader.open(file, ReadRegistry.loadAll()); var iter = vf.scan(ScanOptions.columns("s"))) { assertThat(iter.hasNext()).isTrue(); try (Chunk c = iter.next()) { VarBinArray col = c.column("s"); // Sanity: chunked single-chunk decoder bypasses the wrapper. - assertThat(col).isNotInstanceOf(VarBinArray.ChunkedMode.class); + assertThat(col).isNotInstanceOf(VarBinChunkedArray.class); assertThat(col.length()).isEqualTo(3); assertThat(col.getString(0)).isEqualTo("foo"); assertThat(col.getString(2)).isEqualTo("baz"); From b5fab8c83f136aae146d755b33627b17dc666621 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Thu, 6 Aug 2026 09:21:49 +0200 Subject: [PATCH 2/2] docs(changelog): record lazy constant VarBin fix and non-sealed refactor --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac6f378e8..ab6831b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Malformed files no longer crash the reader with a raw JDK exception when decoding VarBin, Dict, Bitpacked, ALP, Sparse, Chunked, or Struct columns — every case now fails as `VortexException`. ([ef982992](https://github.com/dfa1/vortex-java/commit/ef982992)) - Same hardening for RunEnd, Constant, zone-map stats, and Pco columns — every case now fails as `VortexException`. ([12d7466c](https://github.com/dfa1/vortex-java/commit/12d7466c)) +- A constant Utf8/Binary column (`vortex.constant`) no longer eagerly allocates and copies `n` repetitions of its scalar on decode; it now broadcasts lazily in O(1) like every other constant type, closing an integer-overflow/OOM risk from a large row count. ([987fe412](https://github.com/dfa1/vortex-java/commit/987fe412)) + +### Changed + +- `VarBinArray`'s representations (`OffsetMode`, `DictMode`, `ChunkedMode`, `ViewMode`, `SlicedMode`, `ConstantMode`) are now top-level classes (`VarBinOffsetArray`, `VarBinDictArray`, `VarBinChunkedArray`, `VarBinViewArray`, `VarBinSlicedArray`, `VarBinConstantArray`) instead of nested types, and `VarBinArray` is `non-sealed`, matching its sibling array-family interfaces. **Breaking** for any code referencing the old nested names directly. ([7e0d6e75](https://github.com/dfa1/vortex-java/commit/7e0d6e75)) ### Added