From 42c25a038b8cf2856acb950c2e6252e3054e9c35 Mon Sep 17 00:00:00 2001 From: Roman Heinrich Date: Sun, 2 Aug 2026 19:09:17 +0200 Subject: [PATCH] fix(parquet): enable Zstandard-compressed reads Enable the parquet crate's zstd feature so ExArrow can read Zstandard-compressed Parquet files. Previously, reads failed at runtime because default Parquet features were disabled and zstd was not explicitly enabled. Add an externally generated Zstandard fixture and regression test, and document compression support in the Parquet reader guide and changelog. --- CHANGELOG.md | 9 ++++ docs/parquet_guide.md | 6 +++ lib/ex_arrow/parquet/reader.ex | 2 + native/ex_arrow_native/Cargo.lock | 66 ++++++++++++++++++++++++++++- native/ex_arrow_native/Cargo.toml | 2 +- test/ex_arrow/parquet_test.exs | 15 +++++++ test/fixtures/README.md | 7 ++- test/fixtures/parquet_zstd.parquet | Bin 0 -> 362 bytes 8 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/parquet_zstd.parquet diff --git a/CHANGELOG.md b/CHANGELOG.md index bb62a67..7782bdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Parquet readers now support Zstandard-compressed files. The native crate had + disabled the `parquet` crate's default features without explicitly enabling + its `zstd` feature, so reads failed at runtime with `Disabled feature at + compile time: zstd`. + ## [0.7.2] - 2026-07-08 ### Fixed diff --git a/docs/parquet_guide.md b/docs/parquet_guide.md index 369d9d0..42dcae8 100644 --- a/docs/parquet_guide.md +++ b/docs/parquet_guide.md @@ -40,6 +40,12 @@ parquet_bytes = File.read!("/data/events.parquet") batch = ExArrow.Stream.next(stream) ``` +### Compression support + +ExArrow reads uncompressed and Zstandard-compressed Parquet files. Compression +is decoded in native memory as record batches are requested; it does not change +the lazy row-group streaming behavior. + ### Schema introspection `ExArrow.Stream.schema/1` never fails for Parquet streams (the schema is diff --git a/lib/ex_arrow/parquet/reader.ex b/lib/ex_arrow/parquet/reader.ex index 946bf14..4276b45 100644 --- a/lib/ex_arrow/parquet/reader.ex +++ b/lib/ex_arrow/parquet/reader.ex @@ -3,6 +3,8 @@ defmodule ExArrow.Parquet.Reader do Parquet file reader: open a `.parquet` file or an in-memory binary and receive an `ExArrow.Stream` that yields record batches. + Uncompressed and Zstandard-compressed Parquet files are supported. + The stream interface is identical to `ExArrow.IPC.Reader` and ADBC streams — use `ExArrow.Stream.schema/1`, `ExArrow.Stream.next/1`, and `ExArrow.Stream.to_list/1` to consume it. diff --git a/native/ex_arrow_native/Cargo.lock b/native/ex_arrow_native/Cargo.lock index f2fe457..df28700 100644 --- a/native/ex_arrow_native/Cargo.lock +++ b/native/ex_arrow_native/Cargo.lock @@ -388,6 +388,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -615,10 +617,21 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "h2" version = "0.4.13" @@ -827,6 +840,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.89" @@ -1081,6 +1104,7 @@ dependencies = [ "seq-macro", "thrift", "twox-hash", + "zstd", ] [[package]] @@ -1127,6 +1151,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1183,6 +1213,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "regex" version = "1.12.3" @@ -2031,3 +2067,31 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/native/ex_arrow_native/Cargo.toml b/native/ex_arrow_native/Cargo.toml index 24e3958..b7fb7a4 100644 --- a/native/ex_arrow_native/Cargo.toml +++ b/native/ex_arrow_native/Cargo.toml @@ -20,7 +20,7 @@ arrow-buffer = { version = "56", default-features = false } arrow-data = { version = "56", default-features = false } arrow-select = { version = "56", default-features = false } arrow-ord = { version = "56", default-features = false } -parquet = { version = "56", default-features = false, features = ["arrow"] } +parquet = { version = "56", default-features = false, features = ["arrow", "zstd"] } arrow-flight = { version = "56", features = ["flight-sql"] } futures = "0.3" tokio = { version = "1", features = ["rt-multi-thread", "net", "sync", "time", "macros"] } diff --git a/test/ex_arrow/parquet_test.exs b/test/ex_arrow/parquet_test.exs index 8a49bea..fd02174 100644 --- a/test/ex_arrow/parquet_test.exs +++ b/test/ex_arrow/parquet_test.exs @@ -63,6 +63,21 @@ defmodule ExArrow.ParquetTest do assert ExArrow.RecordBatch.num_rows(rt_batch) == ExArrow.RecordBatch.num_rows(batch) end + test "reads a Zstandard-compressed file" do + path = Path.expand("../fixtures/parquet_zstd.parquet", __DIR__) + + assert {:ok, stream} = Parquet.Reader.from_file(path) + assert {:ok, schema} = Stream.schema(stream) + assert Schema.field_names(schema) == ["id", "name"] + + rows = + stream + |> Stream.to_list() + |> Enum.sum_by(&ExArrow.RecordBatch.num_rows/1) + + assert rows == 3 + end + test "from_file returns error for missing file" do assert {:error, _msg} = Parquet.Reader.from_file("/tmp/this_does_not_exist_xyz.parquet") end diff --git a/test/fixtures/README.md b/test/fixtures/README.md index 6c739e1..d9ec7df 100644 --- a/test/fixtures/README.md +++ b/test/fixtures/README.md @@ -2,5 +2,10 @@ - **IPC file format (golden):** File-format bytes are produced by `ExArrow.Native.ipc_test_fixture_file_binary/0` (schema: `id` int64, `name` utf8; one batch of 2 rows). Tests use this for `ExArrow.IPC.File.from_binary/1` and for compatibility checks. - **IPC from_file:** Tests that need a path write a temp file with `ExArrow.Native.ipc_file_writer_to_file/3` and remove it in an `after` block. +- **Zstandard Parquet:** `parquet_zstd.parquet` contains three rows with `id` (int64) and `name` (utf8) columns. It verifies compatibility with compressed files produced outside ExArrow. Regenerate it with: -No pre-generated `.arrow` files are committed; the single file-format fixture is generated in Rust for reproducibility. + ```sh + duckdb -c "COPY (SELECT * FROM (VALUES (1::BIGINT, 'alpha'), (2::BIGINT, 'beta'), (3::BIGINT, 'gamma')) AS t(id, name)) TO 'test/fixtures/parquet_zstd.parquet' (FORMAT PARQUET, COMPRESSION ZSTD);" + ``` + +No pre-generated `.arrow` files are committed; the IPC file-format fixture is generated in Rust for reproducibility. diff --git a/test/fixtures/parquet_zstd.parquet b/test/fixtures/parquet_zstd.parquet new file mode 100644 index 0000000000000000000000000000000000000000..84f0a33c7a44ee3a828252c7baeaeaa670d8a116 GIT binary patch literal 362 zcmZutJ5Iwu6nwkbD;K6n@m6aoLR(s6j4Vx0I-fNY!L^Ge9n=}&~pj8 z6dZvIAn|r33J|ZFot>FCdb8{K5CJp1?V%tbPynQ_=U@8d9bnG^KrjXzjtJS|@bJM} zM$#*s|Mh4mwHBYGX4zK8BaoLHV~I!8GzNK;U4S|+wu`4Czh5t^l1LBGi@D-;5uXBv z1ih7M${0Y3s2lYX=%N=PL{@^K6!K11Nsy`FrXpj>W} zHdDT9r+n9%B7VJ{S-vqTl0Y69r0JCEPjB?Dd|B7eE4>@`$Nhm$^KHE>bUrbI%W`Z+ MqcixVwKKZ%H^9_AM*si- literal 0 HcmV?d00001