diff --git a/CHANGELOG.md b/CHANGELOG.md index bb62a67..7782bdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Parquet readers now support Zstandard-compressed files. The native crate had + disabled the `parquet` crate's default features without explicitly enabling + its `zstd` feature, so reads failed at runtime with `Disabled feature at + compile time: zstd`. + ## [0.7.2] - 2026-07-08 ### Fixed diff --git a/docs/parquet_guide.md b/docs/parquet_guide.md index 369d9d0..42dcae8 100644 --- a/docs/parquet_guide.md +++ b/docs/parquet_guide.md @@ -40,6 +40,12 @@ parquet_bytes = File.read!("/data/events.parquet") batch = ExArrow.Stream.next(stream) ``` +### Compression support + +ExArrow reads uncompressed and Zstandard-compressed Parquet files. Compression +is decoded in native memory as record batches are requested; it does not change +the lazy row-group streaming behavior. + ### Schema introspection `ExArrow.Stream.schema/1` never fails for Parquet streams (the schema is diff --git a/lib/ex_arrow/parquet/reader.ex b/lib/ex_arrow/parquet/reader.ex index 946bf14..4276b45 100644 --- a/lib/ex_arrow/parquet/reader.ex +++ b/lib/ex_arrow/parquet/reader.ex @@ -3,6 +3,8 @@ defmodule ExArrow.Parquet.Reader do Parquet file reader: open a `.parquet` file or an in-memory binary and receive an `ExArrow.Stream` that yields record batches. + Uncompressed and Zstandard-compressed Parquet files are supported. + The stream interface is identical to `ExArrow.IPC.Reader` and ADBC streams — use `ExArrow.Stream.schema/1`, `ExArrow.Stream.next/1`, and `ExArrow.Stream.to_list/1` to consume it. diff --git a/native/ex_arrow_native/Cargo.lock b/native/ex_arrow_native/Cargo.lock index f2fe457..df28700 100644 --- a/native/ex_arrow_native/Cargo.lock +++ b/native/ex_arrow_native/Cargo.lock @@ -388,6 +388,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -615,10 +617,21 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "h2" version = "0.4.13" @@ -827,6 +840,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.89" @@ -1081,6 +1104,7 @@ dependencies = [ "seq-macro", "thrift", "twox-hash", + "zstd", ] [[package]] @@ -1127,6 +1151,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1183,6 +1213,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "regex" version = "1.12.3" @@ -2031,3 +2067,31 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/native/ex_arrow_native/Cargo.toml b/native/ex_arrow_native/Cargo.toml index 24e3958..b7fb7a4 100644 --- a/native/ex_arrow_native/Cargo.toml +++ b/native/ex_arrow_native/Cargo.toml @@ -20,7 +20,7 @@ arrow-buffer = { version = "56", default-features = false } arrow-data = { version = "56", default-features = false } arrow-select = { version = "56", default-features = false } arrow-ord = { version = "56", default-features = false } -parquet = { version = "56", default-features = false, features = ["arrow"] } +parquet = { version = "56", default-features = false, features = ["arrow", "zstd"] } arrow-flight = { version = "56", features = ["flight-sql"] } futures = "0.3" tokio = { version = "1", features = ["rt-multi-thread", "net", "sync", "time", "macros"] } diff --git a/test/ex_arrow/parquet_test.exs b/test/ex_arrow/parquet_test.exs index 8a49bea..fd02174 100644 --- a/test/ex_arrow/parquet_test.exs +++ b/test/ex_arrow/parquet_test.exs @@ -63,6 +63,21 @@ defmodule ExArrow.ParquetTest do assert ExArrow.RecordBatch.num_rows(rt_batch) == ExArrow.RecordBatch.num_rows(batch) end + test "reads a Zstandard-compressed file" do + path = Path.expand("../fixtures/parquet_zstd.parquet", __DIR__) + + assert {:ok, stream} = Parquet.Reader.from_file(path) + assert {:ok, schema} = Stream.schema(stream) + assert Schema.field_names(schema) == ["id", "name"] + + rows = + stream + |> Stream.to_list() + |> Enum.sum_by(&ExArrow.RecordBatch.num_rows/1) + + assert rows == 3 + end + test "from_file returns error for missing file" do assert {:error, _msg} = Parquet.Reader.from_file("/tmp/this_does_not_exist_xyz.parquet") end diff --git a/test/fixtures/README.md b/test/fixtures/README.md index 6c739e1..d9ec7df 100644 --- a/test/fixtures/README.md +++ b/test/fixtures/README.md @@ -2,5 +2,10 @@ - **IPC file format (golden):** File-format bytes are produced by `ExArrow.Native.ipc_test_fixture_file_binary/0` (schema: `id` int64, `name` utf8; one batch of 2 rows). Tests use this for `ExArrow.IPC.File.from_binary/1` and for compatibility checks. - **IPC from_file:** Tests that need a path write a temp file with `ExArrow.Native.ipc_file_writer_to_file/3` and remove it in an `after` block. +- **Zstandard Parquet:** `parquet_zstd.parquet` contains three rows with `id` (int64) and `name` (utf8) columns. It verifies compatibility with compressed files produced outside ExArrow. Regenerate it with: -No pre-generated `.arrow` files are committed; the single file-format fixture is generated in Rust for reproducibility. + ```sh + duckdb -c "COPY (SELECT * FROM (VALUES (1::BIGINT, 'alpha'), (2::BIGINT, 'beta'), (3::BIGINT, 'gamma')) AS t(id, name)) TO 'test/fixtures/parquet_zstd.parquet' (FORMAT PARQUET, COMPRESSION ZSTD);" + ``` + +No pre-generated `.arrow` files are committed; the IPC file-format fixture is generated in Rust for reproducibility. diff --git a/test/fixtures/parquet_zstd.parquet b/test/fixtures/parquet_zstd.parquet new file mode 100644 index 0000000..84f0a33 Binary files /dev/null and b/test/fixtures/parquet_zstd.parquet differ