diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d260ef470..db2340414 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -208,7 +208,7 @@ jobs: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} name: Meson - ${{ matrix.title }} runs-on: ${{ matrix.runs-on }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false @@ -221,9 +221,22 @@ jobs: meson-setup-args: -Drest_integration_test=enabled - title: AMD64 Windows 2025 runs-on: windows-2025 - meson-setup-args: --vsenv + meson-setup-args: --vsenv --buildtype=release --cmake-prefix-path=C:/vcpkg/installed/x64-windows-static-md - title: AArch64 macOS 26 runs-on: macos-26 + - title: AMD64 Ubuntu 26.04, all catalogs, S3 and library variants + runs-on: ubuntu-26.04 + CC: gcc-14 + CXX: g++-14 + sql-clients: true + benchmarks: true + meson-setup-args: --default-library=both --buildtype=release -Db_ndebug=true --force-fallback-for=arrow,avro -Dhive=enabled -Dsql_catalog=enabled -Dsql_sqlite=enabled -Dsql_postgresql=enabled -Dsql_mysql=enabled -Dbenchmarks=enabled -Ds3=enabled -Dsigv4=enabled + - title: AMD64 Ubuntu 26.04, bundle disabled + runs-on: ubuntu-26.04 + CC: gcc-14 + CXX: g++-14 + bundle: disabled + meson-setup-args: -Dbundle=disabled steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -244,13 +257,32 @@ jobs: run: | echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV + - name: Install SQL client development packages + if: ${{ matrix.sql-clients }} + run: | + # Match the PostgreSQL packages preinstalled from PGDG on the runner. + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install -y libsqlite3-dev libpq-dev "postgresql-server-dev-$(pg_config --version | awk '{print $2}' | cut -d. -f1)" default-libmysqlclient-dev libcurl4-openssl-dev libgpg-error-dev libunistring-dev libsasl2-dev libdb-dev + echo "LIBRARY_PATH=$(pg_config --pkglibdir)" >> "$GITHUB_ENV" + # Complete the system packages' static link interfaces. + sudo sed -i '/^Requires.private:/s/$/ libsasl2/' "$(pkg-config --variable=pcfiledir libcurl)/libcurl.pc" + sudo sed -i '/^Libs.private:/s/$/ -ldb -lsqlite3 -lpq -lmysqlclient/' "$(pkg-config --variable=pcfiledir libsasl2)/libsasl2.pc" + sudo sed -i 's/-lpgcommon /-lpgcommon_shlib /' "$(pkg-config --variable=pcfiledir libpq)/libpq.pc" + - name: Install OpenSSL static dependencies + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libjitterentropy3-dev + - name: Install Zlib and pkgconf + if: runner.os == 'Windows' + run: | + vcpkg install zlib:x64-windows-static-md pkgconf:x64-windows-static-md + "PKG_CONFIG=C:/vcpkg/installed/x64-windows-static-md/tools/pkgconf/pkgconf.exe" >> $env:GITHUB_ENV - name: Set up sccache uses: ./.github/actions/setup-sccache with: key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Build Iceberg run: | - meson setup builddir ${{ matrix.meson-setup-args || '' }} + meson setup builddir ${{ matrix.meson-setup-args || '' }} --prefix="${{ github.workspace }}/install" --libdir=lib meson compile -C builddir - name: Save sccache if: always() @@ -261,3 +293,8 @@ jobs: - name: Test Iceberg run: | meson test -C builddir --timeout-multiplier 0 --print-errorlogs + - name: Test installed Meson libraries + run: python ci/scripts/test_meson_install.py builddir --bundle=${{ matrix.bundle || 'enabled' }} + - name: Test Meson benchmarks + if: ${{ matrix.benchmarks }} + run: meson test -C builddir --benchmark --print-errorlogs diff --git a/README.md b/README.md index 7206c4597..52c50c12a 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ C++ implementation of [Apache Iceberg™](https://iceberg.apache.org/). **Required:** - C++23 compliant compiler (GCC 14+, Clang 18+, MSVC 2022+) -- CMake 3.25+ or Meson 1.5+ +- CMake 3.25+ or Meson 1.8.3+ - [Ninja](https://ninja-build.org/) (recommended build backend) **Optional:** @@ -113,6 +113,31 @@ meson compile -C builddir meson test -C builddir --timeout-multiplier 0 ``` +Meson builds `iceberg_bundle` by default, including Arrow filesystem, Avro, +and Parquet support. Use `-Dbundle=disabled` to omit it. + +System dependencies are preferred. System Arrow and Parquet are resolved through +their CMake packages; separate Arrow component pkg-config files are not required. +Arrow, Avro, CRoaring, and sqlpp23 source fallbacks use upstream CMake. Use +`--force-fallback-for=arrow,avro` to force Arrow and Avro source builds. + +Enable S3 with `-Ds3=enabled`. System Arrow must include S3 support; the Arrow +source fallback bundles AWS SDK by default. +On Linux, the bundled SDK uses system OpenSSL for s2n TLS support. +Use `-Dbundle_awssdk=false` to use an installed AWS SDK in the Arrow source build. +When S3 bundles AWS SDK, `-Dsigv4=enabled` reuses it for REST authentication; +SigV4 without S3 requires an installed AWS SDK. + +Static libraries are the default. Installed libraries are available through +pkg-config, for example `dependency('iceberg_bundle')`. With +`--default-library=both`, use `dependency('iceberg_bundle_static', static: true)` +or `dependency('iceberg_bundle_shared', static: false)` to select a variant; +the same suffixes and `static` argument apply to other modules. + +Enable Hive with `-Dhive=enabled`. It reuses Thrift from the Arrow source fallback +by default; use `-Dbundle_thrift=false` for system Thrift. SQL connectors require +their native client development packages. + Meson provides built-in equivalents for several CMake options: - `--default-library=` instead of `ICEBERG_BUILD_STATIC` / `ICEBERG_BUILD_SHARED` @@ -123,7 +148,16 @@ Meson-specific options (configured via `-D