From 6c46eea7aebbb63eebf8f33a8894253fc648d659 Mon Sep 17 00:00:00 2001 From: Zhao Junwang Date: Sat, 5 Sep 2026 23:10:10 +0800 Subject: [PATCH 1/6] feat(meson): align library modules with CMake Provide the same library modules through Meson and CMake, including the bundle, Hive, and SQL catalogs, so applications can use either build system. Keep installed static and shared libraries usable by downstream consumers and cover their public interfaces with installation checks. Closes #256 AI assistance was used in this change. --- .github/workflows/test.yml | 43 +- README.md | 28 +- ci/scripts/test_meson_install.py | 93 +++ ci/test-install/bundle_smoke.cc | 124 ++++ ci/test-install/library_smoke.cc | 108 +++ ci/test-install/meson.build | 70 ++ ci/test-install/meson.options | 24 + cmake_modules/IcebergMesonSubproject.cmake | 34 + .../IcebergThirdpartyToolchain.cmake | 21 +- meson.build | 4 +- meson.options | 49 ++ mkdocs/docs/getting-started.md | 28 +- mkdocs/docs/library.md | 3 +- requirements.txt | 2 +- src/iceberg/CMakeLists.txt | 1 + src/iceberg/arrow/meson.build | 20 + src/iceberg/arrow/s3/meson.build | 18 + src/iceberg/avro/meson.build | 27 + src/iceberg/benchmark/meson.build | 12 +- src/iceberg/catalog/CMakeLists.txt | 2 + src/iceberg/catalog/hive/meson.build | 62 ++ src/iceberg/catalog/meson.build | 9 + src/iceberg/catalog/rest/auth/meson.build | 1 + src/iceberg/catalog/rest/meson.build | 41 +- src/iceberg/catalog/sql/meson.build | 60 ++ src/iceberg/meson.build | 692 ++++++++++++++++-- src/iceberg/parquet/meson.build | 21 + src/iceberg/test/meson.build | 211 +++++- subprojects/arrow.wrap | 29 + subprojects/avro.wrap | 28 + subprojects/packagefiles/arrow/CMakeLists.txt | 76 ++ subprojects/packagefiles/avro/CMakeLists.txt | 20 + subprojects/sqlpp23.wrap | 26 + 33 files changed, 1862 insertions(+), 125 deletions(-) create mode 100644 ci/scripts/test_meson_install.py create mode 100644 ci/test-install/bundle_smoke.cc create mode 100644 ci/test-install/library_smoke.cc create mode 100644 ci/test-install/meson.build create mode 100644 ci/test-install/meson.options create mode 100644 cmake_modules/IcebergMesonSubproject.cmake create mode 100644 src/iceberg/arrow/meson.build create mode 100644 src/iceberg/arrow/s3/meson.build create mode 100644 src/iceberg/avro/meson.build create mode 100644 src/iceberg/catalog/hive/meson.build create mode 100644 src/iceberg/catalog/sql/meson.build create mode 100644 src/iceberg/parquet/meson.build create mode 100644 subprojects/arrow.wrap create mode 100644 subprojects/avro.wrap create mode 100644 subprojects/packagefiles/arrow/CMakeLists.txt create mode 100644 subprojects/packagefiles/avro/CMakeLists.txt create mode 100644 subprojects/sqlpp23.wrap diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d260ef470..b2ed01f15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -208,7 +208,7 @@ jobs: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} name: Meson - ${{ matrix.title }} runs-on: ${{ matrix.runs-on }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: max-parallel: 15 fail-fast: false @@ -221,9 +221,22 @@ jobs: meson-setup-args: -Drest_integration_test=enabled - title: AMD64 Windows 2025 runs-on: windows-2025 - meson-setup-args: --vsenv + meson-setup-args: --vsenv --buildtype=release --cmake-prefix-path=C:/vcpkg/installed/x64-windows-static-md - title: AArch64 macOS 26 runs-on: macos-26 + - title: AMD64 Ubuntu 26.04, all catalogs and library variants + runs-on: ubuntu-26.04 + CC: gcc-14 + CXX: g++-14 + sql-clients: true + benchmarks: true + meson-setup-args: --default-library=both --buildtype=release -Db_ndebug=true --force-fallback-for=arrow,avro -Dhive=enabled -Dsql_catalog=enabled -Dsql_sqlite=enabled -Dsql_postgresql=enabled -Dsql_mysql=enabled -Dbenchmarks=enabled + - title: AMD64 Ubuntu 26.04, bundle disabled + runs-on: ubuntu-26.04 + CC: gcc-14 + CXX: g++-14 + bundle: disabled + meson-setup-args: -Dbundle=disabled steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -244,13 +257,32 @@ jobs: run: | echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV + - name: Install SQL client development packages + if: ${{ matrix.sql-clients }} + run: | + # Match the PostgreSQL packages preinstalled from PGDG on the runner. + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install -y libsqlite3-dev libpq-dev "postgresql-server-dev-$(pg_config --version | awk '{print $2}' | cut -d. -f1)" default-libmysqlclient-dev libcurl4-openssl-dev libgpg-error-dev libunistring-dev libsasl2-dev libdb-dev + echo "LIBRARY_PATH=$(pg_config --pkglibdir)" >> "$GITHUB_ENV" + # Complete the system packages' static link interfaces. + sudo sed -i '/^Requires.private:/s/$/ libsasl2/' "$(pkg-config --variable=pcfiledir libcurl)/libcurl.pc" + sudo sed -i '/^Libs.private:/s/$/ -ldb -lsqlite3 -lpq -lmysqlclient/' "$(pkg-config --variable=pcfiledir libsasl2)/libsasl2.pc" + sudo sed -i 's/-lpgcommon /-lpgcommon_shlib /' "$(pkg-config --variable=pcfiledir libpq)/libpq.pc" + - name: Install OpenSSL static dependencies + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libjitterentropy3-dev + - name: Install Zlib and pkgconf + if: runner.os == 'Windows' + run: | + vcpkg install zlib:x64-windows-static-md pkgconf:x64-windows-static-md + "PKG_CONFIG=C:/vcpkg/installed/x64-windows-static-md/tools/pkgconf/pkgconf.exe" >> $env:GITHUB_ENV - name: Set up sccache uses: ./.github/actions/setup-sccache with: key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Build Iceberg run: | - meson setup builddir ${{ matrix.meson-setup-args || '' }} + meson setup builddir ${{ matrix.meson-setup-args || '' }} --prefix="${{ github.workspace }}/install" --libdir=lib meson compile -C builddir - name: Save sccache if: always() @@ -261,3 +293,8 @@ jobs: - name: Test Iceberg run: | meson test -C builddir --timeout-multiplier 0 --print-errorlogs + - name: Test installed Meson libraries + run: python ci/scripts/test_meson_install.py builddir --bundle=${{ matrix.bundle || 'enabled' }} + - name: Test Meson benchmarks + if: ${{ matrix.benchmarks }} + run: meson test -C builddir --benchmark --print-errorlogs diff --git a/README.md b/README.md index 7206c4597..e8d20d216 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ C++ implementation of [Apache Iceberg™](https://iceberg.apache.org/). **Required:** - C++23 compliant compiler (GCC 14+, Clang 18+, MSVC 2022+) -- CMake 3.25+ or Meson 1.5+ +- CMake 3.25+ or Meson 1.8.3+ - [Ninja](https://ninja-build.org/) (recommended build backend) **Optional:** @@ -113,6 +113,24 @@ meson compile -C builddir meson test -C builddir --timeout-multiplier 0 ``` +Meson builds `iceberg_bundle` by default, including Arrow filesystem, Avro, +and Parquet support. Use `-Dbundle=disabled` to omit it. + +System dependencies are preferred. Arrow, Avro, CRoaring, and sqlpp23 source +fallbacks use upstream CMake; use `--force-fallback-for=arrow,avro` to force +Arrow and Avro source builds. S3 requires `-Ds3=enabled` and either a system +Arrow with S3 support or an installed AWS SDK for the Arrow source build. + +Static libraries are the default. Installed libraries are available through +pkg-config, for example `dependency('iceberg_bundle')`. With +`--default-library=both`, use `dependency('iceberg_bundle_static', static: true)` +or `dependency('iceberg_bundle_shared', static: false)` to select a variant; +the same suffixes and `static` argument apply to other modules. + +Enable Hive with `-Dhive=enabled`. It reuses Thrift from the Arrow source fallback +by default; use `-Dbundle_thrift=false` for system Thrift. SQL connectors require +their native client development packages. + Meson provides built-in equivalents for several CMake options: - `--default-library=` instead of `ICEBERG_BUILD_STATIC` / `ICEBERG_BUILD_SHARED` @@ -123,7 +141,15 @@ Meson-specific options (configured via `-D