feat(ci): Add ASAN and UBSAN build and CI job

Author: czentgr

.github/workflows/linux-build-base.yml

@@ -270,3 +270,111 @@ jobs:
         with:
           path: ${{ env.CCACHE_DIR }}
           key: ccache-fedora-debug-default-gcc
+
+  asan-ubsan-adapters:
+    name: Linux debug with adapters - testing asan and ubsan
+    # prevent errors when forks ff their main branch
+    if: ${{ github.repository == 'facebookincubator/velox' }}
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/facebookincubator/velox-dev:adapters
+      volumes:
+        - /usr:/host_usr
+        - /opt:/host_opt
+    defaults:
+      run:
+        shell: bash
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+      VELOX_DEPENDENCY_SOURCE: SYSTEM
+      faiss_SOURCE: BUNDLED
+    concurrency:
+      group: ${{ github.workflow }}-asan-ubsan-adapters-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    steps:
+      - name: Free Disk Space
+        run: |
+          # Re-used from free-disk-space github action.
+          getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); }
+          # Show before
+          echo "Original available disk space: " $(getAvailableSpace)
+          # Remove DotNet.
+          rm -rf /host_usr/share/dotnet || true
+          # Remove android
+          rm -rf /host_usr/local/lib/android || true
+          # Remove CodeQL
+          rm -rf /host_opt/hostedtoolcache/CodeQL || true
+          # Show after
+          echo "New available disk space: " $(getAvailableSpace)
+
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-asan-ubsan-adapters
+
+      - name: Zero Ccache Statistics
+        run: |
+          ccache -sz
+
+      - name: Make Debug Build
+        env:
+          MAKEFLAGS: NUM_THREADS=4 MAX_HIGH_MEM_JOBS=3 MAX_LINK_JOBS=3
+          CUDA_VERSION: '12.8'
+          CUDA_ARCHITECTURES: 70
+          CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc
+          # Set compiler to GCC 12 for CUDA
+          CUDA_FLAGS: -ccbin /opt/rh/gcc-toolset-12/root/usr/bin
+        run: |
+          EXTRA_CMAKE_FLAGS=(
+            "-DVELOX_ENABLE_ASAN_UBSAN_SANITIZERS=ON"
+            "-DVELOX_ENABLE_BENCHMARKS=ON"
+            "-DVELOX_ENABLE_EXAMPLES=ON"
+            "-DVELOX_ENABLE_ARROW=ON"
+            "-DVELOX_ENABLE_GEO=ON"
+            "-DVELOX_ENABLE_PARQUET=ON"
+            "-DVELOX_ENABLE_HDFS=ON"
+            "-DVELOX_ENABLE_S3=ON"
+            "-DVELOX_ENABLE_GCS=ON"
+            "-DVELOX_ENABLE_ABFS=ON"
+            "-DVELOX_ENABLE_WAVE=ON"
+            "-DVELOX_MONO_LIBRARY=ON"
+            "-DVELOX_BUILD_SHARED=ON"
+            "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
+          )
+          clang++ --version
+          dnf install -y compiler-rt
+          CC=clang CXX=clang++ make debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
+
+      - name: Ccache after
+        run: ccache -s
+
+      - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-asan-ubsan-adapters
+
+      - name: Run Tests
+        env:
+          LIBHDFS3_CONF: ${{ github.workspace }}/scripts/ci/hdfs-client.xml
+        working-directory: _build/debug
+        run: |
+          # Can be removed after images are rebuild
+          if [ -f "/opt/miniforge/etc/profile.d/conda.sh" ]; then
+            source "/opt/miniforge/etc/profile.d/conda.sh"
+            conda activate adapters
+          fi
+          # Needed for HADOOP 3.3.6 minicluster. Can remove after updating to 3.4.2.
+          wget https://repo1.maven.org/maven2/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4.jar -O /usr/local/hadoop/share/hadoop/mapreduce/mockito-core-2.23.4.jar
+
+          export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob`
+          ctest -j 4 --label-exclude cuda_driver --output-on-failure --no-tests=error

CMakeLists.txt

@@ -179,6 +179,13 @@ option(VELOX_ENABLE_FAISS "Build faiss vector search support" OFF)
 # is broken if you use ninja.
 option(VELOX_FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." OFF)
 
+# Enable the address and undefined behavior sanitizers. They can run together.
+option(
+  VELOX_ENABLE_ASAN_UBSAN_SANITIZERS
+  "Builds velox with Clang ASAN and UBSAN sanitizer flags"
+  OFF
+)
+
 if(${VELOX_BUILD_MINIMAL} OR ${VELOX_BUILD_MINIMAL_WITH_DWIO})
   # Enable and disable components for velox base build
   set(VELOX_BUILD_TESTING OFF)
@@ -508,10 +515,7 @@ endif()
 
 # Set after the test of the CUDA compiler. Otherwise, the test fails with
 # -latomic not found because it is added right after the compiler exe.
-if(
-  "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang"
-  AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15
-)
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_EQUAL 15)
   set(CMAKE_EXE_LINKER_FLAGS "-latomic")
 endif()
 
@@ -729,7 +733,23 @@ if("${TREAT_WARNINGS_AS_ERRORS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
 endif()
 
+# Enable the ASAN and UBSAN sanitzers for Clang 20 and above.
+# Also only eligible with the type debug build.
+if(
+  VELOX_ENABLE_ASAN_UBSAN_SANITIZERS
+  AND CMAKE_BUILD_TYPE STREQUAL "Debug"
+  AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
+  AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 20
+)
+  set(
+    CMAKE_CXX_FLAGS
+    "${CMAKE_CXX_FLAGS} -O1 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer"
+  )
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined")
+endif()
+
 message("FINAL CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
+message("FINAL CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS}")
 
 if(VELOX_ENABLE_ARROW)
   velox_set_source(Arrow)

scripts/setup-centos9.sh

@@ -56,7 +56,8 @@ function install_build_prerequisites {
     dnf config-manager --set-enabled crb
     dnf update -y
   fi
-  dnf_install autoconf automake ccache clang gcc-toolset-12 gcc-toolset-14 git libtool \
+  dnf_install autoconf automake ccache clang compiler-rt \
+    gcc-toolset-12 gcc-toolset-14 git libtool \
     llvm ninja-build python3-pip python3-devel wget which
 
   install_uv

velox/common/base/Macros.h

@@ -28,6 +28,19 @@
   _Pragma("GCC diagnostic pop");
 #endif
 
+#if defined(__clang__) && defined(__has_warning) && \
+    __has_warning("-Wmissing-designated-field-initializers")
+#define VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING \
+  _Pragma("clang diagnostic push");                                  \
+  _Pragma(                                                           \
+      "clang diagnostic ignored \"-Wmissing-designated-field-initializers\"")
+#define VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING \
+  _Pragma("clang diagnostic pop");
+#else
+#define VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING
+#define VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING
+#endif
+
 #define VELOX_CONCAT(x, y) x##y
 // Need this extra layer to expand __COUNTER__.
 #define VELOX_VARNAME_IMPL(x, y) VELOX_CONCAT(x, y)

velox/functions/prestosql/DateTimeFunctions.h

@@ -1963,10 +1963,11 @@ struct ParseDurationFunction {
   FOLLY_ALWAYS_INLINE void call(
       out_type<IntervalDayTime>& result,
       const arg_type<Varchar>& amountUnit) {
+    VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING
     static const LazyRE2 kDurationRegex{
         .pattern_ = R"(^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$)",
-        .options_ = {},
     };
+    VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING
     // TODO: Remove re2::StringPiece != std::string_view hacks.
     // It's needed because for some systems in CI,
     // re2 and abseil libraries are old.