diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index 8e486f885e9a..afc13c580b7a 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -270,3 +270,111 @@ jobs: with: path: ${{ env.CCACHE_DIR }} key: ccache-fedora-debug-default-gcc + + asan-ubsan-adapters: + name: Linux debug with adapters - testing asan and ubsan + # prevent errors when forks ff their main branch + if: ${{ github.repository == 'facebookincubator/velox' }} + runs-on: ubuntu-latest + container: + image: ghcr.io/facebookincubator/velox-dev:adapters + volumes: + - /usr:/host_usr + - /opt:/host_opt + defaults: + run: + shell: bash + env: + CCACHE_DIR: ${{ github.workspace }}/ccache + VELOX_DEPENDENCY_SOURCE: SYSTEM + faiss_SOURCE: BUNDLED + concurrency: + group: ${{ github.workflow }}-asan-ubsan-adapters-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Free Disk Space + run: | + # Re-used from free-disk-space github action. + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + # Show before + echo "Original available disk space: " $(getAvailableSpace) + # Remove DotNet. + rm -rf /host_usr/share/dotnet || true + # Remove android + rm -rf /host_usr/local/lib/android || true + # Remove CodeQL + rm -rf /host_opt/hostedtoolcache/CodeQL || true + # Show after + echo "New available disk space: " $(getAvailableSpace) + + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + persist-credentials: false + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-asan-ubsan-adapters + + - name: Zero Ccache Statistics + run: | + ccache -sz + + - name: Make Debug Build + env: + MAKEFLAGS: NUM_THREADS=4 MAX_HIGH_MEM_JOBS=3 MAX_LINK_JOBS=3 + CUDA_VERSION: '12.8' + CUDA_ARCHITECTURES: 70 + CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc + # Set compiler to GCC 12 for CUDA + CUDA_FLAGS: -ccbin /opt/rh/gcc-toolset-12/root/usr/bin + run: | + EXTRA_CMAKE_FLAGS=( + "-DVELOX_ENABLE_ASAN_UBSAN_SANITIZERS=ON" + "-DVELOX_ENABLE_BENCHMARKS=ON" + "-DVELOX_ENABLE_EXAMPLES=ON" + "-DVELOX_ENABLE_ARROW=ON" + "-DVELOX_ENABLE_GEO=ON" + "-DVELOX_ENABLE_PARQUET=ON" + "-DVELOX_ENABLE_HDFS=ON" + "-DVELOX_ENABLE_S3=ON" + "-DVELOX_ENABLE_GCS=ON" + "-DVELOX_ENABLE_ABFS=ON" + "-DVELOX_ENABLE_WAVE=ON" + "-DVELOX_MONO_LIBRARY=ON" + "-DVELOX_BUILD_SHARED=ON" + "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" + ) + clang++ --version + dnf install -y compiler-rt + CC=clang CXX=clang++ make debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-asan-ubsan-adapters + + - name: Run Tests + env: + LIBHDFS3_CONF: ${{ github.workspace }}/scripts/ci/hdfs-client.xml + working-directory: _build/debug + run: | + # Can be removed after images are rebuild + if [ -f "/opt/miniforge/etc/profile.d/conda.sh" ]; then + source "/opt/miniforge/etc/profile.d/conda.sh" + conda activate adapters + fi + # Needed for HADOOP 3.3.6 minicluster. Can remove after updating to 3.4.2. + wget https://repo1.maven.org/maven2/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4.jar -O /usr/local/hadoop/share/hadoop/mapreduce/mockito-core-2.23.4.jar + + export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob` + ctest -j 4 --label-exclude cuda_driver --output-on-failure --no-tests=error diff --git a/CMakeLists.txt b/CMakeLists.txt index 24704c659a3c..452d12534b02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,6 +179,13 @@ option(VELOX_ENABLE_FAISS "Build faiss vector search support" OFF) # is broken if you use ninja. option(VELOX_FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." OFF) +# Enable the address and undefined behavior sanitizers. They can run together. +option( + VELOX_ENABLE_ASAN_UBSAN_SANITIZERS + "Builds velox with Clang ASAN and UBSAN sanitizer flags" + OFF +) + if(${VELOX_BUILD_MINIMAL} OR ${VELOX_BUILD_MINIMAL_WITH_DWIO}) # Enable and disable components for velox base build set(VELOX_BUILD_TESTING OFF) @@ -508,10 +515,7 @@ endif() # Set after the test of the CUDA compiler. Otherwise, the test fails with # -latomic not found because it is added right after the compiler exe. -if( - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" - AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15 -) +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_EQUAL 15) set(CMAKE_EXE_LINKER_FLAGS "-latomic") endif() @@ -729,7 +733,23 @@ if("${TREAT_WARNINGS_AS_ERRORS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") endif() +# Enable the ASAN and UBSAN sanitzers for Clang 20 and above. +# Also only eligible with the type debug build. +if( + VELOX_ENABLE_ASAN_UBSAN_SANITIZERS + AND CMAKE_BUILD_TYPE STREQUAL "Debug" + AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 20 +) + set( + CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -O1 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" + ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined") +endif() + message("FINAL CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") +message("FINAL CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS}") if(VELOX_ENABLE_ARROW) velox_set_source(Arrow) diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 54f6f94d697c..9dbd9c6f7ed0 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -56,7 +56,8 @@ function install_build_prerequisites { dnf config-manager --set-enabled crb dnf update -y fi - dnf_install autoconf automake ccache clang gcc-toolset-12 gcc-toolset-14 git libtool \ + dnf_install autoconf automake ccache clang compiler-rt \ + gcc-toolset-12 gcc-toolset-14 git libtool \ llvm ninja-build python3-pip python3-devel wget which install_uv diff --git a/velox/common/base/Macros.h b/velox/common/base/Macros.h index 62664a488f53..bc428397c029 100644 --- a/velox/common/base/Macros.h +++ b/velox/common/base/Macros.h @@ -28,6 +28,19 @@ _Pragma("GCC diagnostic pop"); #endif +#if defined(__clang__) && defined(__has_warning) && \ + __has_warning("-Wmissing-designated-field-initializers") +#define VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING \ + _Pragma("clang diagnostic push"); \ + _Pragma( \ + "clang diagnostic ignored \"-Wmissing-designated-field-initializers\"") +#define VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING \ + _Pragma("clang diagnostic pop"); +#else +#define VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING +#define VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING +#endif + #define VELOX_CONCAT(x, y) x##y // Need this extra layer to expand __COUNTER__. #define VELOX_VARNAME_IMPL(x, y) VELOX_CONCAT(x, y) diff --git a/velox/functions/prestosql/DateTimeFunctions.h b/velox/functions/prestosql/DateTimeFunctions.h index ada5351314c8..0b919b2eb040 100644 --- a/velox/functions/prestosql/DateTimeFunctions.h +++ b/velox/functions/prestosql/DateTimeFunctions.h @@ -1963,10 +1963,11 @@ struct ParseDurationFunction { FOLLY_ALWAYS_INLINE void call( out_type& result, const arg_type& amountUnit) { + VELOX_SUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING static const LazyRE2 kDurationRegex{ .pattern_ = R"(^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$)", - .options_ = {}, }; + VELOX_UNSUPPRESS_MISSING_DESIGNATED_FIELD_INITIALIZERS_WARNING // TODO: Remove re2::StringPiece != std::string_view hacks. // It's needed because for some systems in CI, // re2 and abseil libraries are old.