From a18d5ffbaebddd7041eb140d6b168a8fce7e6308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Thu, 8 Oct 2026 23:17:49 +0000 Subject: [PATCH 1/4] Inflection-210 Fix signed integer overflow in bitmask property extraction --- .../DictionaryMetaData_MMappedDictionary.cpp | 10 ++++++---- .../inflection/dictionary/DictionaryMetaDataTest-c.cpp | 9 +++++---- tools/buildDictionary/LexicalDictionaryBuilder.cpp | 7 ++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp index 81f535a63..2c69970da 100644 --- a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp +++ b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp @@ -98,11 +98,13 @@ std::optional<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypeOfV return {}; } -std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(int64_t value) const { +std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(int64_t value) const +{ ::std::vector<::std::u16string> result; - while (value != 0) { - const auto singleProperty = ((int64_t)(((uint64_t)(value ^ (value - 1))) >> 1)) + 1; - value &= value - 1; + auto uValue = static_cast(value); + while (uValue != 0) { + const auto singleProperty = static_cast(((uValue ^ (uValue - 1)) >> 1) + 1); + uValue &= uValue - 1; const auto name = getTypeOfValue(singleProperty); if (name.has_value()) { result.push_back(name.value()); diff --git a/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp b/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp index 6c69488ce..308fc89fe 100644 --- a/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp +++ b/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp @@ -19,10 +19,11 @@ static void checkForFailure(UErrorCode* status) std::vector convertToPropertyNames(const IDDictionaryMetaData* dictionary, int64_t value) { std::vector result; - char16_t name[64] = { }; - while (value != 0) { - const auto singleProperty = ((int64_t)(((uint64_t)(value ^ (value - 1))) >> 1)) + 1; - value &= value - 1; + char16_t name[64] = { }; + auto uValue = static_cast(value); + while (uValue != 0) { + const auto singleProperty = static_cast(((uValue ^ (uValue - 1)) >> 1) + 1); + uValue &= uValue - 1; auto error = U_ZERO_ERROR; int len = iddmd_getPropertyName(dictionary, singleProperty, name, std::size(name), &error); REQUIRE(U_SUCCESS(error)); diff --git a/tools/buildDictionary/LexicalDictionaryBuilder.cpp b/tools/buildDictionary/LexicalDictionaryBuilder.cpp index 9fd228b79..1bc018327 100644 --- a/tools/buildDictionary/LexicalDictionaryBuilder.cpp +++ b/tools/buildDictionary/LexicalDictionaryBuilder.cpp @@ -128,9 +128,10 @@ static ::std::map createBitsToContainerBitsMap( static int64_t mapValueToContainerBits(int64_t binaryProperties, std::map bitmap) { int64_t result = 0; - while (binaryProperties != 0) { - auto singleProperty = ((int64_t) (((uint64_t) (binaryProperties ^ (binaryProperties - 1))) >> 1)) + 1; - binaryProperties &= binaryProperties - 1; + auto uBinaryProperties = static_cast(binaryProperties); + while (uBinaryProperties != 0) { + auto singleProperty = static_cast(((uBinaryProperties ^ (uBinaryProperties - 1)) >> 1) + 1); + uBinaryProperties &= uBinaryProperties - 1; auto newBitItr = bitmap.find(singleProperty); if (newBitItr == bitmap.end()) { throw inflection::exception::IllegalArgumentException(u"bit not found in bitmap"); From 711d4a440b5122b0f13d37460d0505cfcd149cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Thu, 8 Oct 2026 23:37:06 +0000 Subject: [PATCH 2/4] Inflection-210 Use uint64_t parameter directly and enable UBSan in Debug builds --- CMakeLists.txt | 3 +++ .../dictionary/DictionaryMetaData_MMappedDictionary.cpp | 9 ++++----- .../dictionary/DictionaryMetaData_MMappedDictionary.hpp | 3 ++- .../inflection/dictionary/DictionaryMetaDataTest-c.cpp | 9 ++++----- tools/buildDictionary/LexicalDictionaryBuilder.cpp | 9 ++++----- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b15b24d20..18d44a2de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,9 @@ if("${CMAKE_BUILD_TYPE}" MATCHES "MinSizeRel" OR "${CMAKE_BUILD_TYPE}" MATCHES " # versioned LLVMgold.so can drift out of sync with the compiler and break LTO linking. add_link_options(-fuse-ld=lld) endif() +elseif("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC) + add_compile_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) + add_link_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) endif() if(ALIGNMENT_TEST) diff --git a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp index 2c69970da..384d8dd97 100644 --- a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp +++ b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp @@ -98,13 +98,12 @@ std::optional<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypeOfV return {}; } -std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(int64_t value) const +std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(uint64_t value) const { ::std::vector<::std::u16string> result; - auto uValue = static_cast(value); - while (uValue != 0) { - const auto singleProperty = static_cast(((uValue ^ (uValue - 1)) >> 1) + 1); - uValue &= uValue - 1; + while (value != 0) { + const auto singleProperty = ((value ^ (value - 1)) >> 1) + 1; + value &= value - 1; const auto name = getTypeOfValue(singleProperty); if (name.has_value()) { result.push_back(name.value()); diff --git a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.hpp b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.hpp index 31b185f06..9bfeed324 100644 --- a/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.hpp +++ b/src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.hpp @@ -29,7 +29,8 @@ class INFLECTION_INTERNAL_API inflection::dictionary::DictionaryMetaData_MMapped ::std::optional getValueOfType(std::u16string_view type) const; int64_t getValuesOfTypes(const std::vector &types) const; ::std::optional<::std::u16string> getTypeOfValue(int64_t value) const; - ::std::vector<::std::u16string> getTypesOfValues(int64_t value) const; + ::std::vector<::std::u16string> getTypesOfValues(uint64_t value) const; + private: void getPropertyMapInternalIdentifiers(std::vector &propertyIdentifiers, int32_t startingOffset, int32_t length) const; typedef enum { diff --git a/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp b/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp index 308fc89fe..a4eedebd3 100644 --- a/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp +++ b/test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp @@ -16,14 +16,13 @@ static void checkForFailure(UErrorCode* status) *status = U_ZERO_ERROR; } -std::vector convertToPropertyNames(const IDDictionaryMetaData* dictionary, int64_t value) +std::vector convertToPropertyNames(const IDDictionaryMetaData* dictionary, uint64_t value) { std::vector result; char16_t name[64] = { }; - auto uValue = static_cast(value); - while (uValue != 0) { - const auto singleProperty = static_cast(((uValue ^ (uValue - 1)) >> 1) + 1); - uValue &= uValue - 1; + while (value != 0) { + const auto singleProperty = ((value ^ (value - 1)) >> 1) + 1; + value &= value - 1; auto error = U_ZERO_ERROR; int len = iddmd_getPropertyName(dictionary, singleProperty, name, std::size(name), &error); REQUIRE(U_SUCCESS(error)); diff --git a/tools/buildDictionary/LexicalDictionaryBuilder.cpp b/tools/buildDictionary/LexicalDictionaryBuilder.cpp index 1bc018327..95fcb2046 100644 --- a/tools/buildDictionary/LexicalDictionaryBuilder.cpp +++ b/tools/buildDictionary/LexicalDictionaryBuilder.cpp @@ -125,13 +125,12 @@ static ::std::map createBitsToContainerBitsMap( return valueBitsToContainerBits; } -static int64_t mapValueToContainerBits(int64_t binaryProperties, std::map bitmap) +static int64_t mapValueToContainerBits(uint64_t binaryProperties, std::map bitmap) { int64_t result = 0; - auto uBinaryProperties = static_cast(binaryProperties); - while (uBinaryProperties != 0) { - auto singleProperty = static_cast(((uBinaryProperties ^ (uBinaryProperties - 1)) >> 1) + 1); - uBinaryProperties &= uBinaryProperties - 1; + while (binaryProperties != 0) { + auto singleProperty = ((binaryProperties ^ (binaryProperties - 1)) >> 1) + 1; + binaryProperties &= binaryProperties - 1; auto newBitItr = bitmap.find(singleProperty); if (newBitItr == bitmap.end()) { throw inflection::exception::IllegalArgumentException(u"bit not found in bitmap"); From 06e766f10f66eecacf75f2e8c0fbe3a1e47549ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Thu, 8 Oct 2026 23:53:54 +0000 Subject: [PATCH 3/4] Inflection-210 Document UBSan exclusions in CMakeLists.txt --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18d44a2de..b8008a380 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,9 @@ if("${CMAKE_BUILD_TYPE}" MATCHES "MinSizeRel" OR "${CMAKE_BUILD_TYPE}" MATCHES " add_link_options(-fuse-ld=lld) endif() elseif("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC) + # Enable UBSan in Debug builds. Exclude: + # - vptr: avoids false positives across shared library boundaries with -fvisibility=hidden. + # - nonnull-attribute: third-party marisa-trie (keyset.cc) calls std::memcpy(nullptr, ptr, 0) when reserving a zero-length key. add_compile_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) add_link_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) endif() From 68e1d47396f069cf5c492071a7ec8140df66fbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Fri, 9 Oct 2026 19:19:32 +0000 Subject: [PATCH 4/4] Inflection-210 Scope nonnull-attribute exclusion to Marisa CMakeLists.txt --- CMakeLists.txt | 7 ++----- ext/lib/Marisa/CMakeLists.txt | 5 +++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8008a380..790fd584b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,11 +91,8 @@ if("${CMAKE_BUILD_TYPE}" MATCHES "MinSizeRel" OR "${CMAKE_BUILD_TYPE}" MATCHES " add_link_options(-fuse-ld=lld) endif() elseif("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC) - # Enable UBSan in Debug builds. Exclude: - # - vptr: avoids false positives across shared library boundaries with -fvisibility=hidden. - # - nonnull-attribute: third-party marisa-trie (keyset.cc) calls std::memcpy(nullptr, ptr, 0) when reserving a zero-length key. - add_compile_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) - add_link_options(-fsanitize=undefined -fno-sanitize=vptr,nonnull-attribute -fno-sanitize-recover=undefined) + add_compile_options(-fsanitize=undefined -fno-sanitize-recover=undefined) + add_link_options(-fsanitize=undefined -fno-sanitize-recover=undefined) endif() if(ALIGNMENT_TEST) diff --git a/ext/lib/Marisa/CMakeLists.txt b/ext/lib/Marisa/CMakeLists.txt index 43b1cc795..778b97ec0 100644 --- a/ext/lib/Marisa/CMakeLists.txt +++ b/ext/lib/Marisa/CMakeLists.txt @@ -41,6 +41,11 @@ else() endif() # (DEFINED ENV{MARISA_ROOT}) +if("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC) + # marisa-trie (keyset.cc, grimoire/vector/vector.h) passes nullptr to std::memcpy with length 0. + set_source_files_properties(${MARISA_SOURCES} PROPERTIES COMPILE_OPTIONS "-fno-sanitize=nonnull-attribute") +endif() + if(XCODE) add_library(marisa_objs INTERFACE) target_sources(marisa_objs INTERFACE ${MARISA_SOURCES})