Skip to content

Inflection-210 Fix signed integer overflow in bitmask property extraction #211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nciric merged 4 commits into from
Oct 9, 2026
Merged

Inflection-210 Fix signed integer overflow in bitmask property extraction #211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a18d5ff
Inflection-210 Fix signed integer overflow in bitmask property extrac…
nciric Oct 8, 2026
711d4a4
Inflection-210 Use uint64_t parameter directly and enable UBSan in De…
nciric Oct 8, 2026
06e766f
Inflection-210 Document UBSan exclusions in CMakeLists.txt
nciric Oct 8, 2026
68e1d47
Inflection-210 Scope nonnull-attribute exclusion to Marisa CMakeLists…
nciric Oct 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ if("${CMAKE_BUILD_TYPE}" MATCHES "MinSizeRel" OR "${CMAKE_BUILD_TYPE}" MATCHES "
# versioned LLVMgold.so can drift out of sync with the compiler and break LTO linking.
add_link_options(-fuse-ld=lld)
endif()
elseif("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC)
add_compile_options(-fsanitize=undefined -fno-sanitize-recover=undefined)
add_link_options(-fsanitize=undefined -fno-sanitize-recover=undefined)
endif()

if(ALIGNMENT_TEST)
Expand Down
5 changes: 5 additions & 0 deletions ext/lib/Marisa/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ else()
endif() # (DEFINED ENV{MARISA_ROOT})


if("${CMAKE_BUILD_TYPE}" MATCHES "Debug" AND NOT MSVC)
# marisa-trie (keyset.cc, grimoire/vector/vector.h) passes nullptr to std::memcpy with length 0.
set_source_files_properties(${MARISA_SOURCES} PROPERTIES COMPILE_OPTIONS "-fno-sanitize=nonnull-attribute")
endif()

if(XCODE)
add_library(marisa_objs INTERFACE)
target_sources(marisa_objs INTERFACE ${MARISA_SOURCES})
Expand Down
5 changes: 3 additions & 2 deletions src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,11 @@ std::optional<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypeOfV
return {};
}

std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(int64_t value) const {
std::vector<::std::u16string> DictionaryMetaData_MMappedDictionary::getTypesOfValues(uint64_t value) const
{
::std::vector<::std::u16string> result;
while (value != 0) {
const auto singleProperty = ((int64_t)(((uint64_t)(value ^ (value - 1))) >> 1)) + 1;
const auto singleProperty = ((value ^ (value - 1)) >> 1) + 1;
value &= value - 1;
const auto name = getTypeOfValue(singleProperty);
if (name.has_value()) {
Expand Down
3 changes: 2 additions & 1 deletion src/inflection/dictionary/DictionaryMetaData_MMappedDictionary.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ class INFLECTION_INTERNAL_API inflection::dictionary::DictionaryMetaData_MMapped
::std::optional<int64_t> getValueOfType(std::u16string_view type) const;
int64_t getValuesOfTypes(const std::vector<std::u16string> &types) const;
::std::optional<::std::u16string> getTypeOfValue(int64_t value) const;
::std::vector<::std::u16string> getTypesOfValues(int64_t value) const;
::std::vector<::std::u16string> getTypesOfValues(uint64_t value) const;

private:
void getPropertyMapInternalIdentifiers(std::vector<int32_t> &propertyIdentifiers, int32_t startingOffset, int32_t length) const;
typedef enum {
Expand Down
6 changes: 3 additions & 3 deletions test/src/inflection/dictionary/DictionaryMetaDataTest-c.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ static void checkForFailure(UErrorCode* status)
*status = U_ZERO_ERROR;
}

std::vector<std::u16string> convertToPropertyNames(const IDDictionaryMetaData* dictionary, int64_t value)
std::vector<std::u16string> convertToPropertyNames(const IDDictionaryMetaData* dictionary, uint64_t value)
{
std::vector<std::u16string> result;
char16_t name[64] = { };
char16_t name[64] = { };
while (value != 0) {
const auto singleProperty = ((int64_t)(((uint64_t)(value ^ (value - 1))) >> 1)) + 1;
const auto singleProperty = ((value ^ (value - 1)) >> 1) + 1;
value &= value - 1;
auto error = U_ZERO_ERROR;
int len = iddmd_getPropertyName(dictionary, singleProperty, name, std::size(name), &error);
Expand Down
4 changes: 2 additions & 2 deletions tools/buildDictionary/LexicalDictionaryBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,11 @@ static ::std::map<int64_t, int64_t> createBitsToContainerBitsMap(
return valueBitsToContainerBits;
}

static int64_t mapValueToContainerBits(int64_t binaryProperties, std::map<int64_t, int64_t> bitmap)
static int64_t mapValueToContainerBits(uint64_t binaryProperties, std::map<int64_t, int64_t> bitmap)
{
int64_t result = 0;
while (binaryProperties != 0) {
auto singleProperty = ((int64_t) (((uint64_t) (binaryProperties ^ (binaryProperties - 1))) >> 1)) + 1;
auto singleProperty = ((binaryProperties ^ (binaryProperties - 1)) >> 1) + 1;
binaryProperties &= binaryProperties - 1;
auto newBitItr = bitmap.find(singleProperty);
if (newBitItr == bitmap.end()) {
Expand Down
Loading