From 14499688496f523f3802d398640db73869cda950 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Mon, 31 Aug 2026 13:25:40 -0400 Subject: [PATCH 1/5] Add python/cython interface for unicode-normalizer APIs --- python/cudf/cudf/core/unicode_normalizer.py | 109 +++++++++ .../cudf/cudf/tests/text/test_text_methods.py | 38 +++ .../pylibcudf/libcudf/nvtext/CMakeLists.txt | 4 +- .../libcudf/nvtext/unicode_normalize.pxd | 36 +++ .../libcudf/nvtext/unicode_normalize.pyx | 2 + .../pylibcudf/pylibcudf/nvtext/CMakeLists.txt | 3 +- python/pylibcudf/pylibcudf/nvtext/__init__.py | 4 +- .../pylibcudf/nvtext/unicode_normalize.pxd | 21 ++ .../pylibcudf/nvtext/unicode_normalize.pyi | 32 +++ .../pylibcudf/nvtext/unicode_normalize.pyx | 116 +++++++++ .../tests/test_nvtext_unicode_normalize.py | 228 ++++++++++++++++++ 11 files changed, 589 insertions(+), 4 deletions(-) create mode 100644 python/cudf/cudf/core/unicode_normalizer.py create mode 100644 python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd create mode 100644 python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx create mode 100644 python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd create mode 100644 python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi create mode 100644 python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx create mode 100644 python/pylibcudf/tests/test_nvtext_unicode_normalize.py diff --git a/python/cudf/cudf/core/unicode_normalizer.py b/python/cudf/cudf/core/unicode_normalizer.py new file mode 100644 index 000000000000..027109d98dc4 --- /dev/null +++ b/python/cudf/cudf/core/unicode_normalizer.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unicode TR15 normalization support for cudf strings columns.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pylibcudf as plc + +from cudf.core.column.column import ColumnBase +from cudf.core.series import Series + +if TYPE_CHECKING: + from cudf.core.dataframe import DataFrame + + +class UnicodeNormalizer: + """ + A normalizer object for Unicode TR15 normalization (NFD, NFC, NFKD, NFKC). + + The normalizer is constructed from the contents of the Unicode Character + Database ``UnicodeData.txt`` file, which must be loaded by the caller as a + :class:`cudf.DataFrame` (e.g. via :func:`cudf.read_csv`). + + The file is published by the Unicode Consortium and can be downloaded from: + https://unicode.org/Public/15.1.0/ucd/UnicodeData.txt + + It is a semicolon-delimited file with 15 fields per row. The three fields + required by this API are field 0 (code point hex), field 3 (CCC), and + field 5 (decomposition mapping). Load only those three columns: + + .. code-block:: python + + unicode_data = cudf.read_csv( + "UnicodeData.txt", + sep=";", + header=None, + usecols=[0, 3, 5], + dtype={0: "str", 3: "int32", 5: "str"}, + ) + normalizer = UnicodeNormalizer(unicode_data, form="NFKC") + result = normalizer.normalize(series) + + The ``unicode_data`` DataFrame must contain exactly three columns in the + following order: + + - column[0]: ``str`` Code point values as uppercase hex strings (e.g. ``"00C9"``) + - column[1]: ``int32`` Canonical_Combining_Class (CCC) values in range [0, 254] + - column[2]: ``str`` Decomposition_Mapping field; empty string for identity + mappings, optionally prefixed with a compatibility tag such as + ````, ````, ````, etc. + + Decomposition of Hangul syllables (U+AC00..U+D7A3) is performed + algorithmically per the Unicode standard and does not require entries in + the provided table. + + Parameters + ---------- + unicode_data : cudf.DataFrame + Three-column DataFrame loaded from ``UnicodeData.txt`` as described above. + form : str + Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``, + or ``"NFKC"``. + """ + + _FORM_MAP = { + "NFD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFD, + "NFC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFC, + "NFKD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKD, + "NFKC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKC, + } + + def __init__( + self, + unicode_data: DataFrame, + form: str = "NFC", + ) -> None: + if form not in self._FORM_MAP: + raise ValueError( + f"Invalid normalization form {form!r}. " + f"Expected one of: {list(self._FORM_MAP)}" + ) + tbl = plc.Table.from_arrow(unicode_data.to_arrow()) + self._normalizer = plc.nvtext.unicode_normalize.UnicodeNormalizer( + tbl, self._FORM_MAP[form] + ) + + def normalize(self, text: Series) -> Series: + """ + Normalize a strings Series using Unicode TR15 normalization. + + Parameters + ---------- + text : cudf.Series + The UTF-8 strings to normalize. Null entries produce null output. + + Returns + ------- + cudf.Series + New Series of normalized UTF-8 strings. + """ + plc_column = plc.nvtext.unicode_normalize.normalize_unicode( + text._column.plc_column, self._normalizer + ) + return Series._from_column( + ColumnBase.create(plc_column, text._column.dtype) + ) diff --git a/python/cudf/cudf/tests/text/test_text_methods.py b/python/cudf/cudf/tests/text/test_text_methods.py index 047c27269cbf..e9f01894713a 100644 --- a/python/cudf/cudf/tests/text/test_text_methods.py +++ b/python/cudf/cudf/tests/text/test_text_methods.py @@ -10,6 +10,7 @@ import cudf from cudf.core.character_normalizer import CharacterNormalizer from cudf.core.tokenize_vocabulary import TokenizeVocabulary +from cudf.core.unicode_normalizer import UnicodeNormalizer from cudf.testing import assert_eq @@ -274,6 +275,43 @@ def test_normalize_characters(): assert_eq(expected, actual) +def test_unicode_normalize(): + # Minimal unicode_data DataFrame: é (U+00E9) and combining acute (U+0301), + # plus fi ligature (U+FB01) with compatibility-only decomposition. + unicode_data = cudf.DataFrame( + { + "cp": cudf.Series(["00E9", "0301", "FB01"]), + "ccc": cudf.Series([0, 230, 0], dtype="int32"), + "decomp": cudf.Series(["0065 0301", "", " 0066 0069"]), + } + ) + + # NFC: decomposed e + combining acute → precomposed é; fi unchanged + nfc = UnicodeNormalizer(unicode_data, form="NFC") + assert_eq( + nfc.normalize(cudf.Series(["é", "café", "fi", None])), + cudf.Series(["é", "café", "fi", None]), + ) + + # NFD: precomposed é (U+00E9) → e (U+0065) + combining acute (U+0301) + nfd = UnicodeNormalizer(unicode_data, form="NFD") + assert_eq( + nfd.normalize(cudf.Series(["é"])), + cudf.Series(["é"]), + ) + + # NFKC: fi ligature expands to "fi" + nfkc = UnicodeNormalizer(unicode_data, form="NFKC") + assert_eq( + nfkc.normalize(cudf.Series(["fi", "café"])), + cudf.Series(["fi", "café"]), + ) + + # invalid form raises ValueError + with pytest.raises(ValueError, match="Invalid normalization form"): + UnicodeNormalizer(unicode_data, form="XYZ") + + @pytest.mark.parametrize( "n, separator, expected_values", [ diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt index 0a999dc704a6..cfa53955cc32 100644 --- a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt @@ -1,11 +1,11 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= -set(cython_sources stemmer.pyx) +set(cython_sources stemmer.pyx unicode_normalize.pyx) set(linked_libraries cudf::cudf) diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd new file mode 100644 index 000000000000..8fdbd8b376a8 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from libc.stdint cimport int32_t +from libcpp.memory cimport unique_ptr +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from cuda.bindings.cyruntime cimport cudaStream_t +from rmm.librmm.memory_resource cimport device_async_resource_ref + + +cdef extern from "nvtext/unicode_normalize.hpp" namespace "nvtext" nogil: + + cpdef enum class unicode_normalization_form(int32_t): + NFD + NFC + NFKD + NFKC + + cdef struct unicode_normalizer: + pass + + cdef unique_ptr[unicode_normalizer] create_unicode_normalizer( + const table_view &unicode_data, + unicode_normalization_form form, + cudaStream_t stream, + device_async_resource_ref mr + ) except +libcudf_exception_handler + + cdef unique_ptr[column] normalize_unicode( + const column_view &input, + const unicode_normalizer &normalizer, + cudaStream_t stream, + device_async_resource_ref mr + ) except +libcudf_exception_handler diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx new file mode 100644 index 000000000000..8eca3cc68ad4 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 diff --git a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt index fd5fae932bb9..c0227fbf98b2 100644 --- a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -17,6 +17,7 @@ set(cython_sources replace.pyx stemmer.pyx tokenize.pyx + unicode_normalize.pyx wordpiece_tokenize.pyx ) diff --git a/python/pylibcudf/pylibcudf/nvtext/__init__.py b/python/pylibcudf/pylibcudf/nvtext/__init__.py index bf433f5ccb48..816f0e59e93e 100644 --- a/python/pylibcudf/pylibcudf/nvtext/__init__.py +++ b/python/pylibcudf/pylibcudf/nvtext/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from . import ( @@ -13,6 +13,7 @@ replace, stemmer, tokenize, + unicode_normalize, wordpiece_tokenize, ) @@ -28,5 +29,6 @@ "replace", "stemmer", "tokenize", + "unicode_normalize", "wordpiece_tokenize", ] diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd new file mode 100644 index 000000000000..46e92bac088a --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libcpp.memory cimport unique_ptr +from pylibcudf.column cimport Column +from pylibcudf.libcudf.nvtext.unicode_normalize cimport ( + unicode_normalizer as cpp_unicode_normalizer, + unicode_normalization_form as cpp_unicode_normalization_form, +) +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource + + +cdef class UnicodeNormalizer: + cdef unique_ptr[cpp_unicode_normalizer] c_obj + +cpdef Column normalize_unicode( + Column input, + UnicodeNormalizer normalizer, + object stream=*, + DeviceMemoryResource mr=*, +) diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi new file mode 100644 index 000000000000..6d63d3b9ac7a --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from enum import IntEnum + +from rmm.pylibrmm.memory_resource import DeviceMemoryResource + +from pylibcudf.column import Column +from pylibcudf.table import Table +from pylibcudf.typing import CudaStreamLike + +class UnicodeNormalizationForm(IntEnum): + NFD = ... + NFC = ... + NFKD = ... + NFKC = ... + +class UnicodeNormalizer: + def __init__( + self, + unicode_data: Table, + form: UnicodeNormalizationForm, + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, + ): ... + +def normalize_unicode( + input: Column, + normalizer: UnicodeNormalizer, + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, +) -> Column: ... diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx new file mode 100644 index 000000000000..e9993bf33674 --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.nvtext.unicode_normalize cimport ( + create_unicode_normalizer as cpp_create_unicode_normalizer, + normalize_unicode as cpp_normalize_unicode, + unicode_normalization_form as cpp_unicode_normalization_form, +) +from pylibcudf.libcudf.nvtext.unicode_normalize import \ + unicode_normalization_form as UnicodeNormalizationForm # no-cython-lint +from pylibcudf.table cimport Table +from pylibcudf.utils cimport _get_stream, _get_memory_resource +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pylibcudf.typing import CudaStreamLike +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource +from rmm.pylibrmm.stream cimport Stream +from cuda.bindings.cyruntime cimport cudaStream_t + +__all__ = ["UnicodeNormalizer", "UnicodeNormalizationForm", "normalize_unicode"] + + +cdef class UnicodeNormalizer: + """Normalizer object for Unicode TR15 normalization (NFD/NFC/NFKD/NFKC). + + Constructed from the three relevant columns of UnicodeData.txt loaded as a + :class:`pylibcudf.Table`. Once built the object can be reused across + multiple calls to :func:`normalize_unicode`. + + For details, see :cpp:class:`nvtext::unicode_normalizer`. + + Parameters + ---------- + unicode_data : Table + Three-column table parsed from UnicodeData.txt: + column[0] STRING code-point hex strings (e.g. "00C9"), + column[1] INT32 Canonical_Combining_Class values, + column[2] STRING Decomposition_Mapping field. + form : unicode_normalization_form + Normalization form to apply (NFD, NFC, NFKD, or NFKC). + stream : CudaStreamLike | None + CUDA stream on which to perform the operation. + mr : DeviceMemoryResource | None + Device memory resource for internal table allocations. + """ + def __cinit__( + self, + Table unicode_data, + cpp_unicode_normalization_form form, + object stream: CudaStreamLike | None = None, + DeviceMemoryResource mr=None, + ): + cdef table_view c_data = unicode_data.view() + cdef Stream _stream = _get_stream(stream) + cdef cudaStream_t _cs = _stream.view().value() + mr = _get_memory_resource(mr) + with nogil: + self.c_obj = move( + cpp_create_unicode_normalizer(c_data, form, _cs, mr.get_mr()) + ) + + __hash__ = None + + +cpdef Column normalize_unicode( + Column input, + UnicodeNormalizer normalizer, + object stream: CudaStreamLike | None = None, + DeviceMemoryResource mr=None, +): + """Normalize a strings column using Unicode TR15 normalization. + + Input and output are UTF-8 encoded. Each string is normalized + independently; null entries produce null output entries. + + For details, see :cpp:func:`nvtext::normalize_unicode`. + + Parameters + ---------- + input : Column + Strings column to normalize. + normalizer : UnicodeNormalizer + Normalizer built by :class:`UnicodeNormalizer`. + stream : CudaStreamLike | None + CUDA stream on which to perform the operation. + mr : DeviceMemoryResource | None + Device memory resource for the returned column. + + Returns + ------- + Column + New strings column of normalized UTF-8 strings. + """ + cdef unique_ptr[column] c_result + cdef Stream _stream = _get_stream(stream) + cdef cudaStream_t _cs = _stream.view().value() + mr = _get_memory_resource(mr) + + cdef column_view c_input = input.view() + with nogil: + c_result = cpp_normalize_unicode( + c_input, + dereference(normalizer.c_obj.get()), + _cs, + mr.get_mr(), + ) + + return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py new file mode 100644 index 000000000000..367d61754a39 --- /dev/null +++ b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unicodedata as ud + +import pyarrow as pa +import pytest +from utils import assert_column_eq + +import pylibcudf as plc +from pylibcudf.nvtext.unicode_normalize import ( + UnicodeNormalizationForm, + UnicodeNormalizer, + normalize_unicode, +) + + +# --------------------------------------------------------------------------- +# Helper to build a minimal unicode_data Table from explicit row data. +# Each row is (cp_hex: str, ccc: int, decomp: str). +# --------------------------------------------------------------------------- +def _make_unicode_table(rows): + cp_col = pa.array([r[0] for r in rows], type=pa.string()) + ccc_col = pa.array([r[1] for r in rows], type=pa.int32()) + decomp_col = pa.array([r[2] for r in rows], type=pa.string()) + tbl = pa.table({"cp": cp_col, "ccc": ccc_col, "decomp": decomp_col}) + return plc.Table.from_arrow(tbl) + + +# Minimal rows that cover NFD/NFC tests: é = U+00E9 → e (U+0065) + U+0301 +_MINIMAL_ROWS = [ + ("00E9", 0, "0065 0301"), # é, canonical decomposition + ("0301", 230, ""), # combining acute accent, CCC=230 +] + +# Rows needed for the NFKC fi test (U+FB01 ligature) +_COMPAT_ROWS = [ + *_MINIMAL_ROWS, + ("FB01", 0, " 0066 0069"), +] # fi ligature, compatibility only + + +@pytest.fixture(scope="module") +def nfc_normalizer(): + tbl = _make_unicode_table(_MINIMAL_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFC) + + +@pytest.fixture(scope="module") +def nfd_normalizer(): + tbl = _make_unicode_table(_MINIMAL_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFD) + + +@pytest.fixture(scope="module") +def nfkc_normalizer(): + tbl = _make_unicode_table(_COMPAT_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFKC) + + +@pytest.fixture(scope="module") +def nfkd_normalizer(): + tbl = _make_unicode_table(_COMPAT_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFKD) + + +# --------------------------------------------------------------------------- +# Basic correctness tests +# --------------------------------------------------------------------------- + + +def test_null_strings(nfc_normalizer): + input_col = plc.Column.from_arrow( + pa.array([None, None, None], type=pa.string()) + ) + result = normalize_unicode(input_col, nfc_normalizer) + expected = pa.array([None, None, None], type=pa.string()) + assert_column_eq(expected, result) + + +def test_ascii_passthrough(nfc_normalizer): + arr = pa.array(["hello", "world", "abc 123", ""]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_nfc_compose(nfc_normalizer): + # "é" (e + combining acute) should compose to U+00E9 (é) + arr = pa.array(["é", "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + expected = pa.array(["é", "café"]) + assert_column_eq(expected, result) + + +def test_nfd_decompose(nfd_normalizer): + # U+00E9 (é) should decompose to "é" + arr = pa.array(["é"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfd_normalizer) + expected = pa.array(["é"]) + assert_column_eq(expected, result) + + +def test_nfc_stable(nfc_normalizer): + # Already-composed strings should be unchanged by NFC + arr = pa.array(["é", "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_nfkc_compat_ligature(nfkc_normalizer): + # U+FB01 (fi) has only a compatibility decomposition; NFKC expands it to "fi" + arr = pa.array(["fi"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfkc_normalizer) + expected = pa.array(["fi"]) + assert_column_eq(expected, result) + + +def test_nfc_compat_ligature_stable(nfc_normalizer): + # U+FB01 is NFC-stable: NFC must leave it unchanged + arr = pa.array(["fi"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_mixed_nulls(nfc_normalizer): + # Null rows must not corrupt adjacent non-null rows + arr = pa.array(["é", None, "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + expected = pa.array(["é", None, "café"]) + assert_column_eq(expected, result) + + +# --------------------------------------------------------------------------- +# Comparison against Python's unicodedata.normalize +# --------------------------------------------------------------------------- +# Build a moderately complete unicode_data table from Python's unicodedata +# module, covering BMP codepoints (U+0080..U+FFFF) that have a non-trivial +# CCC or decomposition mapping. This mirrors what a user would load from +# UnicodeData.txt and lets us cross-check the GPU result against the +# reference implementation. + + +def _build_bmp_unicode_table(): + """Return a pylibcudf Table covering non-trivial BMP codepoints.""" + cp_list, ccc_list, decomp_list = [], [], [] + for cp in range(0x0080, 0x10000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + tbl = pa.table( + { + "cp": pa.array(cp_list, type=pa.string()), + "ccc": pa.array(ccc_list, type=pa.int32()), + "decomp": pa.array(decomp_list, type=pa.string()), + } + ) + return plc.Table.from_arrow(tbl) + + +@pytest.fixture(scope="module") +def bmp_nfc_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFC + ) + + +@pytest.fixture(scope="module") +def bmp_nfkc_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFKC + ) + + +@pytest.fixture(scope="module") +def bmp_nfd_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFD + ) + + +@pytest.fixture(scope="module") +def bmp_nfkd_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFKD + ) + + +# Test strings exercising a range of normalization scenarios +_COMPARISON_STRINGS = [ + "hello world", # pure ASCII + "café", # precomposed é + "café", # decomposed e + combining acute + "élève", # é, è precomposed + "ẛ̣", # ẛ + combining dot below (reordering needed) + "fi fl", # fi fl ligatures (compat) + "Ω", # Ω (ohm sign, compat with U+03A9) + "½", # ½ vulgar fraction (compat) + "　", # ideographic space (compat) + "가", # 가 Hangul syllable (algorithmic) + "힣", # 힣 last Hangul syllable + "", # empty string +] + + +@pytest.mark.parametrize( + "form,fixture_name", + [ + ("NFC", "bmp_nfc_normalizer"), + ("NFKC", "bmp_nfkc_normalizer"), + ("NFD", "bmp_nfd_normalizer"), + ("NFKD", "bmp_nfkd_normalizer"), + ], +) +def test_compare_with_python_unicodedata(form, fixture_name, request): + """GPU normalization must match Python's unicodedata.normalize reference.""" + normalizer = request.getfixturevalue(fixture_name) + input_arr = pa.array(_COMPARISON_STRINGS, type=pa.string()) + gpu_result = normalize_unicode( + plc.Column.from_arrow(input_arr), normalizer + ) + expected = pa.array( + [ud.normalize(form, s) for s in _COMPARISON_STRINGS], type=pa.string() + ) + assert_column_eq(expected, gpu_result) From e1a8c5bf186bf2ecbfb21c78cc8de37d5fd11b87 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Mon, 31 Aug 2026 13:49:21 -0400 Subject: [PATCH 2/5] fix cython semantics per AI review --- python/cudf/cudf/tests/text/test_text_methods.py | 2 +- .../pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx | 12 +++++++----- .../pylibcudf/tests/test_nvtext_unicode_normalize.py | 6 ++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/tests/text/test_text_methods.py b/python/cudf/cudf/tests/text/test_text_methods.py index e9f01894713a..f87ee200cf04 100644 --- a/python/cudf/cudf/tests/text/test_text_methods.py +++ b/python/cudf/cudf/tests/text/test_text_methods.py @@ -289,7 +289,7 @@ def test_unicode_normalize(): # NFC: decomposed e + combining acute → precomposed é; fi unchanged nfc = UnicodeNormalizer(unicode_data, form="NFC") assert_eq( - nfc.normalize(cudf.Series(["é", "café", "fi", None])), + nfc.normalize(cudf.Series(["é", "café", "fi", None])), cudf.Series(["é", "café", "fi", None]), ) diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx index e9993bf33674..3cc4b0bf2705 100644 --- a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx @@ -61,10 +61,10 @@ cdef class UnicodeNormalizer: cdef table_view c_data = unicode_data.view() cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() - mr = _get_memory_resource(mr) + cdef DeviceMemoryResource _mr = _get_memory_resource(mr) with nogil: self.c_obj = move( - cpp_create_unicode_normalizer(c_data, form, _cs, mr.get_mr()) + cpp_create_unicode_normalizer(c_data, form, _cs, _mr.get_mr()) ) __hash__ = None @@ -99,10 +99,12 @@ cpdef Column normalize_unicode( Column New strings column of normalized UTF-8 strings. """ + if normalizer is None: + raise TypeError("normalizer must not be None") cdef unique_ptr[column] c_result cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() - mr = _get_memory_resource(mr) + cdef DeviceMemoryResource _mr = _get_memory_resource(mr) cdef column_view c_input = input.view() with nogil: @@ -110,7 +112,7 @@ cpdef Column normalize_unicode( c_input, dereference(normalizer.c_obj.get()), _cs, - mr.get_mr(), + _mr.get_mr(), ) - return Column.from_libcudf(move(c_result), _stream, mr) + return Column.from_libcudf(move(c_result), _stream, _mr) diff --git a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py index 367d61754a39..df78f8f0b86e 100644 --- a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py +++ b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py @@ -78,6 +78,12 @@ def test_null_strings(nfc_normalizer): assert_column_eq(expected, result) +def test_empty_column(nfc_normalizer): + arr = pa.array([], type=pa.string()) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + def test_ascii_passthrough(nfc_normalizer): arr = pa.array(["hello", "world", "abc 123", ""]) result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) From ff7fe703700faf895a56932769062cdd764a28fe Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 2 Sep 2026 10:17:23 -0400 Subject: [PATCH 3/5] add from_python_unicodedata classmethod --- python/cudf/cudf/core/unicode_normalizer.py | 65 +++++++++++++++++++ .../cudf/cudf/tests/text/test_text_methods.py | 14 ++++ 2 files changed, 79 insertions(+) diff --git a/python/cudf/cudf/core/unicode_normalizer.py b/python/cudf/cudf/core/unicode_normalizer.py index 027109d98dc4..b6f52d998a14 100644 --- a/python/cudf/cudf/core/unicode_normalizer.py +++ b/python/cudf/cudf/core/unicode_normalizer.py @@ -87,6 +87,71 @@ def __init__( tbl, self._FORM_MAP[form] ) + @classmethod + def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer: + """ + Construct a :class:`UnicodeNormalizer` from Python's built-in + ``unicodedata`` module, without requiring a ``UnicodeData.txt`` file. + + This is equivalent to downloading ``UnicodeData.txt``, loading it with + :func:`cudf.read_csv`, and passing the result to + :class:`UnicodeNormalizer`, but derives the same three columns directly + from :mod:`unicodedata`: + + .. code-block:: python + + import unicodedata as ud + + cp_list, ccc_list, decomp_list = [], [], [] + for cp in range(0x0080, 0x110000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + + The Unicode version used is whichever version is bundled with the + running Python interpreter (see ``unicodedata.unidata_version``), which + may differ from a specific ``UnicodeData.txt`` download. + + Parameters + ---------- + form : str + Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``, + or ``"NFKC"``. + + Returns + ------- + UnicodeNormalizer + Ready-to-use normalizer for the requested form. + """ + import unicodedata as ud + + import cudf + + cp_list: list[str] = [] + ccc_list: list[int] = [] + decomp_list: list[str] = [] + for cp in range(0x0080, 0x110000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + + unicode_data = cudf.DataFrame( + { + "cp": cudf.Series(cp_list), + "ccc": cudf.Series(ccc_list, dtype="int32"), + "decomp": cudf.Series(decomp_list), + } + ) + return cls(unicode_data, form=form) + def normalize(self, text: Series) -> Series: """ Normalize a strings Series using Unicode TR15 normalization. diff --git a/python/cudf/cudf/tests/text/test_text_methods.py b/python/cudf/cudf/tests/text/test_text_methods.py index f87ee200cf04..e0a787380c78 100644 --- a/python/cudf/cudf/tests/text/test_text_methods.py +++ b/python/cudf/cudf/tests/text/test_text_methods.py @@ -312,6 +312,20 @@ def test_unicode_normalize(): UnicodeNormalizer(unicode_data, form="XYZ") +def test_unicode_normalize_from_python_unicodedata(): + import unicodedata as ud + + strings = ["café", "fi", "½", "가", "", None] + series = cudf.Series(strings) + for form in ("NFC", "NFD", "NFKC", "NFKD"): + normalizer = UnicodeNormalizer.from_python_unicodedata(form=form) + result = normalizer.normalize(series) + expected = cudf.Series( + [ud.normalize(form, s) if s is not None else None for s in strings] + ) + assert_eq(result, expected) + + @pytest.mark.parametrize( "n, separator, expected_values", [ From face831fe7ba4922086e29e07d80ffcd3be5a8b3 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 2 Sep 2026 10:29:26 -0400 Subject: [PATCH 4/5] add form parameter check and pytest --- python/cudf/cudf/core/unicode_normalizer.py | 10 +++++++++- python/cudf/cudf/tests/text/test_text_methods.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/unicode_normalizer.py b/python/cudf/cudf/core/unicode_normalizer.py index b6f52d998a14..d29bc5908862 100644 --- a/python/cudf/cudf/core/unicode_normalizer.py +++ b/python/cudf/cudf/core/unicode_normalizer.py @@ -127,6 +127,12 @@ def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer: UnicodeNormalizer Ready-to-use normalizer for the requested form. """ + if form not in cls._FORM_MAP: + raise ValueError( + f"Invalid normalization form {form!r}. " + f"Expected one of: {list(cls._FORM_MAP)}" + ) + import unicodedata as ud import cudf @@ -170,5 +176,7 @@ def normalize(self, text: Series) -> Series: text._column.plc_column, self._normalizer ) return Series._from_column( - ColumnBase.create(plc_column, text._column.dtype) + ColumnBase.create(plc_column, text._column.dtype), + name=text.name, + index=text.index, ) diff --git a/python/cudf/cudf/tests/text/test_text_methods.py b/python/cudf/cudf/tests/text/test_text_methods.py index e0a787380c78..273605bfccf9 100644 --- a/python/cudf/cudf/tests/text/test_text_methods.py +++ b/python/cudf/cudf/tests/text/test_text_methods.py @@ -307,6 +307,13 @@ def test_unicode_normalize(): cudf.Series(["fi", "café"]), ) + # result preserves index and name from input + nfc = UnicodeNormalizer(unicode_data, form="NFC") + s = cudf.Series(["é"], index=[42], name="my_col") + result = nfc.normalize(s) + assert result.name == "my_col" + assert result.index.to_arrow().to_pylist() == [42] + # invalid form raises ValueError with pytest.raises(ValueError, match="Invalid normalization form"): UnicodeNormalizer(unicode_data, form="XYZ") @@ -317,6 +324,7 @@ def test_unicode_normalize_from_python_unicodedata(): strings = ["café", "fi", "½", "가", "", None] series = cudf.Series(strings) + all_null = cudf.Series([None, None, None], dtype="object") for form in ("NFC", "NFD", "NFKC", "NFKD"): normalizer = UnicodeNormalizer.from_python_unicodedata(form=form) result = normalizer.normalize(series) @@ -324,6 +332,13 @@ def test_unicode_normalize_from_python_unicodedata(): [ud.normalize(form, s) if s is not None else None for s in strings] ) assert_eq(result, expected) + null_result = normalizer.normalize(all_null) + assert_eq(null_result, all_null) + assert null_result.dtype == all_null.dtype + + # invalid form must raise before any codepoint scanning occurs + with pytest.raises(ValueError, match="Invalid normalization form"): + UnicodeNormalizer.from_python_unicodedata(form="XYZ") @pytest.mark.parametrize( From 9014db230ee13d6f6f6fc3f12ae7c52a89cd28de Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 2 Sep 2026 14:10:57 -0400 Subject: [PATCH 5/5] fix imports; add pytest.fixture --- python/cudf/cudf/core/unicode_normalizer.py | 20 ++++------ .../tests/test_nvtext_unicode_normalize.py | 40 ++++++++++--------- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/python/cudf/cudf/core/unicode_normalizer.py b/python/cudf/cudf/core/unicode_normalizer.py index d29bc5908862..cae08dbaf46d 100644 --- a/python/cudf/cudf/core/unicode_normalizer.py +++ b/python/cudf/cudf/core/unicode_normalizer.py @@ -5,16 +5,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import unicodedata as ud import pylibcudf as plc from cudf.core.column.column import ColumnBase +from cudf.core.dataframe import DataFrame from cudf.core.series import Series -if TYPE_CHECKING: - from cudf.core.dataframe import DataFrame - class UnicodeNormalizer: """ @@ -82,7 +80,7 @@ def __init__( f"Invalid normalization form {form!r}. " f"Expected one of: {list(self._FORM_MAP)}" ) - tbl = plc.Table.from_arrow(unicode_data.to_arrow()) + tbl, _ = unicode_data.to_pylibcudf() self._normalizer = plc.nvtext.unicode_normalize.UnicodeNormalizer( tbl, self._FORM_MAP[form] ) @@ -133,10 +131,6 @@ def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer: f"Expected one of: {list(cls._FORM_MAP)}" ) - import unicodedata as ud - - import cudf - cp_list: list[str] = [] ccc_list: list[int] = [] decomp_list: list[str] = [] @@ -149,11 +143,11 @@ def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer: ccc_list.append(ccc) decomp_list.append(decomp) - unicode_data = cudf.DataFrame( + unicode_data = DataFrame( { - "cp": cudf.Series(cp_list), - "ccc": cudf.Series(ccc_list, dtype="int32"), - "decomp": cudf.Series(decomp_list), + "cp": Series(cp_list), + "ccc": Series(ccc_list, dtype="int32"), + "decomp": Series(decomp_list), } ) return cls(unicode_data, form=form) diff --git a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py index df78f8f0b86e..f4422913c049 100644 --- a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py +++ b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py @@ -195,21 +195,23 @@ def bmp_nfkd_normalizer(): ) -# Test strings exercising a range of normalization scenarios -_COMPARISON_STRINGS = [ - "hello world", # pure ASCII - "café", # precomposed é - "café", # decomposed e + combining acute - "élève", # é, è precomposed - "ẛ̣", # ẛ + combining dot below (reordering needed) - "fi fl", # fi fl ligatures (compat) - "Ω", # Ω (ohm sign, compat with U+03A9) - "½", # ½ vulgar fraction (compat) - "　", # ideographic space (compat) - "가", # 가 Hangul syllable (algorithmic) - "힣", # 힣 last Hangul syllable - "", # empty string -] +@pytest.fixture(scope="module") +def comparison_strings(): + """Strings exercising a range of normalization scenarios.""" + return [ + "hello world", # pure ASCII + "café", # precomposed é + "café", # decomposed e + combining acute + "élève", # é, è precomposed + "ẛ̣", # ẛ + combining dot below (reordering needed) + "fi fl", # fi fl ligatures (compat) + "Ω", # Ω (ohm sign, compat with U+03A9) + "½", # ½ vulgar fraction (compat) + "　", # ideographic space (compat) + "가", # 가 Hangul syllable (algorithmic) + "힣", # 힣 last Hangul syllable + "", # empty string + ] @pytest.mark.parametrize( @@ -221,14 +223,16 @@ def bmp_nfkd_normalizer(): ("NFKD", "bmp_nfkd_normalizer"), ], ) -def test_compare_with_python_unicodedata(form, fixture_name, request): +def test_compare_with_python_unicodedata( + form, fixture_name, comparison_strings, request +): """GPU normalization must match Python's unicodedata.normalize reference.""" normalizer = request.getfixturevalue(fixture_name) - input_arr = pa.array(_COMPARISON_STRINGS, type=pa.string()) + input_arr = pa.array(comparison_strings, type=pa.string()) gpu_result = normalize_unicode( plc.Column.from_arrow(input_arr), normalizer ) expected = pa.array( - [ud.normalize(form, s) for s in _COMPARISON_STRINGS], type=pa.string() + [ud.normalize(form, s) for s in comparison_strings], type=pa.string() ) assert_column_eq(expected, gpu_result)