Add python/cython interface for unicode-normalizer APIs

python/cudf/cudf/core/unicode_normalizer.py

@@ -0,0 +1,176 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Unicode TR15 normalization support for cudf strings columns."""
+
+from __future__ import annotations
+
+import unicodedata as ud
+
+import pylibcudf as plc
+
+from cudf.core.column.column import ColumnBase
+from cudf.core.dataframe import DataFrame
+from cudf.core.series import Series
+
+
+class UnicodeNormalizer:
+    """
+    A normalizer object for Unicode TR15 normalization (NFD, NFC, NFKD, NFKC).
+
+    The normalizer is constructed from the contents of the Unicode Character
+    Database ``UnicodeData.txt`` file, which must be loaded by the caller as a
+    :class:`cudf.DataFrame` (e.g. via :func:`cudf.read_csv`).
+
+    The file is published by the Unicode Consortium and can be downloaded from:
+    https://unicode.org/Public/15.1.0/ucd/UnicodeData.txt
+
+    It is a semicolon-delimited file with 15 fields per row.  The three fields
+    required by this API are field 0 (code point hex), field 3 (CCC), and
+    field 5 (decomposition mapping).  Load only those three columns:
+
+    .. code-block:: python
+
+        unicode_data = cudf.read_csv(
+            "UnicodeData.txt",
+            sep=";",
+            header=None,
+            usecols=[0, 3, 5],
+            dtype={0: "str", 3: "int32", 5: "str"},
+        )
+        normalizer = UnicodeNormalizer(unicode_data, form="NFKC")
+        result = normalizer.normalize(series)
+
+    The ``unicode_data`` DataFrame must contain exactly three columns in the
+    following order:
+
+    - column[0]: ``str``   Code point values as uppercase hex strings (e.g. ``"00C9"``)
+    - column[1]: ``int32`` Canonical_Combining_Class (CCC) values in range [0, 254]
+    - column[2]: ``str``   Decomposition_Mapping field; empty string for identity
+      mappings, optionally prefixed with a compatibility tag such as
+      ``<compat>``, ``<font>``, ``<wide>``, etc.
+
+    Decomposition of Hangul syllables (U+AC00..U+D7A3) is performed
+    algorithmically per the Unicode standard and does not require entries in
+    the provided table.
+
+    Parameters
+    ----------
+    unicode_data : cudf.DataFrame
+        Three-column DataFrame loaded from ``UnicodeData.txt`` as described above.
+    form : str
+        Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``,
+        or ``"NFKC"``.
+    """
+
+    _FORM_MAP = {
+        "NFD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFD,
+        "NFC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFC,
+        "NFKD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKD,
+        "NFKC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKC,
+    }
+
+    def __init__(
+        self,
+        unicode_data: DataFrame,
+        form: str = "NFC",
+    ) -> None:
+        if form not in self._FORM_MAP:
+            raise ValueError(
+                f"Invalid normalization form {form!r}. "
+                f"Expected one of: {list(self._FORM_MAP)}"
+            )
+        tbl, _ = unicode_data.to_pylibcudf()
+        self._normalizer = plc.nvtext.unicode_normalize.UnicodeNormalizer(
+            tbl, self._FORM_MAP[form]
+        )
+
+    @classmethod
+    def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer:
+        """
+        Construct a :class:`UnicodeNormalizer` from Python's built-in
+        ``unicodedata`` module, without requiring a ``UnicodeData.txt`` file.
+
+        This is equivalent to downloading ``UnicodeData.txt``, loading it with
+        :func:`cudf.read_csv`, and passing the result to
+        :class:`UnicodeNormalizer`, but derives the same three columns directly
+        from :mod:`unicodedata`:
+
+        .. code-block:: python
+
+            import unicodedata as ud
+
+            cp_list, ccc_list, decomp_list = [], [], []
+            for cp in range(0x0080, 0x110000):
+                c = chr(cp)
+                ccc = ud.combining(c)
+                decomp = ud.decomposition(c)
+                if ccc != 0 or decomp:
+                    cp_list.append(f"{cp:04X}")
+                    ccc_list.append(ccc)
+                    decomp_list.append(decomp)
+
+        The Unicode version used is whichever version is bundled with the
+        running Python interpreter (see ``unicodedata.unidata_version``), which
+        may differ from a specific ``UnicodeData.txt`` download.
+
+        Parameters
+        ----------
+        form : str
+            Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``,
+            or ``"NFKC"``.
+
+        Returns
+        -------
+        UnicodeNormalizer
+            Ready-to-use normalizer for the requested form.
+        """
+        if form not in cls._FORM_MAP:
+            raise ValueError(
+                f"Invalid normalization form {form!r}. "
+                f"Expected one of: {list(cls._FORM_MAP)}"
+            )
+
+        cp_list: list[str] = []
+        ccc_list: list[int] = []
+        decomp_list: list[str] = []
+        for cp in range(0x0080, 0x110000):
+            c = chr(cp)
+            ccc = ud.combining(c)
+            decomp = ud.decomposition(c)
+            if ccc != 0 or decomp:
+                cp_list.append(f"{cp:04X}")
+                ccc_list.append(ccc)
+                decomp_list.append(decomp)
+
+        unicode_data = DataFrame(
+            {
+                "cp": Series(cp_list),
+                "ccc": Series(ccc_list, dtype="int32"),
+                "decomp": Series(decomp_list),
+            }
+        )
+        return cls(unicode_data, form=form)
+
+    def normalize(self, text: Series) -> Series:
+        """
+        Normalize a strings Series using Unicode TR15 normalization.
+
+        Parameters
+        ----------
+        text : cudf.Series
+            The UTF-8 strings to normalize.  Null entries produce null output.
+
+        Returns
+        -------
+        cudf.Series
+            New Series of normalized UTF-8 strings.
+        """
+        plc_column = plc.nvtext.unicode_normalize.normalize_unicode(
+            text._column.plc_column, self._normalizer
+        )
+        return Series._from_column(
+            ColumnBase.create(plc_column, text._column.dtype),
+            name=text.name,
+            index=text.index,
+        )

python/cudf/cudf/tests/text/test_text_methods.py

@@ -10,6 +10,7 @@
 import cudf
 from cudf.core.character_normalizer import CharacterNormalizer
 from cudf.core.tokenize_vocabulary import TokenizeVocabulary
+from cudf.core.unicode_normalizer import UnicodeNormalizer
 from cudf.testing import assert_eq
 
 
@@ -274,6 +275,72 @@ def test_normalize_characters():
     assert_eq(expected, actual)
 
 
+def test_unicode_normalize():
+    # Minimal unicode_data DataFrame: é (U+00E9) and combining acute (U+0301),
+    # plus fi ligature (U+FB01) with compatibility-only decomposition.
+    unicode_data = cudf.DataFrame(
+        {
+            "cp": cudf.Series(["00E9", "0301", "FB01"]),
+            "ccc": cudf.Series([0, 230, 0], dtype="int32"),
+            "decomp": cudf.Series(["0065 0301", "", "<compat> 0066 0069"]),
+        }
+    )
+
+    # NFC: decomposed e + combining acute → precomposed é; fi unchanged
+    nfc = UnicodeNormalizer(unicode_data, form="NFC")
+    assert_eq(
+        nfc.normalize(cudf.Series(["é", "café", "fi", None])),
+        cudf.Series(["é", "café", "fi", None]),
+    )
+
+    # NFD: precomposed é (U+00E9) → e (U+0065) + combining acute (U+0301)
+    nfd = UnicodeNormalizer(unicode_data, form="NFD")
+    assert_eq(
+        nfd.normalize(cudf.Series(["é"])),
+        cudf.Series(["é"]),
+    )
+
+    # NFKC: fi ligature expands to "fi"
+    nfkc = UnicodeNormalizer(unicode_data, form="NFKC")
+    assert_eq(
+        nfkc.normalize(cudf.Series(["fi", "café"])),
+        cudf.Series(["fi", "café"]),
+    )
+
+    # result preserves index and name from input
+    nfc = UnicodeNormalizer(unicode_data, form="NFC")
+    s = cudf.Series(["é"], index=[42], name="my_col")
+    result = nfc.normalize(s)
+    assert result.name == "my_col"
+    assert result.index.to_arrow().to_pylist() == [42]
+
+    # invalid form raises ValueError
+    with pytest.raises(ValueError, match="Invalid normalization form"):
+        UnicodeNormalizer(unicode_data, form="XYZ")
+
+
+def test_unicode_normalize_from_python_unicodedata():
+    import unicodedata as ud
+
+    strings = ["café", "fi", "½", "가", "", None]
+    series = cudf.Series(strings)
+    all_null = cudf.Series([None, None, None], dtype="object")
+    for form in ("NFC", "NFD", "NFKC", "NFKD"):
+        normalizer = UnicodeNormalizer.from_python_unicodedata(form=form)
+        result = normalizer.normalize(series)
+        expected = cudf.Series(
+            [ud.normalize(form, s) if s is not None else None for s in strings]
+        )
+        assert_eq(result, expected)
+        null_result = normalizer.normalize(all_null)
+        assert_eq(null_result, all_null)
+        assert null_result.dtype == all_null.dtype
+
+    # invalid form must raise before any codepoint scanning occurs
+    with pytest.raises(ValueError, match="Invalid normalization form"):
+        UnicodeNormalizer.from_python_unicodedata(form="XYZ")
+
+
 @pytest.mark.parametrize(
     "n, separator, expected_values",
     [

python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt

@@ -1,11 +1,11 @@
 # =============================================================================
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 # =============================================================================
 
-set(cython_sources stemmer.pyx)
+set(cython_sources stemmer.pyx unicode_normalize.pyx)
 
 set(linked_libraries cudf::cudf)
 

python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+from libc.stdint cimport int32_t
+from libcpp.memory cimport unique_ptr
+from pylibcudf.exception_handler cimport libcudf_exception_handler
+from pylibcudf.libcudf.column.column cimport column
+from pylibcudf.libcudf.column.column_view cimport column_view
+from pylibcudf.libcudf.table.table_view cimport table_view
+from cuda.bindings.cyruntime cimport cudaStream_t
+from rmm.librmm.memory_resource cimport device_async_resource_ref
+
+
+cdef extern from "nvtext/unicode_normalize.hpp" namespace "nvtext" nogil:
+
+    cpdef enum class unicode_normalization_form(int32_t):
+        NFD
+        NFC
+        NFKD
+        NFKC
+
+    cdef struct unicode_normalizer:
+        pass
+
+    cdef unique_ptr[unicode_normalizer] create_unicode_normalizer(
+        const table_view &unicode_data,
+        unicode_normalization_form form,
+        cudaStream_t stream,
+        device_async_resource_ref mr
+    ) except +libcudf_exception_handler
+
+    cdef unique_ptr[column] normalize_unicode(
+        const column_view &input,
+        const unicode_normalizer &normalizer,
+        cudaStream_t stream,
+        device_async_resource_ref mr
+    ) except +libcudf_exception_handler

python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0

python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt

@@ -1,6 +1,6 @@
 # =============================================================================
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 # =============================================================================
@@ -17,6 +17,7 @@ set(cython_sources
     replace.pyx
     stemmer.pyx
     tokenize.pyx
+    unicode_normalize.pyx
     wordpiece_tokenize.pyx
 )
 

python/pylibcudf/pylibcudf/nvtext/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 from . import (
@@ -13,6 +13,7 @@
     replace,
     stemmer,
     tokenize,
+    unicode_normalize,
     wordpiece_tokenize,
 )
 
@@ -28,5 +29,6 @@
     "replace",
     "stemmer",
     "tokenize",
+    "unicode_normalize",
     "wordpiece_tokenize",
 ]

python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from libcpp.memory cimport unique_ptr
+from pylibcudf.column cimport Column
+from pylibcudf.libcudf.nvtext.unicode_normalize cimport (
+    unicode_normalizer as cpp_unicode_normalizer,
+    unicode_normalization_form as cpp_unicode_normalization_form,
+)
+from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource
+
+
+cdef class UnicodeNormalizer:
+    cdef unique_ptr[cpp_unicode_normalizer] c_obj
+
+cpdef Column normalize_unicode(
+    Column input,
+    UnicodeNormalizer normalizer,
+    object stream=*,
+    DeviceMemoryResource mr=*,
+)