diff --git a/python/cudf/cudf/core/unicode_normalizer.py b/python/cudf/cudf/core/unicode_normalizer.py new file mode 100644 index 000000000000..cae08dbaf46d --- /dev/null +++ b/python/cudf/cudf/core/unicode_normalizer.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unicode TR15 normalization support for cudf strings columns.""" + +from __future__ import annotations + +import unicodedata as ud + +import pylibcudf as plc + +from cudf.core.column.column import ColumnBase +from cudf.core.dataframe import DataFrame +from cudf.core.series import Series + + +class UnicodeNormalizer: + """ + A normalizer object for Unicode TR15 normalization (NFD, NFC, NFKD, NFKC). + + The normalizer is constructed from the contents of the Unicode Character + Database ``UnicodeData.txt`` file, which must be loaded by the caller as a + :class:`cudf.DataFrame` (e.g. via :func:`cudf.read_csv`). + + The file is published by the Unicode Consortium and can be downloaded from: + https://unicode.org/Public/15.1.0/ucd/UnicodeData.txt + + It is a semicolon-delimited file with 15 fields per row. The three fields + required by this API are field 0 (code point hex), field 3 (CCC), and + field 5 (decomposition mapping). Load only those three columns: + + .. code-block:: python + + unicode_data = cudf.read_csv( + "UnicodeData.txt", + sep=";", + header=None, + usecols=[0, 3, 5], + dtype={0: "str", 3: "int32", 5: "str"}, + ) + normalizer = UnicodeNormalizer(unicode_data, form="NFKC") + result = normalizer.normalize(series) + + The ``unicode_data`` DataFrame must contain exactly three columns in the + following order: + + - column[0]: ``str`` Code point values as uppercase hex strings (e.g. ``"00C9"``) + - column[1]: ``int32`` Canonical_Combining_Class (CCC) values in range [0, 254] + - column[2]: ``str`` Decomposition_Mapping field; empty string for identity + mappings, optionally prefixed with a compatibility tag such as + ````, ````, ````, etc. + + Decomposition of Hangul syllables (U+AC00..U+D7A3) is performed + algorithmically per the Unicode standard and does not require entries in + the provided table. + + Parameters + ---------- + unicode_data : cudf.DataFrame + Three-column DataFrame loaded from ``UnicodeData.txt`` as described above. + form : str + Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``, + or ``"NFKC"``. + """ + + _FORM_MAP = { + "NFD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFD, + "NFC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFC, + "NFKD": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKD, + "NFKC": plc.nvtext.unicode_normalize.UnicodeNormalizationForm.NFKC, + } + + def __init__( + self, + unicode_data: DataFrame, + form: str = "NFC", + ) -> None: + if form not in self._FORM_MAP: + raise ValueError( + f"Invalid normalization form {form!r}. " + f"Expected one of: {list(self._FORM_MAP)}" + ) + tbl, _ = unicode_data.to_pylibcudf() + self._normalizer = plc.nvtext.unicode_normalize.UnicodeNormalizer( + tbl, self._FORM_MAP[form] + ) + + @classmethod + def from_python_unicodedata(cls, form: str = "NFC") -> UnicodeNormalizer: + """ + Construct a :class:`UnicodeNormalizer` from Python's built-in + ``unicodedata`` module, without requiring a ``UnicodeData.txt`` file. + + This is equivalent to downloading ``UnicodeData.txt``, loading it with + :func:`cudf.read_csv`, and passing the result to + :class:`UnicodeNormalizer`, but derives the same three columns directly + from :mod:`unicodedata`: + + .. code-block:: python + + import unicodedata as ud + + cp_list, ccc_list, decomp_list = [], [], [] + for cp in range(0x0080, 0x110000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + + The Unicode version used is whichever version is bundled with the + running Python interpreter (see ``unicodedata.unidata_version``), which + may differ from a specific ``UnicodeData.txt`` download. + + Parameters + ---------- + form : str + Normalization form: one of ``"NFD"``, ``"NFC"``, ``"NFKD"``, + or ``"NFKC"``. + + Returns + ------- + UnicodeNormalizer + Ready-to-use normalizer for the requested form. + """ + if form not in cls._FORM_MAP: + raise ValueError( + f"Invalid normalization form {form!r}. " + f"Expected one of: {list(cls._FORM_MAP)}" + ) + + cp_list: list[str] = [] + ccc_list: list[int] = [] + decomp_list: list[str] = [] + for cp in range(0x0080, 0x110000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + + unicode_data = DataFrame( + { + "cp": Series(cp_list), + "ccc": Series(ccc_list, dtype="int32"), + "decomp": Series(decomp_list), + } + ) + return cls(unicode_data, form=form) + + def normalize(self, text: Series) -> Series: + """ + Normalize a strings Series using Unicode TR15 normalization. + + Parameters + ---------- + text : cudf.Series + The UTF-8 strings to normalize. Null entries produce null output. + + Returns + ------- + cudf.Series + New Series of normalized UTF-8 strings. + """ + plc_column = plc.nvtext.unicode_normalize.normalize_unicode( + text._column.plc_column, self._normalizer + ) + return Series._from_column( + ColumnBase.create(plc_column, text._column.dtype), + name=text.name, + index=text.index, + ) diff --git a/python/cudf/cudf/tests/text/test_text_methods.py b/python/cudf/cudf/tests/text/test_text_methods.py index 047c27269cbf..273605bfccf9 100644 --- a/python/cudf/cudf/tests/text/test_text_methods.py +++ b/python/cudf/cudf/tests/text/test_text_methods.py @@ -10,6 +10,7 @@ import cudf from cudf.core.character_normalizer import CharacterNormalizer from cudf.core.tokenize_vocabulary import TokenizeVocabulary +from cudf.core.unicode_normalizer import UnicodeNormalizer from cudf.testing import assert_eq @@ -274,6 +275,72 @@ def test_normalize_characters(): assert_eq(expected, actual) +def test_unicode_normalize(): + # Minimal unicode_data DataFrame: é (U+00E9) and combining acute (U+0301), + # plus fi ligature (U+FB01) with compatibility-only decomposition. + unicode_data = cudf.DataFrame( + { + "cp": cudf.Series(["00E9", "0301", "FB01"]), + "ccc": cudf.Series([0, 230, 0], dtype="int32"), + "decomp": cudf.Series(["0065 0301", "", " 0066 0069"]), + } + ) + + # NFC: decomposed e + combining acute → precomposed é; fi unchanged + nfc = UnicodeNormalizer(unicode_data, form="NFC") + assert_eq( + nfc.normalize(cudf.Series(["é", "café", "fi", None])), + cudf.Series(["é", "café", "fi", None]), + ) + + # NFD: precomposed é (U+00E9) → e (U+0065) + combining acute (U+0301) + nfd = UnicodeNormalizer(unicode_data, form="NFD") + assert_eq( + nfd.normalize(cudf.Series(["é"])), + cudf.Series(["é"]), + ) + + # NFKC: fi ligature expands to "fi" + nfkc = UnicodeNormalizer(unicode_data, form="NFKC") + assert_eq( + nfkc.normalize(cudf.Series(["fi", "café"])), + cudf.Series(["fi", "café"]), + ) + + # result preserves index and name from input + nfc = UnicodeNormalizer(unicode_data, form="NFC") + s = cudf.Series(["é"], index=[42], name="my_col") + result = nfc.normalize(s) + assert result.name == "my_col" + assert result.index.to_arrow().to_pylist() == [42] + + # invalid form raises ValueError + with pytest.raises(ValueError, match="Invalid normalization form"): + UnicodeNormalizer(unicode_data, form="XYZ") + + +def test_unicode_normalize_from_python_unicodedata(): + import unicodedata as ud + + strings = ["café", "fi", "½", "가", "", None] + series = cudf.Series(strings) + all_null = cudf.Series([None, None, None], dtype="object") + for form in ("NFC", "NFD", "NFKC", "NFKD"): + normalizer = UnicodeNormalizer.from_python_unicodedata(form=form) + result = normalizer.normalize(series) + expected = cudf.Series( + [ud.normalize(form, s) if s is not None else None for s in strings] + ) + assert_eq(result, expected) + null_result = normalizer.normalize(all_null) + assert_eq(null_result, all_null) + assert null_result.dtype == all_null.dtype + + # invalid form must raise before any codepoint scanning occurs + with pytest.raises(ValueError, match="Invalid normalization form"): + UnicodeNormalizer.from_python_unicodedata(form="XYZ") + + @pytest.mark.parametrize( "n, separator, expected_values", [ diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt index 0a999dc704a6..cfa53955cc32 100644 --- a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt @@ -1,11 +1,11 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= -set(cython_sources stemmer.pyx) +set(cython_sources stemmer.pyx unicode_normalize.pyx) set(linked_libraries cudf::cudf) diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd new file mode 100644 index 000000000000..8fdbd8b376a8 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pxd @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from libc.stdint cimport int32_t +from libcpp.memory cimport unique_ptr +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from cuda.bindings.cyruntime cimport cudaStream_t +from rmm.librmm.memory_resource cimport device_async_resource_ref + + +cdef extern from "nvtext/unicode_normalize.hpp" namespace "nvtext" nogil: + + cpdef enum class unicode_normalization_form(int32_t): + NFD + NFC + NFKD + NFKC + + cdef struct unicode_normalizer: + pass + + cdef unique_ptr[unicode_normalizer] create_unicode_normalizer( + const table_view &unicode_data, + unicode_normalization_form form, + cudaStream_t stream, + device_async_resource_ref mr + ) except +libcudf_exception_handler + + cdef unique_ptr[column] normalize_unicode( + const column_view &input, + const unicode_normalizer &normalizer, + cudaStream_t stream, + device_async_resource_ref mr + ) except +libcudf_exception_handler diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx new file mode 100644 index 000000000000..8eca3cc68ad4 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/unicode_normalize.pyx @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 diff --git a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt index fd5fae932bb9..c0227fbf98b2 100644 --- a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -17,6 +17,7 @@ set(cython_sources replace.pyx stemmer.pyx tokenize.pyx + unicode_normalize.pyx wordpiece_tokenize.pyx ) diff --git a/python/pylibcudf/pylibcudf/nvtext/__init__.py b/python/pylibcudf/pylibcudf/nvtext/__init__.py index bf433f5ccb48..816f0e59e93e 100644 --- a/python/pylibcudf/pylibcudf/nvtext/__init__.py +++ b/python/pylibcudf/pylibcudf/nvtext/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from . import ( @@ -13,6 +13,7 @@ replace, stemmer, tokenize, + unicode_normalize, wordpiece_tokenize, ) @@ -28,5 +29,6 @@ "replace", "stemmer", "tokenize", + "unicode_normalize", "wordpiece_tokenize", ] diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd new file mode 100644 index 000000000000..46e92bac088a --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pxd @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libcpp.memory cimport unique_ptr +from pylibcudf.column cimport Column +from pylibcudf.libcudf.nvtext.unicode_normalize cimport ( + unicode_normalizer as cpp_unicode_normalizer, + unicode_normalization_form as cpp_unicode_normalization_form, +) +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource + + +cdef class UnicodeNormalizer: + cdef unique_ptr[cpp_unicode_normalizer] c_obj + +cpdef Column normalize_unicode( + Column input, + UnicodeNormalizer normalizer, + object stream=*, + DeviceMemoryResource mr=*, +) diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi new file mode 100644 index 000000000000..6d63d3b9ac7a --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyi @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from enum import IntEnum + +from rmm.pylibrmm.memory_resource import DeviceMemoryResource + +from pylibcudf.column import Column +from pylibcudf.table import Table +from pylibcudf.typing import CudaStreamLike + +class UnicodeNormalizationForm(IntEnum): + NFD = ... + NFC = ... + NFKD = ... + NFKC = ... + +class UnicodeNormalizer: + def __init__( + self, + unicode_data: Table, + form: UnicodeNormalizationForm, + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, + ): ... + +def normalize_unicode( + input: Column, + normalizer: UnicodeNormalizer, + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, +) -> Column: ... diff --git a/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx new file mode 100644 index 000000000000..3cc4b0bf2705 --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/unicode_normalize.pyx @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view +from pylibcudf.libcudf.nvtext.unicode_normalize cimport ( + create_unicode_normalizer as cpp_create_unicode_normalizer, + normalize_unicode as cpp_normalize_unicode, + unicode_normalization_form as cpp_unicode_normalization_form, +) +from pylibcudf.libcudf.nvtext.unicode_normalize import \ + unicode_normalization_form as UnicodeNormalizationForm # no-cython-lint +from pylibcudf.table cimport Table +from pylibcudf.utils cimport _get_stream, _get_memory_resource +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pylibcudf.typing import CudaStreamLike +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource +from rmm.pylibrmm.stream cimport Stream +from cuda.bindings.cyruntime cimport cudaStream_t + +__all__ = ["UnicodeNormalizer", "UnicodeNormalizationForm", "normalize_unicode"] + + +cdef class UnicodeNormalizer: + """Normalizer object for Unicode TR15 normalization (NFD/NFC/NFKD/NFKC). + + Constructed from the three relevant columns of UnicodeData.txt loaded as a + :class:`pylibcudf.Table`. Once built the object can be reused across + multiple calls to :func:`normalize_unicode`. + + For details, see :cpp:class:`nvtext::unicode_normalizer`. + + Parameters + ---------- + unicode_data : Table + Three-column table parsed from UnicodeData.txt: + column[0] STRING code-point hex strings (e.g. "00C9"), + column[1] INT32 Canonical_Combining_Class values, + column[2] STRING Decomposition_Mapping field. + form : unicode_normalization_form + Normalization form to apply (NFD, NFC, NFKD, or NFKC). + stream : CudaStreamLike | None + CUDA stream on which to perform the operation. + mr : DeviceMemoryResource | None + Device memory resource for internal table allocations. + """ + def __cinit__( + self, + Table unicode_data, + cpp_unicode_normalization_form form, + object stream: CudaStreamLike | None = None, + DeviceMemoryResource mr=None, + ): + cdef table_view c_data = unicode_data.view() + cdef Stream _stream = _get_stream(stream) + cdef cudaStream_t _cs = _stream.view().value() + cdef DeviceMemoryResource _mr = _get_memory_resource(mr) + with nogil: + self.c_obj = move( + cpp_create_unicode_normalizer(c_data, form, _cs, _mr.get_mr()) + ) + + __hash__ = None + + +cpdef Column normalize_unicode( + Column input, + UnicodeNormalizer normalizer, + object stream: CudaStreamLike | None = None, + DeviceMemoryResource mr=None, +): + """Normalize a strings column using Unicode TR15 normalization. + + Input and output are UTF-8 encoded. Each string is normalized + independently; null entries produce null output entries. + + For details, see :cpp:func:`nvtext::normalize_unicode`. + + Parameters + ---------- + input : Column + Strings column to normalize. + normalizer : UnicodeNormalizer + Normalizer built by :class:`UnicodeNormalizer`. + stream : CudaStreamLike | None + CUDA stream on which to perform the operation. + mr : DeviceMemoryResource | None + Device memory resource for the returned column. + + Returns + ------- + Column + New strings column of normalized UTF-8 strings. + """ + if normalizer is None: + raise TypeError("normalizer must not be None") + cdef unique_ptr[column] c_result + cdef Stream _stream = _get_stream(stream) + cdef cudaStream_t _cs = _stream.view().value() + cdef DeviceMemoryResource _mr = _get_memory_resource(mr) + + cdef column_view c_input = input.view() + with nogil: + c_result = cpp_normalize_unicode( + c_input, + dereference(normalizer.c_obj.get()), + _cs, + _mr.get_mr(), + ) + + return Column.from_libcudf(move(c_result), _stream, _mr) diff --git a/python/pylibcudf/tests/test_nvtext_unicode_normalize.py b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py new file mode 100644 index 000000000000..f4422913c049 --- /dev/null +++ b/python/pylibcudf/tests/test_nvtext_unicode_normalize.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unicodedata as ud + +import pyarrow as pa +import pytest +from utils import assert_column_eq + +import pylibcudf as plc +from pylibcudf.nvtext.unicode_normalize import ( + UnicodeNormalizationForm, + UnicodeNormalizer, + normalize_unicode, +) + + +# --------------------------------------------------------------------------- +# Helper to build a minimal unicode_data Table from explicit row data. +# Each row is (cp_hex: str, ccc: int, decomp: str). +# --------------------------------------------------------------------------- +def _make_unicode_table(rows): + cp_col = pa.array([r[0] for r in rows], type=pa.string()) + ccc_col = pa.array([r[1] for r in rows], type=pa.int32()) + decomp_col = pa.array([r[2] for r in rows], type=pa.string()) + tbl = pa.table({"cp": cp_col, "ccc": ccc_col, "decomp": decomp_col}) + return plc.Table.from_arrow(tbl) + + +# Minimal rows that cover NFD/NFC tests: é = U+00E9 → e (U+0065) + U+0301 +_MINIMAL_ROWS = [ + ("00E9", 0, "0065 0301"), # é, canonical decomposition + ("0301", 230, ""), # combining acute accent, CCC=230 +] + +# Rows needed for the NFKC fi test (U+FB01 ligature) +_COMPAT_ROWS = [ + *_MINIMAL_ROWS, + ("FB01", 0, " 0066 0069"), +] # fi ligature, compatibility only + + +@pytest.fixture(scope="module") +def nfc_normalizer(): + tbl = _make_unicode_table(_MINIMAL_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFC) + + +@pytest.fixture(scope="module") +def nfd_normalizer(): + tbl = _make_unicode_table(_MINIMAL_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFD) + + +@pytest.fixture(scope="module") +def nfkc_normalizer(): + tbl = _make_unicode_table(_COMPAT_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFKC) + + +@pytest.fixture(scope="module") +def nfkd_normalizer(): + tbl = _make_unicode_table(_COMPAT_ROWS) + return UnicodeNormalizer(tbl, UnicodeNormalizationForm.NFKD) + + +# --------------------------------------------------------------------------- +# Basic correctness tests +# --------------------------------------------------------------------------- + + +def test_null_strings(nfc_normalizer): + input_col = plc.Column.from_arrow( + pa.array([None, None, None], type=pa.string()) + ) + result = normalize_unicode(input_col, nfc_normalizer) + expected = pa.array([None, None, None], type=pa.string()) + assert_column_eq(expected, result) + + +def test_empty_column(nfc_normalizer): + arr = pa.array([], type=pa.string()) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_ascii_passthrough(nfc_normalizer): + arr = pa.array(["hello", "world", "abc 123", ""]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_nfc_compose(nfc_normalizer): + # "é" (e + combining acute) should compose to U+00E9 (é) + arr = pa.array(["é", "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + expected = pa.array(["é", "café"]) + assert_column_eq(expected, result) + + +def test_nfd_decompose(nfd_normalizer): + # U+00E9 (é) should decompose to "é" + arr = pa.array(["é"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfd_normalizer) + expected = pa.array(["é"]) + assert_column_eq(expected, result) + + +def test_nfc_stable(nfc_normalizer): + # Already-composed strings should be unchanged by NFC + arr = pa.array(["é", "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_nfkc_compat_ligature(nfkc_normalizer): + # U+FB01 (fi) has only a compatibility decomposition; NFKC expands it to "fi" + arr = pa.array(["fi"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfkc_normalizer) + expected = pa.array(["fi"]) + assert_column_eq(expected, result) + + +def test_nfc_compat_ligature_stable(nfc_normalizer): + # U+FB01 is NFC-stable: NFC must leave it unchanged + arr = pa.array(["fi"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + assert_column_eq(arr, result) + + +def test_mixed_nulls(nfc_normalizer): + # Null rows must not corrupt adjacent non-null rows + arr = pa.array(["é", None, "café"]) + result = normalize_unicode(plc.Column.from_arrow(arr), nfc_normalizer) + expected = pa.array(["é", None, "café"]) + assert_column_eq(expected, result) + + +# --------------------------------------------------------------------------- +# Comparison against Python's unicodedata.normalize +# --------------------------------------------------------------------------- +# Build a moderately complete unicode_data table from Python's unicodedata +# module, covering BMP codepoints (U+0080..U+FFFF) that have a non-trivial +# CCC or decomposition mapping. This mirrors what a user would load from +# UnicodeData.txt and lets us cross-check the GPU result against the +# reference implementation. + + +def _build_bmp_unicode_table(): + """Return a pylibcudf Table covering non-trivial BMP codepoints.""" + cp_list, ccc_list, decomp_list = [], [], [] + for cp in range(0x0080, 0x10000): + c = chr(cp) + ccc = ud.combining(c) + decomp = ud.decomposition(c) + if ccc != 0 or decomp: + cp_list.append(f"{cp:04X}") + ccc_list.append(ccc) + decomp_list.append(decomp) + tbl = pa.table( + { + "cp": pa.array(cp_list, type=pa.string()), + "ccc": pa.array(ccc_list, type=pa.int32()), + "decomp": pa.array(decomp_list, type=pa.string()), + } + ) + return plc.Table.from_arrow(tbl) + + +@pytest.fixture(scope="module") +def bmp_nfc_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFC + ) + + +@pytest.fixture(scope="module") +def bmp_nfkc_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFKC + ) + + +@pytest.fixture(scope="module") +def bmp_nfd_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFD + ) + + +@pytest.fixture(scope="module") +def bmp_nfkd_normalizer(): + return UnicodeNormalizer( + _build_bmp_unicode_table(), UnicodeNormalizationForm.NFKD + ) + + +@pytest.fixture(scope="module") +def comparison_strings(): + """Strings exercising a range of normalization scenarios.""" + return [ + "hello world", # pure ASCII + "café", # precomposed é + "café", # decomposed e + combining acute + "élève", # é, è precomposed + "ẛ̣", # ẛ + combining dot below (reordering needed) + "fi fl", # fi fl ligatures (compat) + "Ω", # Ω (ohm sign, compat with U+03A9) + "½", # ½ vulgar fraction (compat) + "　", # ideographic space (compat) + "가", # 가 Hangul syllable (algorithmic) + "힣", # 힣 last Hangul syllable + "", # empty string + ] + + +@pytest.mark.parametrize( + "form,fixture_name", + [ + ("NFC", "bmp_nfc_normalizer"), + ("NFKC", "bmp_nfkc_normalizer"), + ("NFD", "bmp_nfd_normalizer"), + ("NFKD", "bmp_nfkd_normalizer"), + ], +) +def test_compare_with_python_unicodedata( + form, fixture_name, comparison_strings, request +): + """GPU normalization must match Python's unicodedata.normalize reference.""" + normalizer = request.getfixturevalue(fixture_name) + input_arr = pa.array(comparison_strings, type=pa.string()) + gpu_result = normalize_unicode( + plc.Column.from_arrow(input_arr), normalizer + ) + expected = pa.array( + [ud.normalize(form, s) for s in comparison_strings], type=pa.string() + ) + assert_column_eq(expected, gpu_result)