diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index fb995178..ea835942 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -9,12 +9,14 @@ on: - main paths: - "nlpo3-python/**" + - "nlpo3-deepcut-python/**" - ".github/workflows/build-python-wheels.yml" pull_request: branches: - main paths: - "nlpo3-python/**" + - "nlpo3-deepcut-python/**" - ".github/workflows/build-python-wheels.yml" release: types: [published] @@ -50,8 +52,12 @@ jobs: run: bash build_tools/github/check_build_trigger.sh # To trigger the build steps, add "[cd build]" to commit message + # --------------------------------------------------------------------------- + # nlpo3 (base package — no deepcut / tract-onnx) + # --------------------------------------------------------------------------- + build_wheels: - name: Build ${{ matrix.os }} + name: Build nlpo3 ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -128,7 +134,7 @@ jobs: retention-days: 1 build_sdist: - name: Build source distribution + name: Build nlpo3 source distribution runs-on: ubuntu-latest needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -158,14 +164,120 @@ jobs: overwrite: true retention-days: 1 + # --------------------------------------------------------------------------- + # nlpo3-deepcut (optional deepcut model + tract-onnx extension) + # --------------------------------------------------------------------------- + + build_wheels_deepcut: + name: Build nlpo3-deepcut ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: ["3.14"] + + steps: + - name: Checkout source code + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install Python dependencies + run: python -m pip install --upgrade pip + + - name: Setup Rust toolchain + if: startsWith(matrix.os, 'ubuntu-') == false + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Setup rustup target + if: startsWith(matrix.os, 'macos-') + run: rustup target add x86_64-apple-darwin + + - name: Build Python wheels + uses: pypa/cibuildwheel@v3.3.1 + with: + package-dir: nlpo3-deepcut-python + output-dir: wheelhouse-deepcut + env: + CIBW_ENABLE: "graalpy pypy" + CIBW_BUILD: "cp* gp* pp*" + CIBW_BUILD_VERBOSITY: 1 + CIBW_SKIP: "*-musllinux_i686 gp*-win_*" + CIBW_ARCHS_MACOS: "x86_64 arm64" + CIBW_ENVIRONMENT_MACOS: | + MACOSX_DEPLOYMENT_TARGET=10.12 + PATH="$HOME/.cargo/bin:$PATH" + CC=/usr/bin/clang + CXX=/usr/bin/clang++ + CIBW_ARCHS_LINUX: "x86_64 i686" + CIBW_ENVIRONMENT_LINUX: PATH="$HOME/.cargo/bin:$PATH" + CIBW_BEFORE_BUILD_LINUX: | + pip install --upgrade setuptools-rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + CIBW_ARCHS_WINDOWS: "AMD64 x86" + + - name: List wheel files + run: | + echo "Files to upload (binary wheels):" + find ./wheelhouse-deepcut -maxdepth 1 -type f -name '*.whl' -print | sort || true + echo "Count: $(find ./wheelhouse-deepcut -maxdepth 1 -type f -name '*.whl' -print | wc -l || true)" + shell: bash + - name: Store artifacts + uses: actions/upload-artifact@v6 + with: + path: ./wheelhouse-deepcut/*.whl + name: ${{ github.job }}-wheelhouse-${{ matrix.os }}-${{ github.run_id }} + if-no-files-found: warn + overwrite: true + retention-days: 1 + + build_sdist_deepcut: + name: Build nlpo3-deepcut source distribution + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + steps: + - name: Checkout source code + uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + cache: "pip" + - name: Build source distribution + run: | + cd nlpo3-deepcut-python + bash ../build_tools/github/build_source.sh + - name: List sdist files + run: | + echo "Files to upload (sdist):" + find nlpo3-deepcut-python/dist -maxdepth 1 -type f -name '*.tar.gz' -print | sort || true + echo "Count: $(find nlpo3-deepcut-python/dist -maxdepth 1 -type f -name '*.tar.gz' -print | wc -l || true)" + shell: bash + - name: Store artifacts + uses: actions/upload-artifact@v6 + with: + path: nlpo3-deepcut-python/dist/*.tar.gz + name: sdist-deepcut + overwrite: true + retention-days: 1 + + # --------------------------------------------------------------------------- + # Publish both packages to PyPI + # --------------------------------------------------------------------------- + publish_pypi: name: Publish to PyPI runs-on: ubuntu-latest - needs: [build_wheels, build_sdist] + needs: [build_wheels, build_sdist, build_wheels_deepcut, build_sdist_deepcut] # Publish when a GitHub Release is created: if: github.event_name == 'release' && github.event.action == 'published' - # Alternatively, upload to PyPI on every tag starting with 'v': - #if: github.event_name == 'push' && startsWith(github.event.ref, 'v') steps: - name: Retrieve all artifacts uses: actions/download-artifact@v6 @@ -178,3 +290,4 @@ jobs: skip-existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + diff --git a/.github/workflows/test-nlpo3-python.yml b/.github/workflows/test-nlpo3-python.yml index 2bdce62f..27d14f25 100644 --- a/.github/workflows/test-nlpo3-python.yml +++ b/.github/workflows/test-nlpo3-python.yml @@ -13,6 +13,11 @@ on: - "nlpo3-python/tests/**" - "nlpo3-python/Cargo.toml" - "nlpo3-python/pyproject.toml" + - "nlpo3-deepcut-python/src/**" + - "nlpo3-deepcut-python/nlpo3_deepcut/**" + - "nlpo3-deepcut-python/tests/**" + - "nlpo3-deepcut-python/Cargo.toml" + - "nlpo3-deepcut-python/pyproject.toml" - ".github/workflows/test-nlpo3-python.yml" pull_request: branches: @@ -27,13 +32,18 @@ on: - "nlpo3-python/tests/**" - "nlpo3-python/Cargo.toml" - "nlpo3-python/pyproject.toml" + - "nlpo3-deepcut-python/src/**" + - "nlpo3-deepcut-python/nlpo3_deepcut/**" + - "nlpo3-deepcut-python/tests/**" + - "nlpo3-deepcut-python/Cargo.toml" + - "nlpo3-deepcut-python/pyproject.toml" - ".github/workflows/test-nlpo3-python.yml" -defaults: - run: - working-directory: nlpo3-python - jobs: + # --------------------------------------------------------------------------- + # Test nlpo3 (base package — no deepcut) + # --------------------------------------------------------------------------- + test-python: strategy: fail-fast: true @@ -47,6 +57,9 @@ jobs: python-version: "3.14" runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: nlpo3-python steps: - name: Checkout source code @@ -80,3 +93,63 @@ jobs: run: | cd tests python -m unittest + + # --------------------------------------------------------------------------- + # Test nlpo3-deepcut (optional deepcut package) + # --------------------------------------------------------------------------- + + test-python-deepcut: + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + python-version: ["3.14", "3.13", "3.12", "3.11", "3.10", "3.9"] + include: + - os: windows-latest + python-version: "3.14" + - os: macos-latest + python-version: "3.14" + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout source code + uses: actions/checkout@v6 + + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Build nlpo3 base wheel + working-directory: nlpo3-python + run: | + pip install -U pip + pip install -U build setuptools setuptools-rust twine wheel + python -m build --wheel + + - name: Build nlpo3-deepcut wheel + working-directory: nlpo3-deepcut-python + run: | + pip install -U build setuptools setuptools-rust wheel + python -m build --wheel + + - name: Check wheel metadata + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' + run: | + pip install twine + python -m twine check nlpo3-deepcut-python/dist/* + + - name: Install both wheels + run: | + pip install --no-index --find-links=nlpo3-python/dist nlpo3 + pip install --no-index --find-links=nlpo3-deepcut-python/dist nlpo3-deepcut + + - name: Test + working-directory: nlpo3-deepcut-python/tests + run: python -m unittest + diff --git a/CHANGELOG.md b/CHANGELOG.md index e65ecf38..87639a2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Major release with breaking changes. `NewmmTokenizer`, `NewmmFstTokenizer`, and `DeepcutTokenizer`. - Added better parallel processing controls for large input. - Added CLI tokenizer selection (`newmm`, `nf`, `deepcut`). +- **New package `nlpo3-deepcut`**: optional PyPI package that ships the + deepcut ONNX model and the `tract-onnx` inference engine. Install with + `pip install nlpo3-deepcut` to enable `DeepcutTokenizer`. ### Changed @@ -32,6 +35,11 @@ Major release with breaking changes. tokenizer-specific `segment_to_string*` methods. Use `segment(...)` and `segment_with_options(...)` instead; both remain fallible and return `AnyResult>` (`anyhow::Result>`). +- **Breaking (Python):** `DeepcutTokenizer` is no longer bundled in the base + `nlpo3` package. It is now provided by the optional `nlpo3-deepcut` package. + The `nlpo3.DeepcutTokenizer` name is still available as a shim; if + `nlpo3-deepcut` is not installed, constructing it raises `ImportError` with + an installation hint. ### Removed @@ -44,7 +52,21 @@ Major release with breaking changes. ### Migration -Python: +Python (DeepcutTokenizer): + +```bash +# Install both packages to use DeepcutTokenizer +pip install nlpo3 nlpo3-deepcut +``` + +```python +# Usage is unchanged +from nlpo3 import DeepcutTokenizer +tok = DeepcutTokenizer() +tokens = tok.segment("สวัสดีครับ") +``` + +Python (NewmmTokenizer): ```python # Before (v1.x) diff --git a/docs/design.md b/docs/design.md index 58b5bdd9..02417b87 100644 --- a/docs/design.md +++ b/docs/design.md @@ -5,10 +5,9 @@ SPDX-License-Identifier: CC0-1.0 # Design notes -## Proposed split packaging for bindings (design only) +## Split packaging for Python bindings -Status: proposed architecture. This section documents a packaging direction and -release strategy. It is not implemented yet. +Status: implemented in v2.0.0. ### Goals @@ -17,39 +16,49 @@ release strategy. It is not implemented yet. - Keep user-facing API stable where possible. - Let users install optional assets/features explicitly. -### Python packaging proposal +### Python packages -Proposed packages: +Two packages are published to PyPI: - `nlpo3` (base): - - Includes NewMM tokenizers and core Rust extension. - - Does not bundle default dictionary. - - Does not include Deepcut model or ONNX-heavy dependency path. - - Users pass their own dictionary path. -- `nlpo3-dict` (optional data package): - - Provides default `words_th.txt` and dictionary access helpers. - - No ONNX/model dependency. -- `nlpo3-deepcut` (optional model package): - - Provides Deepcut model assets and Deepcut-enabled extension/runtime. - - Pulls required model/runtime dependencies. + - Includes `NewmmTokenizer`, `NewmmFstTokenizer`, and the core Rust extension. + - Compiled without the `deepcut` Cargo feature: no `tract-onnx`, no `ndarray`, + no embedded ONNX model. + - `DeepcutTokenizer` is available as a Python-level shim that delegates to + `nlpo3_deepcut.DeepcutTokenizer` when the optional package is present. + - If `nlpo3-deepcut` is absent, constructing `DeepcutTokenizer` raises + `ImportError` with a clear installation hint. +- `nlpo3-deepcut` (optional model package, `nlpo3-deepcut-python/`): + - Provides `DeepcutTokenizer` backed by the deepcut ONNX model. + - Compiled with the `deepcut` Cargo feature: embeds the ONNX model and + links `tract-onnx` + `ndarray`. + - Declares `nlpo3~=2.0` as a runtime dependency. + +Install flows: + +- `pip install nlpo3` — lightweight dictionary-based tokenization with + user-supplied dictionary. +- `pip install nlpo3 nlpo3-deepcut` — adds `DeepcutTokenizer` support. -Expected install flows: +API behavior: -- `pip install nlpo3` for lightweight dictionary-based tokenization with - user-supplied dictionary. -- `pip install nlpo3-dict` to add default dictionary data. -- `pip install nlpo3-deepcut` to add `DeepcutTokenizer` support. +- Base package always exports `NewmmTokenizer` and `NewmmFstTokenizer`. +- `DeepcutTokenizer` in the base package is a forwarding shim: + - If `nlpo3-deepcut` is installed: `DeepcutTokenizer(...)` constructs and + returns an `nlpo3_deepcut.DeepcutTokenizer` instance. + - If absent: raises `ImportError` with the hint + `pip install nlpo3-deepcut`. -API behavior proposal: +### Rust package layout -- Base package exports NewMM classes always. -- Deepcut import is conditional: - - If deepcut package is present, `DeepcutTokenizer` is available. - - If absent, import or construction returns a clear guidance error message. +| Directory | Cargo crate | Compiled feature set | +|-----------|-------------|----------------------| +| `nlpo3-python/` | `nlpo3-python` | no `deepcut` feature | +| `nlpo3-deepcut-python/` | `nlpo3-deepcut-python` | `deepcut` feature (default) | ### npm packaging proposal -Proposed packages: +Proposed packages (not yet implemented): - `nlpo3` (base): - Includes dictionary-based tokenizers only. @@ -77,27 +86,30 @@ API behavior proposal: - Maintain separate release pipelines per package. - Keep core tokenizer code in one source tree; package-specific wrappers select enabled features and bundled assets. -- Use compatibility constraints so `-dict` and `-deepcut` packages track the - same major/minor line as base. +- Use compatibility constraints so `-deepcut` package tracks the same + major/minor line as base (`nlpo3~=2.0`). ### CI and test strategy - Base package CI validates: - - NewMM tokenizers, dictionary-from-user-path flows, no deepcut dependency. -- Dict package CI validates: - - Data packaging, default dictionary loading behavior, version sync. + - NewMM tokenizers, dictionary-from-user-path flows. + - `DeepcutTokenizer` raises `ImportError` with guidance when `nlpo3-deepcut` + is absent. - Deepcut package CI validates: - - Model availability, deepcut inference path, platform wheel/addon coverage. -- Add compatibility tests for mixed installs (base + dict, base + deepcut, - base + dict + deepcut). + - Model availability, deepcut inference path, platform wheel coverage. + - `nlpo3.DeepcutTokenizer` shim delegates correctly when `nlpo3-deepcut` is + installed. +- Both packages are built and published in the same CI release workflow. ### Migration and compatibility notes -- Keep current monolithic behavior for one transition window. -- Provide deprecation notes and migration examples in README/CHANGELOG. -- Preserve class names and method signatures where possible. -- Document package-selection matrix for common use cases (small install, - dictionary convenience, neural tokenizer support). +- Class names and method signatures are preserved. +- Existing code that imports `DeepcutTokenizer` from `nlpo3` continues to work + when `nlpo3-deepcut` is installed. +- Code that imports from `nlpo3_deepcut` directly also works. +- Users upgrading from `nlpo3<2.0` should run `pip install nlpo3-deepcut` to + restore `DeepcutTokenizer` functionality. + ## Benchmark dictionary files diff --git a/docs/implementation.md b/docs/implementation.md index 17c2c957..c1a8a8f1 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -106,7 +106,39 @@ default settings. - Safe mode remains available for highly ambiguous text. - Previous BFS path explosion risk was mitigated with visited-set controls. -## Concurrency model +## Split Python packaging + +The Python binding is split into two PyPI packages to keep the base install lightweight. + +### Package layout + +| Package | Directory | Cargo feature | Contents | +|---------|-----------|---------------|----------| +| `nlpo3` | `nlpo3-python/` | none | `NewmmTokenizer`, `NewmmFstTokenizer`, `DeepcutTokenizer` shim | +| `nlpo3-deepcut` | `nlpo3-deepcut-python/` | `deepcut` (default) | `DeepcutTokenizer` (ONNX model embedded) | + +### DeepcutTokenizer shim + +`nlpo3.DeepcutTokenizer` is a pure-Python class defined in +`nlpo3-python/nlpo3/__init__.py`. Its `__new__` method: + +1. Attempts `from nlpo3_deepcut import DeepcutTokenizer`. +2. On success, constructs and returns the real `nlpo3_deepcut.DeepcutTokenizer` + instance (GIL-releasing Rust extension backed by tract-onnx). +3. On `ImportError`, raises a new `ImportError` with the message: + `DeepcutTokenizer requires the nlpo3-deepcut package. Install it with: pip install nlpo3-deepcut`. + +The shim defines stub `segment(...)` and `__init__` methods for static type +checkers, but they are never reached at runtime — `__new__` returns the real +object before `__init__` is invoked. + +### Rust extension module names + +- `nlpo3._nlpo3_python_backend` — compiled without `deepcut` feature. +- `nlpo3_deepcut._nlpo3_deepcut_backend` — compiled with `deepcut` feature; + embeds `model/deepcut.onnx` via `include_bytes!`. + + - Tokenizer instances are designed for read-heavy concurrent usage. - Dictionary structures are shared where possible. diff --git a/nlpo3-deepcut-python/.cargo/config.toml b/nlpo3-deepcut-python/.cargo/config.toml new file mode 100644 index 00000000..d47f983e --- /dev/null +++ b/nlpo3-deepcut-python/.cargo/config.toml @@ -0,0 +1,11 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/nlpo3-deepcut-python/CHANGELOG.md b/nlpo3-deepcut-python/CHANGELOG.md new file mode 100644 index 00000000..269b2450 --- /dev/null +++ b/nlpo3-deepcut-python/CHANGELOG.md @@ -0,0 +1,30 @@ +--- +SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +SPDX-License-Identifier: CC0-1.0 +--- + +# Changelog — nlpo3-deepcut + +All notable changes to the `nlpo3-deepcut` Python package are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +This package uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [2.0.0] — 2026-04-04 + +### Added + +- Initial release as a standalone optional package. +- `DeepcutTokenizer` class: CNN-based Thai word tokenizer backed by the + deepcut ONNX model and the `tract-onnx` inference engine. +- Bundled deepcut ONNX model (derived from LEKCut / deepcut, MIT License). +- `segment(text, parallel_chunk_size=None)` method with GIL-releasing + parallel chunk mode. +- PEP 561 support: `py.typed` marker and `_nlpo3_deepcut_backend.pyi` stub. + +### Notes + +- Install alongside `nlpo3>=2.0,<3` for `DeepcutTokenizer` support. +- Users who install only `nlpo3` (without this package) receive a descriptive + `ImportError` with an installation hint when they try to use + `DeepcutTokenizer`. diff --git a/nlpo3-deepcut-python/CITATION.cff b/nlpo3-deepcut-python/CITATION.cff new file mode 100644 index 00000000..2944e075 --- /dev/null +++ b/nlpo3-deepcut-python/CITATION.cff @@ -0,0 +1,36 @@ +cff-version: "1.2.0" +title: "nlpO3" +message: >- + If you use this software, please cite it using these + metadata. +type: software +authors: + - family-names: Suntorntip + given-names: Thanathip + - family-names: Suriyawongkul + given-names: Arthit + orcid: "https://orcid.org/0000-0002-9698-1899" + - family-names: Phatthiyaphaibun + given-names: Wannaphong + orcid: "https://orcid.org/0000-0002-4153-4354" +repository: "https://crates.io/crates/nlpo3" +repository-code: "https://github.com/PyThaiNLP/nlpo3/" +url: "https://github.com/PyThaiNLP/nlpo3/" +abstract: "Thai natural language processing library in Rust, with Python and Node bindings. Formerly oxidized-thainlp." +keywords: + - "tokenizer" + - "tokenization" + - "Thai" + - "natural language processing" + - "NLP" + - "Rust" + - "JavaScript" + - "Python" + - "Node.js" + - "text processing" + - "word segmentation" + - "Thai language" + - "Thai NLP" +license: Apache-2.0 +version: v2.0.0 +date-released: "2026-04-01" diff --git a/nlpo3-deepcut-python/Cargo.lock b/nlpo3-deepcut-python/Cargo.lock new file mode 100644 index 00000000..ee5ab244 --- /dev/null +++ b/nlpo3-deepcut-python/Cargo.lock @@ -0,0 +1,1466 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + +[[package]] +name = "anymap3" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "binary-heap-plus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4551d8382e911ecc0d0f0ffb602777988669be09447d536ff4388d1def11296" +dependencies = [ + "compare", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "dyn-hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15401da73a9ed8c80e3b2d4dc05fe10e7b72d7243b9f614e516a44fa99986e88" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "serde", + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.7.3", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "liquid" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a494c3f9dad3cb7ed16f1c51812cbe4b29493d6c2e5cd1e2b87477263d9534d" +dependencies = [ + "liquid-core", + "liquid-derive", + "liquid-lib", + "serde", +] + +[[package]] +name = "liquid-core" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc623edee8a618b4543e8e8505584f4847a4e51b805db1af6d9af0a3395d0d57" +dependencies = [ + "anymap2", + "itertools 0.14.0", + "kstring", + "liquid-derive", + "pest", + "pest_derive", + "regex", + "serde", + "time", +] + +[[package]] +name = "liquid-derive" +version = "0.26.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de66c928222984aea59fcaed8ba627f388aaac3c1f57dcb05cc25495ef8faefe" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "liquid-lib" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9befeedd61f5995bc128c571db65300aeb50d62e4f0542c88282dbcb5f72372a" +dependencies = [ + "itertools 0.14.0", + "liquid-core", + "percent-encoding", + "regex", + "time", + "unicode-segmentation", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nlpo3" +version = "2.0.0" +dependencies = [ + "anyhow", + "binary-heap-plus", + "fst", + "lazy_static", + "ndarray", + "rayon", + "regex", + "rustc-hash", + "tract-onnx", +] + +[[package]] +name = "nlpo3-deepcut-python" +version = "2.0.0" +dependencies = [ + "nlpo3", + "pyo3", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pyo3" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "safetensors" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172dd94c5a87b5c79f945c863da53b2ebc7ccef4eca24ac63cca66a41aab2178" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scan_fmt" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248" +dependencies = [ + "regex", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "string-interner" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07f9fdfdd31a0ff38b59deb401be81b73913d76c9cc5b1aed4e1330a223420b9" +dependencies = [ + "cfg-if", + "hashbrown", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tract-core" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d67f5190132365dda73fe215bfc5e01b031e8cbfbea9d486bb5b0dbba3545" +dependencies = [ + "anyhow", + "anymap3", + "bit-set", + "derive-new", + "downcast-rs", + "dyn-clone", + "lazy_static", + "log", + "maplit", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pastey", + "rustfft", + "smallvec", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd7fda1e5e8b854ea3abdd09126a87fc4af81e6d1e29ec1710a8a4abf4f13a" +dependencies = [ + "anyhow", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "itertools 0.12.1", + "lazy_static", + "libm", + "maplit", + "ndarray", + "nom", + "nom-language", + "num-integer", + "num-traits", + "parking_lot", + "scan_fmt", + "smallvec", + "string-interner", +] + +[[package]] +name = "tract-hir" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "554df991b647dba8af0547ee5838b6912ed20b424f2adda0ea0b7faf8db1b151" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72097a89cc4e7c5f1bc4f854b9294dd30fa6f6d8f7f409c556953b49078c94f" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs", + "dyn-clone", + "dyn-hash", + "half", + "lazy_static", + "liquid", + "liquid-core", + "liquid-derive", + "log", + "num-traits", + "pastey", + "scan_fmt", + "smallvec", + "time", + "tract-data", + "unicode-normalization", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b3755dd0948111b407085d11033ba218cb85b85ce8d795cec2b8353db552ea" +dependencies = [ + "byteorder", + "flate2", + "liquid", + "liquid-core", + "log", + "nom", + "nom-language", + "safetensors", + "serde_json", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac23ad1d2d5da3256ae1a78757b1072a8a3fac2a4b28d27cfb561c5942ec2701" +dependencies = [ + "bytes", + "derive-new", + "log", + "memmap2", + "num-integer", + "prost", + "smallvec", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87561bf0b84f74a124afc0f1997682728da6cd821083511e0357432954fd24f6" +dependencies = [ + "getrandom", + "log", + "rand", + "rand_distr", + "rustfft", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/nlpo3-deepcut-python/Cargo.toml b/nlpo3-deepcut-python/Cargo.toml new file mode 100644 index 00000000..c927898e --- /dev/null +++ b/nlpo3-deepcut-python/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "nlpo3-deepcut-python" +version = "2.0.0" +edition = "2024" +rust-version = "1.88.0" +license = "Apache-2.0" +authors = [ + "Thanathip Suntorntip Gorlph", + "Arthit Suriyawongkul", + "Wannaphong Phatthiyaphaibun ", +] +description = "Deepcut ONNX tokenizer for nlpO3 — optional extension for nlpo3" +categories = ["text-processing"] +keywords = [ + "thai", + "tokenizer", + "nlp", + "deepcut", + "pythainlp", +] +homepage = "https://github.com/PyThaiNLP/nlpo3/tree/main/nlpo3-deepcut-python" +repository = "https://github.com/PyThaiNLP/nlpo3/" +documentation = "https://github.com/PyThaiNLP/nlpo3/blob/main/nlpo3-deepcut-python/README.md" +readme = "README.md" +include = [ + "/Cargo.toml", + "/Cargo.lock", + "/README.md", + "/LICENSE", + "/CHANGELOG.md", + "/CITATION.cff", + "/pyproject.toml", + "/MANIFEST.in", + "/src/lib.rs", + "/nlpo3_deepcut/__init__.py", + "/nlpo3_deepcut/_nlpo3_deepcut_backend.pyi", + "/nlpo3_deepcut/py.typed", +] + +[features] +default = ["deepcut"] +deepcut = ["nlpo3/deepcut"] + +[lib] +name = "_nlpo3_deepcut_backend" +path = "src/lib.rs" +crate-type = ["cdylib", "rlib"] + +[dependencies] +nlpo3 = { version = "2.0", path = ".." } + +[dependencies.pyo3] +version = "0.28" +features = ["extension-module"] diff --git a/nlpo3-deepcut-python/LICENSE b/nlpo3-deepcut-python/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/nlpo3-deepcut-python/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/nlpo3-deepcut-python/MANIFEST.in b/nlpo3-deepcut-python/MANIFEST.in new file mode 100644 index 00000000..dae8d2af --- /dev/null +++ b/nlpo3-deepcut-python/MANIFEST.in @@ -0,0 +1,23 @@ +include LICENSE README.md pyproject.toml Cargo.toml +include Cargo.lock + +include src/lib.rs + +# Ensure the Python package files are included in the sdist +graft nlpo3_deepcut + +# Include PEP 561 marker and type stubs for compiled extension +include nlpo3_deepcut/py.typed +recursive-include nlpo3_deepcut *.pyi + +# Exclude tests from the sdist to keep it small +prune tests +prune build +prune target +prune nlpo3_deepcut.egg-info +prune nlpo3_deepcut/__pycache__ + +# Keep source distributions deterministic and free of runtime caches. +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude .DS_Store diff --git a/nlpo3-deepcut-python/README.md b/nlpo3-deepcut-python/README.md new file mode 100644 index 00000000..7f7369dd --- /dev/null +++ b/nlpo3-deepcut-python/README.md @@ -0,0 +1,73 @@ +--- +SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +SPDX-License-Identifier: Apache-2.0 +--- + +# nlpo3-deepcut + +Deepcut CNN-based Thai word tokenizer for [nlpO3](https://github.com/PyThaiNLP/nlpo3). + +This is an optional extension package for [`nlpo3`](https://pypi.org/project/nlpo3/). +It ships the deepcut ONNX model and the [tract-onnx](https://github.com/sonos/tract) +runtime compiled into a compact Rust extension module. + +## Install + +```bash +pip install nlpo3 nlpo3-deepcut +``` + +## Usage + +```python +from nlpo3 import DeepcutTokenizer + +# Use the bundled model (installed by nlpo3-deepcut) +tok = DeepcutTokenizer() +tokens = tok.segment("สวัสดีครับ") +print(tokens) + +# Use a custom ONNX model file +tok = DeepcutTokenizer(model_path="/path/to/custom.onnx") +``` + +You can also import directly from `nlpo3_deepcut`: + +```python +from nlpo3_deepcut import DeepcutTokenizer + +tok = DeepcutTokenizer() +tokens = tok.segment("สวัสดีครับ") +``` + +## Why a separate package? + +The deepcut tokenizer uses an ONNX neural model and the `tract-onnx` inference +engine. Bundling them in the base `nlpo3` package would make every install +significantly larger, even for users who only need the lightweight +dictionary-based `NewmmTokenizer`. + +Splitting the packages lets users choose: + +| Need | Install | +|------|---------| +| Dictionary-based tokenization only | `pip install nlpo3` | +| Dictionary + neural deepcut | `pip install nlpo3 nlpo3-deepcut` | + +## Error without nlpo3-deepcut + +If you call `DeepcutTokenizer()` without installing this package, `nlpo3` +raises a descriptive `ImportError`: + +``` +ImportError: DeepcutTokenizer requires the nlpo3-deepcut package. +Install it with: pip install nlpo3-deepcut +``` + +## License + +Apache-2.0. See [LICENSE](https://github.com/PyThaiNLP/nlpo3/blob/main/nlpo3-deepcut-python/LICENSE). + +The bundled deepcut ONNX model is derived from +[LEKCut](https://github.com/PyThaiNLP/LEKCut) / +[deepcut](https://github.com/rkcosmos/deepcut) (MIT License). diff --git a/nlpo3-deepcut-python/nlpo3_deepcut/__init__.py b/nlpo3-deepcut-python/nlpo3_deepcut/__init__.py new file mode 100644 index 00000000..f4f6caed --- /dev/null +++ b/nlpo3-deepcut-python/nlpo3_deepcut/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +# Deepcut tokenizer package for nlpO3. +# +# Install this package alongside `nlpo3` to enable DeepcutTokenizer: +# +# pip install nlpo3 nlpo3-deepcut +# +# Authors: +# Thanathip Suntorntip +# Arthit Suriyawongkul + +from __future__ import annotations + +from ._nlpo3_deepcut_backend import DeepcutTokenizer + +__all__ = ["DeepcutTokenizer"] diff --git a/nlpo3-deepcut-python/nlpo3_deepcut/_nlpo3_deepcut_backend.pyi b/nlpo3-deepcut-python/nlpo3_deepcut/_nlpo3_deepcut_backend.pyi new file mode 100644 index 00000000..374dd4d3 --- /dev/null +++ b/nlpo3-deepcut-python/nlpo3_deepcut/_nlpo3_deepcut_backend.pyi @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +"""Type stubs for _nlpo3_deepcut_backend Rust extension module.""" + +from __future__ import annotations + +from typing import List, Optional + +class DeepcutTokenizer: + """Deepcut CNN-based Thai word tokenizer. + + Each instance compiles and owns the ONNX model. Internally the compiled + model is reference-counted (Arc), so cloning is O(1). The same instance + is safe to call from multiple threads simultaneously. + + For distributed or parallel workloads, create one instance per worker + process to avoid sharing state across process boundaries. + + Example:: + + from nlpo3_deepcut import DeepcutTokenizer + + # Bundled model + tok = DeepcutTokenizer() + tokens = tok.segment("สวัสดีครับ") + + # Custom ONNX model + tok = DeepcutTokenizer(model_path="/path/to/custom.onnx") + + You can also import from the base package when ``nlpo3-deepcut`` is + installed:: + + from nlpo3 import DeepcutTokenizer + tok = DeepcutTokenizer() + """ + + def __new__(cls, model_path: Optional[str] = None) -> "DeepcutTokenizer": + """Create a DeepcutTokenizer. + + Args: + model_path: Path to a custom deepcut ONNX model file, or ``None`` + to use the bundled default model. + + Raises: + RuntimeError: If the ONNX model cannot be loaded. + """ + ... + + def segment( + self, text: str, parallel_chunk_size: Optional[int] = None + ) -> List[str]: + """Tokenize text using the deepcut CNN model. + + Inference is thread-safe: the same instance may be called concurrently + from multiple threads. + + Args: + text: Input text to tokenize. + parallel_chunk_size: Target chunk size in bytes for chunked parallel + processing. `None`, `0`, or low values disable parallel mode. + + Note: + When ``parallel_chunk_size`` is set, text is split into chunks. + Because deepcut uses a fixed-width context window, characters near + chunk boundaries have fewer adjacent context characters, so token + sequences near boundaries may differ from full-text results. + Suitable for classification and embedding tasks; not recommended + for tasks requiring precise token boundaries. + + Returns: + List of word tokens. + + Raises: + RuntimeError: If ONNX inference fails. + """ + ... diff --git a/nlpo3-deepcut-python/nlpo3_deepcut/py.typed b/nlpo3-deepcut-python/nlpo3_deepcut/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/nlpo3-deepcut-python/pyproject.toml b/nlpo3-deepcut-python/pyproject.toml new file mode 100644 index 00000000..42fa95ee --- /dev/null +++ b/nlpo3-deepcut-python/pyproject.toml @@ -0,0 +1,106 @@ +[build-system] +requires = ["setuptools", "setuptools-rust", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nlpo3-deepcut" +version = "2.0.0" +description = "Deepcut ONNX tokenizer for nlpO3 — optional extension for nlpo3" +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.9" +license = "Apache-2.0" +license-files = ["LICENSE"] +keywords = [ + "thai", + "tokenizer", + "tokenization", + "deepcut", + "nlp", + "natural-language-processing", + "word-segmentation", + "pythainlp", + "rust", +] +authors = [ + { name = "Thanathip Suntorntip" }, + { name = "Arthit Suriyawongkul" }, + { name = "Wannaphong Phatthiyaphaibun", email = "wannaphong@yahoo.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "Natural Language :: Thai", + "Topic :: Text Processing :: Linguistic", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "nlpo3~=2.0", +] + +[project.urls] +homepage = "https://github.com/PyThaiNLP/nlpo3/tree/main/nlpo3-deepcut-python" +repository = "https://github.com/PyThaiNLP/nlpo3/" +documentation = "https://github.com/PyThaiNLP/nlpo3/blob/main/nlpo3-deepcut-python/README.md" + +[tool.black] +line-length = 79 +exclude = ''' +/( + \.eggs + | \.git + | \.mypy_cache + | \.vscode + | build + | dist +)/ +''' + +[tool.mypy] +pretty = true +show_column_numbers = true +show_error_code_links = true +show_error_context = true +strict = true +strict_equality = true +strict_equality_for_none = true +warn_unreachable = true +# _nlpo3_deepcut_backend is a compiled Rust extension; types come from the +# .pyi stub. Ignore the missing-module error at the package level. +ignore_missing_imports = true + +[tool.pylint.main] +# Exclude type-stub files: pylint does not fully understand .pyi semantics. +ignore-patterns = [".*\\.pyi$"] + +[tool.pylint.typecheck] +# _nlpo3_deepcut_backend is a compiled Rust extension that requires a build +# step not run during linting. +ignored-modules = ["nlpo3_deepcut._nlpo3_deepcut_backend", "_nlpo3_deepcut_backend"] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["nlpo3_deepcut"] +exclude = ["tests", "nlpo3_deepcut.__pycache__"] + +[tool.setuptools.package-data] +nlpo3_deepcut = ["py.typed", "*.pyi"] + +[tool.setuptools.exclude-package-data] +nlpo3_deepcut = ["__pycache__/*", "*.pyc", "*.pyo"] + +[[tool.setuptools-rust.ext-modules]] +target = "nlpo3_deepcut._nlpo3_deepcut_backend" +path = "Cargo.toml" +binding = "PyO3" +strip = "No" diff --git a/nlpo3-deepcut-python/src/lib.rs b/nlpo3-deepcut-python/src/lib.rs new file mode 100644 index 00000000..4d1952a7 --- /dev/null +++ b/nlpo3-deepcut-python/src/lib.rs @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +// SPDX-License-Identifier: Apache-2.0 + +/** + * Python binding for the nlpO3 Deepcut tokenizer. + * + * This crate provides only the DeepcutTokenizer class. It compiles with the + * `deepcut` feature of the core `nlpo3` crate, embedding the ONNX model and + * linking tract-onnx. Users install this as the optional `nlpo3-deepcut` + * package alongside the base `nlpo3` package. + * + * Original Rust implementation: Thanathip Suntorntip + * Rewrite and extension: PyThaiNLP Project + */ +use std::path::Path; + +use nlpo3::tokenizer::deepcut::DeepcutTokenizer; +use pyo3::exceptions; +use pyo3::prelude::*; + +// --------------------------------------------------------------------------- +// DeepcutTokenizer Python class +// --------------------------------------------------------------------------- + +/// Deepcut CNN-based Thai word tokenizer. +/// +/// Each instance compiles and owns the ONNX model. Internally the compiled +/// model is reference-counted (:class:`Arc`), so cloning is O(1). The same +/// instance is safe to call from multiple threads simultaneously. +/// +/// For distributed or parallel workloads, create one instance per worker +/// process to avoid sharing state across process boundaries. +/// +/// Example: +/// +/// ```python +/// from nlpo3_deepcut import DeepcutTokenizer +/// +/// # Use the bundled default model +/// tok = DeepcutTokenizer() +/// tokens = tok.segment("สวัสดีครับ") +/// +/// # Use a custom ONNX model file +/// tok = DeepcutTokenizer(model_path="/path/to/custom.onnx") +/// tokens = tok.segment("สวัสดีครับ") +/// ``` +/// +/// You can also import from the base package when ``nlpo3-deepcut`` is +/// installed: +/// +/// ```python +/// from nlpo3 import DeepcutTokenizer +/// tok = DeepcutTokenizer() +/// ``` +#[pyclass(name = "DeepcutTokenizer", frozen)] +struct PyDeepcutTokenizer { + inner: DeepcutTokenizer, +} + +#[pymethods] +impl PyDeepcutTokenizer { + /// Create a new DeepcutTokenizer. + /// + /// If ``model_path`` is ``None`` (the default), the bundled ONNX model is + /// used. Pass a filesystem path to load a custom compatible model. + /// + /// Args: + /// model_path: Path to a custom deepcut ONNX model file, or ``None`` + /// to use the bundled default model. + /// + /// Raises: + /// RuntimeError: If the ONNX model cannot be loaded. + #[new] + #[pyo3(signature = (model_path = None))] + fn new(model_path: Option<&str>) -> PyResult { + let result = match model_path { + None => DeepcutTokenizer::new(), + Some(p) => DeepcutTokenizer::from_path(Path::new(p)), + }; + result + .map(|inner| PyDeepcutTokenizer { inner }) + .map_err(|e| { + exceptions::PyRuntimeError::new_err(format!( + "deepcut: failed to load tokenizer: {}", + e + )) + }) + } + + /// Tokenize text using the deepcut CNN model. + /// + /// Inference is thread-safe: the same instance may be called concurrently + /// from multiple threads. + /// + /// This method releases the Python Global Interpreter Lock (GIL) while + /// running inference so other Python threads can continue executing. + /// + /// Args: + /// text: Input text to tokenize. + /// parallel_chunk_size: Target chunk size in bytes for parallel mode. + /// `None`, `0`, or low values disable parallel mode. + /// + /// Note: + /// When ``parallel_chunk_size`` is set, text is split into chunks. + /// Because deepcut uses a fixed-width context window, characters near + /// chunk boundaries have fewer adjacent context characters, so token + /// sequences near boundaries may differ from full-text results. + /// Suitable for classification and embedding tasks; not recommended + /// for tasks requiring precise token boundaries. + /// + /// Returns: + /// List of word tokens. + /// + /// Raises: + /// RuntimeError: If ONNX inference fails. + #[pyo3(signature = (text, parallel_chunk_size = None))] + fn segment( + &self, + py: Python<'_>, + text: &str, + parallel_chunk_size: Option, + ) -> PyResult> { + if text.is_empty() { + return Ok(vec![]); + } + + py.detach(|| self.inner.segment_with_options(text, parallel_chunk_size)) + .map_err(|e| { + exceptions::PyRuntimeError::new_err(format!("deepcut: inference failed: {}", e)) + }) + } +} + +// --------------------------------------------------------------------------- +// Module +// --------------------------------------------------------------------------- + +#[pymodule] +fn _nlpo3_deepcut_backend(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/nlpo3-deepcut-python/tests/__init__.py b/nlpo3-deepcut-python/tests/__init__.py new file mode 100644 index 00000000..e1b29bce --- /dev/null +++ b/nlpo3-deepcut-python/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 diff --git a/nlpo3-deepcut-python/tests/data/test_dict.txt b/nlpo3-deepcut-python/tests/data/test_dict.txt new file mode 100644 index 00000000..a1fabc0e --- /dev/null +++ b/nlpo3-deepcut-python/tests/data/test_dict.txt @@ -0,0 +1,7 @@ +ค่า +ค่าจ้าง +ค่าจ้างเพื่อชีวิต +ค่าแรง +ค่ายทหาร +คน +ไข่ diff --git a/nlpo3-deepcut-python/tests/test_deepcut.py b/nlpo3-deepcut-python/tests/test_deepcut.py new file mode 100644 index 00000000..66e4f686 --- /dev/null +++ b/nlpo3-deepcut-python/tests/test_deepcut.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2024-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from nlpo3_deepcut import DeepcutTokenizer + + +class TestDeepcutTokenizer(unittest.TestCase): + def setUp(self): + self.tok = DeepcutTokenizer() + + def test_empty_input(self): + self.assertEqual(self.tok.segment(""), []) + + def test_returns_list(self): + result = self.tok.segment("ทดสอบ") + self.assertIsInstance(result, list) + self.assertTrue(len(result) > 0) + + def test_basic(self): + result = self.tok.segment("ทดสอบการตัดคำ") + self.assertIsInstance(result, list) + self.assertTrue(len(result) > 0) + # Reconstructed text must equal the input. + self.assertEqual("".join(result), "ทดสอบการตัดคำ") + + def test_reconstructs_input(self): + text = "หมอนทองตากลมหูว์MBK39" + result = self.tok.segment(text) + self.assertEqual("".join(result), text) + + def test_shared_instance(self): + # The same DeepcutTokenizer instance (with Arc-backed model) can be + # reused across many calls — the ONNX model is compiled once and + # shared by all calls to this tokenizer object. + tok = self.tok + texts = ["ทดสอบ", "สวัสดีครับ", "การตัดคำ"] + results = [tok.segment(t) for t in texts] + self.assertEqual(len(results), 3) + for r in results: + self.assertIsInstance(r, list) + + +# --------------------------------------------------------------------------- +# DeepcutTokenizer is also accessible via the nlpo3 base package shim +# --------------------------------------------------------------------------- + + +class TestDeepcutTokenizerViaBasePackage(unittest.TestCase): + """Verify that nlpo3.DeepcutTokenizer delegates to nlpo3_deepcut when + the latter is installed. + """ + + def test_importable_from_nlpo3(self): + from nlpo3 import DeepcutTokenizer as NlpO3DeepcutTokenizer + + tok = NlpO3DeepcutTokenizer() + result = tok.segment("ทดสอบ") + self.assertIsInstance(result, list) + self.assertTrue(len(result) > 0) + + def test_returns_same_results(self): + from nlpo3 import DeepcutTokenizer as NlpO3DeepcutTokenizer + + text = "ทดสอบการตัดคำ" + tok_direct = DeepcutTokenizer() + tok_shim = NlpO3DeepcutTokenizer() + self.assertEqual(tok_direct.segment(text), tok_shim.segment(text)) + + +# --------------------------------------------------------------------------- +# Thread safety +# --------------------------------------------------------------------------- + + +class TestDeepcutTokenizerThreadSafety(unittest.TestCase): + def test_concurrent_segment(self): + import concurrent.futures + + tok = DeepcutTokenizer() + texts = ["ทดสอบ", "สวัสดีครับ", "การตัดคำ"] * 10 + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(tok.segment, texts)) + self.assertEqual(len(results), 30) + for r in results: + self.assertIsInstance(r, list) diff --git a/nlpo3-python/Cargo.toml b/nlpo3-python/Cargo.toml index 8bfc4ee2..cd34227c 100644 --- a/nlpo3-python/Cargo.toml +++ b/nlpo3-python/Cargo.toml @@ -38,7 +38,7 @@ include = [ ] [features] -default = ["deepcut"] +default = [] deepcut = ["nlpo3/deepcut"] [lib] diff --git a/nlpo3-python/nlpo3/__init__.py b/nlpo3-python/nlpo3/__init__.py index 83dde90f..8270bb5e 100644 --- a/nlpo3-python/nlpo3/__init__.py +++ b/nlpo3-python/nlpo3/__init__.py @@ -9,10 +9,78 @@ from __future__ import annotations +from typing import List, Optional + from ._nlpo3_python_backend import ( - DeepcutTokenizer, NewmmFstTokenizer, NewmmTokenizer, ) + +class DeepcutTokenizer: + """Deepcut CNN-based Thai word tokenizer. + + Requires the ``nlpo3-deepcut`` package, which ships the ONNX model and + the ONNX runtime. Install it with:: + + pip install nlpo3-deepcut + + Once installed, usage is identical to the bundled tokenizers: + + .. code-block:: python + + from nlpo3 import DeepcutTokenizer + + tok = DeepcutTokenizer() + tokens = tok.segment("สวัสดีครับ") + + # Use a custom ONNX model file + tok = DeepcutTokenizer(model_path="/path/to/custom.onnx") + + This class is a forwarding shim: construction delegates to + ``nlpo3_deepcut.DeepcutTokenizer`` when that package is present, and + raises :class:`ImportError` with an installation hint when it is absent. + """ + + def __new__( # type: ignore[misc] + cls, + model_path: Optional[str] = None, + ) -> "DeepcutTokenizer": + try: + from nlpo3_deepcut import ( # type: ignore[import-not-found] + DeepcutTokenizer as _RealDeepcutTokenizer, + ) + except ImportError: + raise ImportError( + "DeepcutTokenizer requires the nlpo3-deepcut package.\n" + "Install it with: pip install nlpo3-deepcut" + ) from None + return _RealDeepcutTokenizer( # type: ignore[return-value] + model_path=model_path + ) + + def segment( + self, + text: str, + parallel_chunk_size: Optional[int] = None, + ) -> List[str]: + """Tokenize text using the deepcut CNN model. + + This stub is never called directly; ``__new__`` returns the real + ``nlpo3_deepcut.DeepcutTokenizer`` instance. + + Args: + text: Input text to tokenize. + parallel_chunk_size: Target chunk size in bytes for parallel mode. + ``None``, ``0``, or low values disable parallel mode. + + Returns: + List of word tokens. + + Raises: + RuntimeError: If ONNX inference fails. + """ + ... # pragma: no cover + + __all__ = ["DeepcutTokenizer", "NewmmFstTokenizer", "NewmmTokenizer"] diff --git a/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi b/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi index 5e28319c..cb6fe9be 100644 --- a/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi +++ b/nlpo3-python/nlpo3/_nlpo3_python_backend.pyi @@ -130,66 +130,3 @@ class NewmmFstTokenizer: RuntimeError: If tokenization fails. """ ... - -class DeepcutTokenizer: - """Deepcut CNN-based Thai word tokenizer. - - Each instance compiles and owns the ONNX model. Internally the compiled - model is reference-counted (Arc), so cloning is O(1). The same instance - is safe to call from multiple threads simultaneously. - - For distributed or parallel workloads, create one instance per worker - process to avoid sharing state across process boundaries. - - Example:: - - from nlpo3 import DeepcutTokenizer - - # Bundled model - tok = DeepcutTokenizer() - tokens = tok.segment("สวัสดีครับ") - - # Custom ONNX model - tok = DeepcutTokenizer(model_path="/path/to/custom.onnx") - """ - - def __new__(cls, model_path: Optional[str] = None) -> "DeepcutTokenizer": - """Create a DeepcutTokenizer. - - Args: - model_path: Path to a custom deepcut ONNX model file, or ``None`` - to use the bundled default model. - - Raises: - RuntimeError: If the ONNX model cannot be loaded. - """ - ... - - def segment( - self, text: str, parallel_chunk_size: Optional[int] = None - ) -> List[str]: - """Tokenize text using the deepcut CNN model. - - Inference is thread-safe: the same instance may be called concurrently - from multiple threads. - - Args: - text: Input text to tokenize. - parallel_chunk_size: Target chunk size in bytes for chunked parallel - processing. `None`, `0`, or low values disable parallel mode. - - Note: - When ``parallel_chunk_size`` is set, text is split into chunks. - Because deepcut uses a fixed-width context window, characters near - chunk boundaries have fewer adjacent context characters, so token - sequences near boundaries may differ from full-text results. - Suitable for classification and embedding tasks; not recommended - for tasks requiring precise token boundaries. - - Returns: - List of word tokens. - - Raises: - RuntimeError: If ONNX inference fails. - """ - ... diff --git a/nlpo3-python/src/lib.rs b/nlpo3-python/src/lib.rs index 85eec870..1f4d9541 100644 --- a/nlpo3-python/src/lib.rs +++ b/nlpo3-python/src/lib.rs @@ -7,10 +7,11 @@ * Original Rust implementation: Thanathip Suntorntip * Rewrite and extension: PyThaiNLP Project */ +use nlpo3::tokenizer::newmm::{NewmmFstTokenizer, NewmmTokenizer}; +#[cfg(feature = "deepcut")] use std::path::Path; - +#[cfg(feature = "deepcut")] use nlpo3::tokenizer::deepcut::DeepcutTokenizer; -use nlpo3::tokenizer::newmm::{NewmmFstTokenizer, NewmmTokenizer}; use pyo3::exceptions; use pyo3::prelude::*; @@ -192,9 +193,10 @@ impl PyNewmmFstTokenizer { } // --------------------------------------------------------------------------- -// DeepcutTokenizer Python class +// DeepcutTokenizer Python class (only compiled with the "deepcut" feature) // --------------------------------------------------------------------------- +#[cfg(feature = "deepcut")] /// Deepcut CNN-based Thai word tokenizer. /// /// Each instance compiles and owns the ONNX model. Internally the compiled @@ -222,6 +224,7 @@ struct PyDeepcutTokenizer { inner: DeepcutTokenizer, } +#[cfg(feature = "deepcut")] #[pymethods] impl PyDeepcutTokenizer { /// Create a new DeepcutTokenizer. @@ -304,6 +307,7 @@ impl PyDeepcutTokenizer { fn _nlpo3_python_backend(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + #[cfg(feature = "deepcut")] m.add_class::()?; Ok(()) } diff --git a/nlpo3-python/tests/test_tokenize.py b/nlpo3-python/tests/test_tokenize.py index ddc756c1..2db4930a 100644 --- a/nlpo3-python/tests/test_tokenize.py +++ b/nlpo3-python/tests/test_tokenize.py @@ -116,40 +116,31 @@ def test_shared_instance(self): self.assertEqual(len(results), 3) -class TestDeepcutTokenizer(unittest.TestCase): - def setUp(self): - self.tok = DeepcutTokenizer() +# --------------------------------------------------------------------------- +# DeepcutTokenizer — availability guard +# --------------------------------------------------------------------------- - def test_empty_input(self): - self.assertEqual(self.tok.segment(""), []) - def test_returns_list(self): - result = self.tok.segment("ทดสอบ") - self.assertIsInstance(result, list) - self.assertTrue(len(result) > 0) +class TestDeepcutTokenizerMissingPackage(unittest.TestCase): + """Verify that DeepcutTokenizer raises a helpful ImportError when the + optional ``nlpo3-deepcut`` package is not installed. - def test_basic(self): - result = self.tok.segment("ทดสอบการตัดคำ") - self.assertIsInstance(result, list) - self.assertTrue(len(result) > 0) - # Reconstructed text must equal the input. - self.assertEqual("".join(result), "ทดสอบการตัดคำ") + This test is meaningful only when ``nlpo3-deepcut`` is absent from the + current environment. When that package *is* installed, construction + succeeds and the ImportError is not raised, so the test is skipped. + """ - def test_reconstructs_input(self): - text = "หมอนทองตากลมหูว์MBK39" - result = self.tok.segment(text) - self.assertEqual("".join(result), text) + def test_raises_import_error_when_deepcut_not_installed(self): + try: + import nlpo3_deepcut # noqa: F401 + self.skipTest("nlpo3-deepcut is installed; skipping absence test") + except ImportError: + pass - def test_shared_instance(self): - # The same DeepcutTokenizer instance (with Arc-backed model) can be - # reused across many calls — the ONNX model is compiled once and - # shared by all calls to this tokenizer object. - tok = self.tok - texts = ["ทดสอบ", "สวัสดีครับ", "การตัดคำ"] - results = [tok.segment(t) for t in texts] - self.assertEqual(len(results), 3) - for r in results: - self.assertIsInstance(r, list) + with self.assertRaises(ImportError) as ctx: + DeepcutTokenizer() + self.assertIn("nlpo3-deepcut", str(ctx.exception)) + self.assertIn("pip install", str(ctx.exception)) # ---------------------------------------------------------------------------