diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index c313463..1ea936f 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -64,4 +64,4 @@ jobs: components: rustfmt, clippy - name: Run test - run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} + run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index c80a5b1..a6bbc0e 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -67,4 +67,4 @@ jobs: components: rustfmt, clippy - name: Run test - run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} + run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9303dae..8785b11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,6 @@ jobs: target: aarch64-pc-windows-msvc skip_test_run: true toolchain: [stable] - features: ["cjk"] runs-on: ${{ matrix.platform.runner }} env: LINDERA_CONFIG_PATH: "./resources/lindera.yml" @@ -70,7 +69,7 @@ jobs: components: rustfmt, clippy - name: Run test - run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} + run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }} create-release: name: Create release @@ -138,12 +137,15 @@ jobs: - value: "embed-cc-cedict" package_name: "lindera-sqlite-cc-cedict" package_description: "Python binding for Lindera with CHinese dictionary (CC-CEDICT)" + - value: "embed-jieba" + package_name: "lindera-sqlite-jieba" + package_description: "Python binding for Lindera with Chinese dictionary (Jieba)" - value: "embed-cjk" package_name: "lindera-sqlite" - package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, CC-CEDICT)" + package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, Jieba)" runs-on: ${{ matrix.platform.runner }} env: - LINDERA_CONFIG_PATH: "./resources/lindera.json" + LINDERA_CONFIG_PATH: "./resources/lindera.yml" permissions: contents: write steps: diff --git a/Cargo.toml b/Cargo.toml index 3bfbad9..34f1be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,23 +13,29 @@ license = "MIT" [features] embed-ipadic = [ - "lindera/embed-ipadic", + "lindera-analysis/embed-ipadic", ] # Include Japanese dictionary (IPADIC) embed-ipadic-neologd = [ - "lindera/embed-ipadic-neologd", + "lindera-analysis/embed-ipadic-neologd", ] # Include Japanese dictionary (IPADIC NEologd) embed-unidic = [ - "lindera/embed-unidic", + "lindera-analysis/embed-unidic", ] # Include Japanese dictionary (UniDic) embed-ko-dic = [ - "lindera/embed-ko-dic", + "lindera-analysis/embed-ko-dic", ] # Include Korean dictionary (ko-dic) embed-cc-cedict = [ - "lindera/embed-cc-cedict", + "lindera-analysis/embed-cc-cedict", ] # Include Chinese dictionary (CC-CEDICT) +embed-jieba = [ + "lindera-analysis/embed-jieba", +] # Include Chinese dictionary (Jieba) embed-cjk = [ - "lindera/embed-cjk", -] # Include CJK dictionary (Chinese, Japanese, Korean) + "embed-ipadic", + "embed-ko-dic", + "embed-jieba", +] # Include CJK dictionaries (IPADIC, ko-dic, Jieba). Mirrors lindera 5.0.1's +# `embed-cjk`; re-check this composition when bumping the lindera dependency. extension = [] default = ["extension"] # No directories included @@ -47,13 +53,10 @@ panic = "abort" # Abort on panic for smaller binary and faster code [dependencies] # libc without `std` libc = { version = "0.2.180", default-features = false, features = [] } -serde_json = "1.0.149" sqlite-loadable = "0.0.6-alpha.6" sqlite3ext-sys = "0.0.1" -unicode-segmentation = "1.12.0" -unicode-normalization = "0.1.25" -lindera = "2.0.1" +lindera-analysis = "5.0.1" [dev-dependencies] criterion = "0.8.1" diff --git a/README.md b/README.md index 8c37f41..939e14b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,18 @@ When used as a custom FTS5 tokenizer this enables application to support Chinese % cargo build --features=embed-cjk ``` +Each `embed-*` feature embeds a different set of dictionaries into the built extension: + +| Feature | Embedded dictionaries | +| --- | --- | +| `embed-ipadic` | Japanese (IPADIC) | +| `embed-ipadic-neologd` | Japanese (IPADIC NEologd) | +| `embed-unidic` | Japanese (UniDic) | +| `embed-ko-dic` | Korean (ko-dic) | +| `embed-cc-cedict` | Chinese (CC-CEDICT) | +| `embed-jieba` | Chinese (Jieba) | +| `embed-cjk` | Japanese (IPADIC) + Korean (ko-dic) + Chinese (Jieba) | + ## Set enviromment variable for Lindera configuration ```sh diff --git a/src/common.rs b/src/common.rs index 5378da0..19d20e2 100644 --- a/src/common.rs +++ b/src/common.rs @@ -7,7 +7,7 @@ use core::convert::TryFrom; use libc::{c_char, c_int, c_void}; -use lindera::tokenizer::Tokenizer; +use lindera_analysis::tokenizer::Tokenizer; // sqlite3.h diff --git a/src/lib.rs b/src/lib.rs index f8417e5..931e83e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,7 @@ //! ## Features //! //! - **CJK Language Support**: Tokenizes Chinese, Japanese, and Korean text using Lindera -//! - **Multiple Dictionaries**: Supports various embedded dictionaries (IPADIC, UniDic, ko-dic, CC-CEDICT) +//! - **Multiple Dictionaries**: Supports various embedded dictionaries (IPADIC, UniDic, ko-dic, CC-CEDICT, Jieba) //! - **Configurable**: Uses YAML configuration for character filters and token filters //! - **SQLite Integration**: Seamlessly integrates with SQLite's FTS5 full-text search //! @@ -15,7 +15,7 @@ //! ### Building the Extension //! //! ```bash -//! cargo build --release --features=embedded-cjk +//! cargo build --release --features=embed-cjk //! ``` //! //! ### Setting Up Configuration @@ -62,7 +62,7 @@ mod extension; use libc::{c_char, c_int, c_uchar, c_void}; -use lindera::tokenizer::{Tokenizer, TokenizerBuilder}; +use lindera_analysis::tokenizer::{Tokenizer, TokenizerBuilder}; pub use crate::common::*;