Upgrade lindera dependency from 2.0.1 to 5.0.1

.github/workflows/periodic.yml

@@ -64,4 +64,4 @@ jobs:
           components: rustfmt, clippy
 
       - name: Run test
-        run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
+        run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}

.github/workflows/regression.yml

@@ -67,4 +67,4 @@ jobs:
           components: rustfmt, clippy
 
       - name: Run test
-        run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
+        run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}

.github/workflows/release.yml

@@ -54,7 +54,6 @@ jobs:
             target: aarch64-pc-windows-msvc
             skip_test_run: true
         toolchain: [stable]
-        features: ["cjk"]
     runs-on: ${{ matrix.platform.runner }}
     env:
       LINDERA_CONFIG_PATH: "./resources/lindera.yml"
@@ -70,7 +69,7 @@ jobs:
           components: rustfmt, clippy
 
       - name: Run test
-        run: cargo test --target "${{ matrix.platform.target }}" --all-features ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
+        run: cargo test --target "${{ matrix.platform.target }}" --features=embed-cjk ${{ matrix.platform.skip_test_run == true && '--no-run' || '' }}
 
   create-release:
     name: Create release
@@ -138,12 +137,15 @@ jobs:
           - value: "embed-cc-cedict"
             package_name: "lindera-sqlite-cc-cedict"
             package_description: "Python binding for Lindera with CHinese dictionary (CC-CEDICT)"
+          - value: "embed-jieba"
+            package_name: "lindera-sqlite-jieba"
+            package_description: "Python binding for Lindera with Chinese dictionary (Jieba)"
           - value: "embed-cjk"
             package_name: "lindera-sqlite"
-            package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, CC-CEDICT)"
+            package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, Jieba)"
     runs-on: ${{ matrix.platform.runner }}
     env:
-      LINDERA_CONFIG_PATH: "./resources/lindera.json"
+      LINDERA_CONFIG_PATH: "./resources/lindera.yml"
     permissions:
       contents: write
     steps:

Cargo.toml

@@ -13,23 +13,29 @@ license = "MIT"
 
 [features]
 embed-ipadic = [
-    "lindera/embed-ipadic",
+    "lindera-analysis/embed-ipadic",
 ] # Include Japanese dictionary (IPADIC)
 embed-ipadic-neologd = [
-    "lindera/embed-ipadic-neologd",
+    "lindera-analysis/embed-ipadic-neologd",
 ] # Include Japanese dictionary (IPADIC NEologd)
 embed-unidic = [
-    "lindera/embed-unidic",
+    "lindera-analysis/embed-unidic",
 ] # Include Japanese dictionary (UniDic)
 embed-ko-dic = [
-    "lindera/embed-ko-dic",
+    "lindera-analysis/embed-ko-dic",
 ] # Include Korean dictionary (ko-dic)
 embed-cc-cedict = [
-    "lindera/embed-cc-cedict",
+    "lindera-analysis/embed-cc-cedict",
 ] # Include Chinese dictionary (CC-CEDICT)
+embed-jieba = [
+    "lindera-analysis/embed-jieba",
+] # Include Chinese dictionary (Jieba)
 embed-cjk = [
-    "lindera/embed-cjk",
-] # Include CJK dictionary (Chinese, Japanese, Korean)
+    "embed-ipadic",
+    "embed-ko-dic",
+    "embed-jieba",
+] # Include CJK dictionaries (IPADIC, ko-dic, Jieba). Mirrors lindera 5.0.1's
+# `embed-cjk`; re-check this composition when bumping the lindera dependency.
 extension = []
 default = ["extension"] # No directories included
 
@@ -47,13 +53,10 @@ panic = "abort"   # Abort on panic for smaller binary and faster code
 [dependencies]
 # libc without `std`
 libc = { version = "0.2.180", default-features = false, features = [] }
-serde_json = "1.0.149"
 sqlite-loadable = "0.0.6-alpha.6"
 sqlite3ext-sys = "0.0.1"
-unicode-segmentation = "1.12.0"
-unicode-normalization = "0.1.25"
 
-lindera = "2.0.1"
+lindera-analysis = "5.0.1"
 
 [dev-dependencies]
 criterion = "0.8.1"

README.md

@@ -10,6 +10,18 @@ When used as a custom FTS5 tokenizer this enables application to support Chinese
 % cargo build --features=embed-cjk
 ```
 
+Each `embed-*` feature embeds a different set of dictionaries into the built extension:
+
+| Feature | Embedded dictionaries |
+| --- | --- |
+| `embed-ipadic` | Japanese (IPADIC) |
+| `embed-ipadic-neologd` | Japanese (IPADIC NEologd) |
+| `embed-unidic` | Japanese (UniDic) |
+| `embed-ko-dic` | Korean (ko-dic) |
+| `embed-cc-cedict` | Chinese (CC-CEDICT) |
+| `embed-jieba` | Chinese (Jieba) |
+| `embed-cjk` | Japanese (IPADIC) + Korean (ko-dic) + Chinese (Jieba) |
+
 ## Set enviromment variable for Lindera configuration
 
 ```sh

src/common.rs

@@ -7,7 +7,7 @@ use core::convert::TryFrom;
 
 use libc::{c_char, c_int, c_void};
 
-use lindera::tokenizer::Tokenizer;
+use lindera_analysis::tokenizer::Tokenizer;
 
 // sqlite3.h
 

src/lib.rs

@@ -6,7 +6,7 @@
 //! ## Features
 //!
 //! - **CJK Language Support**: Tokenizes Chinese, Japanese, and Korean text using Lindera
-//! - **Multiple Dictionaries**: Supports various embedded dictionaries (IPADIC, UniDic, ko-dic, CC-CEDICT)
+//! - **Multiple Dictionaries**: Supports various embedded dictionaries (IPADIC, UniDic, ko-dic, CC-CEDICT, Jieba)
 //! - **Configurable**: Uses YAML configuration for character filters and token filters
 //! - **SQLite Integration**: Seamlessly integrates with SQLite's FTS5 full-text search
 //!
@@ -15,7 +15,7 @@
 //! ### Building the Extension
 //!
 //! ```bash
-//! cargo build --release --features=embedded-cjk
+//! cargo build --release --features=embed-cjk
 //! ```
 //!
 //! ### Setting Up Configuration
@@ -62,7 +62,7 @@ mod extension;
 
 use libc::{c_char, c_int, c_uchar, c_void};
 
-use lindera::tokenizer::{Tokenizer, TokenizerBuilder};
+use lindera_analysis::tokenizer::{Tokenizer, TokenizerBuilder};
 
 pub use crate::common::*;