chore(release): ship a matching lindera.yml in every release archive

.github/workflows/release.yml

@@ -124,25 +124,38 @@ jobs:
             archive: .zip
             extension: ".dll"
         toolchain: [stable]
+        # `config` is the resources/*.yml shipped inside that variant's archive. The tokenizer reads
+        # its dictionary and filter chain from this file at load time (via LINDERA_CONFIG_PATH), so
+        # each archive has to carry the one matching the dictionary it embedded — the IPADIC config
+        # names `embedded://ipadic` and filters on IPADIC part-of-speech tags, which is wrong for
+        # every other variant.
         features:
           - value: "embed-ipadic"
             package_name: "lindera-sqlite-ipadic"
             package_description: "Python binding for Lindera with Japanese dictionary (IPADIC)"
+            config: "lindera.yml"
           - value: "embed-unidic"
             package_name: "lindera-sqlite-unidic"
             package_description: "Python binding for Lindera with Japanese dictionary (Unidic)"
+            config: "lindera-unidic.yml"
           - value: "embed-ko-dic"
             package_name: "lindera-sqlite-ko-dic"
             package_description: "Python binding for Lindera with Korean dictionary (ko-dic)"
+            config: "lindera-ko-dic.yml"
           - value: "embed-cc-cedict"
             package_name: "lindera-sqlite-cc-cedict"
             package_description: "Python binding for Lindera with CHinese dictionary (CC-CEDICT)"
+            config: "lindera-cc-cedict.yml"
           - value: "embed-jieba"
             package_name: "lindera-sqlite-jieba"
             package_description: "Python binding for Lindera with Chinese dictionary (Jieba)"
+            config: "lindera-jieba.yml"
           - value: "embed-cjk"
             package_name: "lindera-sqlite"
             package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, Jieba)"
+            # Embeds three dictionaries but a config selects ONE; IPADIC is the default, and a
+            # consumer wanting ko-dic or Jieba edits the `dictionary:` line.
+            config: "lindera.yml"
     runs-on: ${{ matrix.platform.runner }}
     env:
       LINDERA_CONFIG_PATH: "./resources/lindera.yml"
@@ -166,17 +179,24 @@ jobs:
       - name: Compile
         run: cargo build --release --features=${{ matrix.features.value }} --target=${{ matrix.platform.target }} --target-dir=target/${{ matrix.features.value }}
 
+      # The config is renamed to a plain `lindera.yml` inside every archive: which variant it came
+      # from is already in the archive's own name, and a fixed filename means the setup instructions
+      # are the same for all of them.
+      - name: Stage the tokenizer config
+        shell: bash
+        run: cp resources/${{ matrix.features.config }} lindera.yml
+
       - name: Create artifact for Linux
         if: runner.os == 'Linux'
-        run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }}
+        run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} lindera.yml
 
       - name: Create artifact for Windows
         if: runner.os == 'Windows'
-        run: powershell Compress-Archive -DestinationPath ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} -Path target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/lindera_sqlite${{ matrix.platform.extension }}
+        run: powershell Compress-Archive -DestinationPath ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} -Path target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/lindera_sqlite${{ matrix.platform.extension }},lindera.yml
 
       - name: Create artifact for OSX
         if: runner.os == 'macOS'
-        run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }}
+        run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} lindera.yml
 
       - name: Upload artifact
         env:

README.md

@@ -4,6 +4,17 @@ lindera-sqlite is a C ABI library which exposes a [FTS5](https://www.sqlite.org/
 
 When used as a custom FTS5 tokenizer this enables application to support Chinese, Japanese and Korean in full-text search.
 
+## Use a prebuilt extension
+
+Every [release](https://github.com/lindera/lindera-sqlite/releases) publishes a per-platform, per-dictionary archive containing the shared library and a `lindera.yml` matching the dictionary that archive embedded. Unpack it and point `LINDERA_CONFIG_PATH` at that file:
+
+```sh
+% unzip lindera-sqlite-x86_64-unknown-linux-gnu-v2.0.0.zip
+% export LINDERA_CONFIG_PATH=./lindera.yml
+```
+
+Then skip to [Load extension](#load-extension), using the unpacked library path.
+
 ## Build extension
 
 ```sh
@@ -24,10 +35,22 @@ Each `embed-*` feature embeds a different set of dictionaries into the built ext
 
 ## Set enviromment variable for Lindera configuration
 
+When building from source, use the config in `resources/` matching the feature you built. `resources/lindera.yml` is the IPADIC one, used by `embed-ipadic` and `embed-cjk`:
+
 ```sh
 % export LINDERA_CONFIG_PATH=./resources/lindera.yml
 ```
 
+| Feature | Configuration |
+| --- | --- |
+| `embed-ipadic`, `embed-cjk` | `resources/lindera.yml` |
+| `embed-unidic` | `resources/lindera-unidic.yml` |
+| `embed-ko-dic` | `resources/lindera-ko-dic.yml` |
+| `embed-cc-cedict` | `resources/lindera-cc-cedict.yml` |
+| `embed-jieba` | `resources/lindera-jieba.yml` |
+
+`embed-cjk` embeds three dictionaries, but a configuration selects one; edit the `dictionary:` line to use ko-dic or Jieba instead.
+
 ## Then start SQLite
 
 ```sh

resources/lindera-cc-cedict.yml

@@ -0,0 +1,15 @@
+segmenter:
+  mode: "normal"
+  dictionary: "embedded://cc-cedict"
+  # user_dictionary:
+  #   path: "./resources/cc-cedict_simple.csv"
+  #   kind: "cc-cedict"
+  keep_whitespace: false
+
+character_filters:
+  - kind: "unicode_normalize"
+    args:
+      kind: "nfkc"
+# No token filters: Lindera ships no Chinese-specific ones, and the Japanese and Korean stop-tag
+# filters key on tag vocabularies CC-CEDICT does not use. Segmentation alone is the useful default
+# here; add `stop_words` or `length` (both language-neutral) if a corpus needs them.

resources/lindera-jieba.yml

@@ -0,0 +1,12 @@
+segmenter:
+  mode: "normal"
+  dictionary: "embedded://jieba"
+  keep_whitespace: false
+
+character_filters:
+  - kind: "unicode_normalize"
+    args:
+      kind: "nfkc"
+# No token filters, for the same reason as the CC-CEDICT config: Lindera ships no Chinese-specific
+# ones, and the Japanese/Korean stop-tag filters key on tag vocabularies this dictionary does not
+# use. The language-neutral filters (`stop_words`, `length`, `lowercase`) remain available.

resources/lindera-ko-dic.yml

@@ -0,0 +1,24 @@
+segmenter:
+  mode: "normal"
+  dictionary: "embedded://ko-dic"
+  # user_dictionary:
+  #   path: "./resources/ko-dic_simple.csv"
+  #   kind: "ko-dic"
+  keep_whitespace: false
+
+character_filters:
+  # Only the language-neutral normaliser: `japanese_iteration_mark` has nothing to do here.
+  - kind: "unicode_normalize"
+    args:
+      kind: "nfkc"
+
+token_filters:
+  # ko-dic's own stop-tag filter, not the Japanese one — the tag vocabulary is entirely different.
+  # EP/EF are verb endings and JKG is the possessive particle: the Korean counterparts of the
+  # 助動詞/助詞 the IPADIC config drops.
+  - kind: "korean_stop_tags"
+    args:
+      tags:
+        - "EP"
+        - "EF"
+        - "JKG"

resources/lindera-unidic.yml

@@ -0,0 +1,33 @@
+segmenter:
+  mode: "normal"
+  dictionary: "embedded://unidic"
+  # user_dictionary:
+  #   path: "./resources/unidic_simple.csv"
+  #   kind: "unidic"
+  keep_whitespace: false
+
+character_filters:
+  - kind: "unicode_normalize"
+    args:
+      kind: "nfkc"
+  - kind: "japanese_iteration_mark"
+    args:
+      normalize_kanji: true
+      normalize_kana: true
+
+token_filters:
+  # Only the TOP-LEVEL tags, unlike the IPADIC config's deeper list. IPADIC and UniDic agree on
+  # these three but subdivide them differently below that level, so a deeper list written for one
+  # silently matches nothing in the other.
+  - kind: "japanese_stop_tags"
+    args:
+      tags:
+        - "助詞"
+        - "助動詞"
+        - "記号"
+  - kind: "japanese_katakana_stem"
+    args:
+      min: 3
+  - kind: "remove_diacritical_mark"
+    args:
+      japanese: false