diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8785b11..f5ccd3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,25 +124,38 @@ jobs: archive: .zip extension: ".dll" toolchain: [stable] + # `config` is the resources/*.yml shipped inside that variant's archive. The tokenizer reads + # its dictionary and filter chain from this file at load time (via LINDERA_CONFIG_PATH), so + # each archive has to carry the one matching the dictionary it embedded — the IPADIC config + # names `embedded://ipadic` and filters on IPADIC part-of-speech tags, which is wrong for + # every other variant. features: - value: "embed-ipadic" package_name: "lindera-sqlite-ipadic" package_description: "Python binding for Lindera with Japanese dictionary (IPADIC)" + config: "lindera.yml" - value: "embed-unidic" package_name: "lindera-sqlite-unidic" package_description: "Python binding for Lindera with Japanese dictionary (Unidic)" + config: "lindera-unidic.yml" - value: "embed-ko-dic" package_name: "lindera-sqlite-ko-dic" package_description: "Python binding for Lindera with Korean dictionary (ko-dic)" + config: "lindera-ko-dic.yml" - value: "embed-cc-cedict" package_name: "lindera-sqlite-cc-cedict" package_description: "Python binding for Lindera with CHinese dictionary (CC-CEDICT)" + config: "lindera-cc-cedict.yml" - value: "embed-jieba" package_name: "lindera-sqlite-jieba" package_description: "Python binding for Lindera with Chinese dictionary (Jieba)" + config: "lindera-jieba.yml" - value: "embed-cjk" package_name: "lindera-sqlite" package_description: "Python binding for Lindera with CJK dictionaries (IPADIC, ko-dic, Jieba)" + # Embeds three dictionaries but a config selects ONE; IPADIC is the default, and a + # consumer wanting ko-dic or Jieba edits the `dictionary:` line. + config: "lindera.yml" runs-on: ${{ matrix.platform.runner }} env: LINDERA_CONFIG_PATH: "./resources/lindera.yml" @@ -166,17 +179,24 @@ jobs: - name: Compile run: cargo build --release --features=${{ matrix.features.value }} --target=${{ matrix.platform.target }} --target-dir=target/${{ matrix.features.value }} + # The config is renamed to a plain `lindera.yml` inside every archive: which variant it came + # from is already in the archive's own name, and a fixed filename means the setup instructions + # are the same for all of them. + - name: Stage the tokenizer config + shell: bash + run: cp resources/${{ matrix.features.config }} lindera.yml + - name: Create artifact for Linux if: runner.os == 'Linux' - run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} + run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} lindera.yml - name: Create artifact for Windows if: runner.os == 'Windows' - run: powershell Compress-Archive -DestinationPath ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} -Path target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/lindera_sqlite${{ matrix.platform.extension }} + run: powershell Compress-Archive -DestinationPath ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} -Path target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/lindera_sqlite${{ matrix.platform.extension }},lindera.yml - name: Create artifact for OSX if: runner.os == 'macOS' - run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} + run: zip --junk-paths ${{ matrix.features.package_name }}-${{ matrix.platform.target }}-${{ github.ref_name }}${{ matrix.platform.archive }} target/${{ matrix.features.value }}/${{ matrix.platform.target }}/release/liblindera_sqlite${{ matrix.platform.extension }} lindera.yml - name: Upload artifact env: diff --git a/README.md b/README.md index 939e14b..dd49fb5 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,17 @@ lindera-sqlite is a C ABI library which exposes a [FTS5](https://www.sqlite.org/ When used as a custom FTS5 tokenizer this enables application to support Chinese, Japanese and Korean in full-text search. +## Use a prebuilt extension + +Every [release](https://github.com/lindera/lindera-sqlite/releases) publishes a per-platform, per-dictionary archive containing the shared library and a `lindera.yml` matching the dictionary that archive embedded. Unpack it and point `LINDERA_CONFIG_PATH` at that file: + +```sh +% unzip lindera-sqlite-x86_64-unknown-linux-gnu-v2.0.0.zip +% export LINDERA_CONFIG_PATH=./lindera.yml +``` + +Then skip to [Load extension](#load-extension), using the unpacked library path. + ## Build extension ```sh @@ -24,10 +35,22 @@ Each `embed-*` feature embeds a different set of dictionaries into the built ext ## Set enviromment variable for Lindera configuration +When building from source, use the config in `resources/` matching the feature you built. `resources/lindera.yml` is the IPADIC one, used by `embed-ipadic` and `embed-cjk`: + ```sh % export LINDERA_CONFIG_PATH=./resources/lindera.yml ``` +| Feature | Configuration | +| --- | --- | +| `embed-ipadic`, `embed-cjk` | `resources/lindera.yml` | +| `embed-unidic` | `resources/lindera-unidic.yml` | +| `embed-ko-dic` | `resources/lindera-ko-dic.yml` | +| `embed-cc-cedict` | `resources/lindera-cc-cedict.yml` | +| `embed-jieba` | `resources/lindera-jieba.yml` | + +`embed-cjk` embeds three dictionaries, but a configuration selects one; edit the `dictionary:` line to use ko-dic or Jieba instead. + ## Then start SQLite ```sh diff --git a/resources/lindera-cc-cedict.yml b/resources/lindera-cc-cedict.yml new file mode 100644 index 0000000..4d9aa81 --- /dev/null +++ b/resources/lindera-cc-cedict.yml @@ -0,0 +1,15 @@ +segmenter: + mode: "normal" + dictionary: "embedded://cc-cedict" + # user_dictionary: + # path: "./resources/cc-cedict_simple.csv" + # kind: "cc-cedict" + keep_whitespace: false + +character_filters: + - kind: "unicode_normalize" + args: + kind: "nfkc" +# No token filters: Lindera ships no Chinese-specific ones, and the Japanese and Korean stop-tag +# filters key on tag vocabularies CC-CEDICT does not use. Segmentation alone is the useful default +# here; add `stop_words` or `length` (both language-neutral) if a corpus needs them. diff --git a/resources/lindera-jieba.yml b/resources/lindera-jieba.yml new file mode 100644 index 0000000..99adc3f --- /dev/null +++ b/resources/lindera-jieba.yml @@ -0,0 +1,12 @@ +segmenter: + mode: "normal" + dictionary: "embedded://jieba" + keep_whitespace: false + +character_filters: + - kind: "unicode_normalize" + args: + kind: "nfkc" +# No token filters, for the same reason as the CC-CEDICT config: Lindera ships no Chinese-specific +# ones, and the Japanese/Korean stop-tag filters key on tag vocabularies this dictionary does not +# use. The language-neutral filters (`stop_words`, `length`, `lowercase`) remain available. diff --git a/resources/lindera-ko-dic.yml b/resources/lindera-ko-dic.yml new file mode 100644 index 0000000..ec24390 --- /dev/null +++ b/resources/lindera-ko-dic.yml @@ -0,0 +1,24 @@ +segmenter: + mode: "normal" + dictionary: "embedded://ko-dic" + # user_dictionary: + # path: "./resources/ko-dic_simple.csv" + # kind: "ko-dic" + keep_whitespace: false + +character_filters: + # Only the language-neutral normaliser: `japanese_iteration_mark` has nothing to do here. + - kind: "unicode_normalize" + args: + kind: "nfkc" + +token_filters: + # ko-dic's own stop-tag filter, not the Japanese one — the tag vocabulary is entirely different. + # EP/EF are verb endings and JKG is the possessive particle: the Korean counterparts of the + # 助動詞/助詞 the IPADIC config drops. + - kind: "korean_stop_tags" + args: + tags: + - "EP" + - "EF" + - "JKG" diff --git a/resources/lindera-unidic.yml b/resources/lindera-unidic.yml new file mode 100644 index 0000000..f3d92d3 --- /dev/null +++ b/resources/lindera-unidic.yml @@ -0,0 +1,33 @@ +segmenter: + mode: "normal" + dictionary: "embedded://unidic" + # user_dictionary: + # path: "./resources/unidic_simple.csv" + # kind: "unidic" + keep_whitespace: false + +character_filters: + - kind: "unicode_normalize" + args: + kind: "nfkc" + - kind: "japanese_iteration_mark" + args: + normalize_kanji: true + normalize_kana: true + +token_filters: + # Only the TOP-LEVEL tags, unlike the IPADIC config's deeper list. IPADIC and UniDic agree on + # these three but subdivide them differently below that level, so a deeper list written for one + # silently matches nothing in the other. + - kind: "japanese_stop_tags" + args: + tags: + - "助詞" + - "助動詞" + - "記号" + - kind: "japanese_katakana_stem" + args: + min: 3 + - kind: "remove_diacritical_mark" + args: + japanese: false