Skip to content

fold U+0345 after canonical ordering in utf8proc_decompose_custom #379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shamsManna wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fold U+0345 after canonical ordering in utf8proc_decompose_custom #379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions test/misc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,34 @@

#include "tests.h"

static void issue3(void) /* #3 */
{
/* U+0345 has combining class 240 but case-folds to U+03B9, a starter, so it
must only be folded once the marks next to it are in canonical order;
these four strings are canonically equivalent */
utf8proc_uint8_t input[][8] = {
{0xcf, 0x89, 0xcd, 0x85, 0xcd, 0x82, 0x00}, /* "\u03c9\u0345\u0342" */
{0xcf, 0x89, 0xcd, 0x82, 0xcd, 0x85, 0x00}, /* "\u03c9\u0342\u0345" */
{0xe1, 0xbf, 0xb7, 0x00}, /* "\u1ff7" */
{0xe1, 0xbf, 0xb3, 0xcd, 0x82, 0x00} /* "\u1ff3\u0342" */
};
utf8proc_uint8_t nfkc_casefold[] = {0xe1, 0xbf, 0xb6, 0xce, 0xb9, 0x00}; /* "\u1ff6\u03b9" */
utf8proc_uint8_t nfd_casefold[] = {0xcf, 0x89, 0xcd, 0x82, 0xce, 0xb9, 0x00}; /* "\u03c9\u0342\u03b9" */
/* NFKC_Casefold drops a default ignorable only after the canonical
decomposition, so it still keeps the U+0345 from moving past the U+0301 */
utf8proc_uint8_t ignorable[] = {0xe1, 0xbe, 0xb3, 0xef, 0xb8, 0x8f, 0xcc, 0x81, 0x00}; /* "\u1fb3\ufe0f\u0301" */
utf8proc_uint8_t ignorable_casefold[] = {0xce, 0xb1, 0xce, 0xaf, 0x00}; /* "\u03b1\u03af" */
utf8proc_uint8_t *output;
size_t i;

for (i = 0; i < sizeof(input) / sizeof(input[0]); i++) {
check_compare("NFKC_Casefold", input[i], nfkc_casefold, utf8proc_NFKC_Casefold(input[i]), 1);
utf8proc_map(input[i], 0, &output, UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_CASEFOLD);
check_compare("NFD+casefold", input[i], nfd_casefold, output, 1);
}
check_compare("NFKC_Casefold", ignorable, ignorable_casefold, utf8proc_NFKC_Casefold(ignorable), 1);
}

static void issue128(void) /* #128 */
{
utf8proc_uint8_t input[] = {0x72, 0xcc, 0x87, 0xcc, 0xa3, 0x00}; /* "r\u0307\u0323" */
Expand Down Expand Up @@ -89,6 +117,7 @@ static void issue288(void) /* #288 */

int main(void)
{
issue3();
issue128();
issue102();
issue317();
Expand Down
85 changes: 80 additions & 5 deletions utf8proc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,28 @@ UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
options & ~(unsigned int)UTF8PROC_LUMP, last_boundclass)

UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, unsigned int options, utf8proc_int32_t *last_boundclass) {
/* return the first codepoint of a canonical decomposition that ends in U+0345,
or -1 */
static utf8proc_int32_t ypogegrammeni_base(const utf8proc_property_t *property) {
const utf8proc_uint16_t *entry;
utf8proc_int32_t base;
/* canonical decompositions are one or two codepoints long */
if (property->decomp_seqindex == UINT16_MAX || property->decomp_type ||
(property->decomp_seqindex >> 14) != 1) return -1;
entry = &utf8proc_sequences[property->decomp_seqindex & 0x3FFF];
base = seqindex_decode_entry(&entry);
entry++;
return seqindex_decode_entry(&entry) == 0x0345 ? base : -1;
}

/* U+0345 COMBINING GREEK YPOGEGRAMMENI has combining class 240, but it
case-folds to U+03B9, which is a starter. Folding it before the canonical
ordering would put the marks that follow it on the iota instead of the base
letter, so canonically equivalent strings would fold differently (#3).
With unfold_0345, U+0345 is left unfolded, also where it ends the canonical
decomposition of a precomposed letter, and utf8proc_decompose_custom folds
it with fold_0345_run once its run of marks has ended. */
static utf8proc_ssize_t decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, unsigned int options, utf8proc_int32_t *last_boundclass, utf8proc_bool unfold_0345) {
const utf8proc_property_t *property;
utf8proc_propval_t category;
utf8proc_int32_t hangul_sindex;
Expand Down Expand Up @@ -501,12 +522,25 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc,
}
}
if (options & UTF8PROC_STRIPMARK) {
if (category == UTF8PROC_CATEGORY_MN ||
/* an unfolded U+0345 stands in for U+03B9, so it is kept */
if ((category == UTF8PROC_CATEGORY_MN ||
category == UTF8PROC_CATEGORY_MC ||
category == UTF8PROC_CATEGORY_ME) return 0;
category == UTF8PROC_CATEGORY_ME) && !(unfold_0345 && uc == 0x0345)) return 0;
}
if (options & UTF8PROC_CASEFOLD) {
if (property->casefold_seqindex != UINT16_MAX) {
if (unfold_0345) {
utf8proc_int32_t base = ypogegrammeni_base(property);
utf8proc_ssize_t written = 0;
if (base >= 0) {
written = utf8proc_decompose_char(base, dst, bufsize, options, last_boundclass);
if (written < 0) return written;
}
if (base >= 0 || uc == 0x0345) {
if (bufsize > written) dst[written] = 0x0345;
return written + 1;
}
}
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
}
}
Expand All @@ -530,6 +564,19 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc,
return 1;
}

UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, unsigned int options, utf8proc_int32_t *last_boundclass) {
return decompose_char(uc, dst, bufsize, options, last_boundclass, 0);
}

/* fold the U+0345 in a run of marks to U+03B9, after moving them behind the
other marks, which is where canonical ordering puts combining class 240 */
static void fold_0345_run(utf8proc_int32_t *run, utf8proc_ssize_t len) {
utf8proc_ssize_t rpos, wpos = 0;
for (rpos = 0; rpos < len; rpos++)
if (run[rpos] != 0x0345) run[wpos++] = run[rpos];
while (wpos < len) run[wpos++] = 0x03B9;
}

UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, unsigned int options
Expand All @@ -554,6 +601,11 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
utf8proc_ssize_t rpos = 0;
utf8proc_ssize_t decomp_result;
utf8proc_int32_t boundclass = UTF8PROC_BOUNDCLASS_START;
/* see decompose_char; the CHARBOUND sentinels would keep U+0345 from being
reordered, so that case is left as it was */
utf8proc_bool unfold_0345 = (options & UTF8PROC_CASEFOLD) &&
(options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && !(options & UTF8PROC_CHARBOUND);
utf8proc_ssize_t run_0345 = -1; /* start of a run of marks with an unfolded U+0345 */
while (1) {
if (options & UTF8PROC_NULLTERM) {
utf8proc_ssize_t nread = utf8proc_iterate(str + rpos, -1, &uc);
Expand All @@ -571,16 +623,39 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
decomp_result = utf8proc_decompose_char(
decomp_result = decompose_char(
uc, buffer ? buffer+wpos : buffer, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
&boundclass, unfold_0345
);
if (decomp_result < 0) return decomp_result;
/* prohibiting integer overflows due to too long strings: */
if (decomp_result > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2) - wpos)
return UTF8PROC_ERROR_OVERFLOW;
wpos += decomp_result;
if (unfold_0345 && buffer && bufsize >= wpos) {
if (run_0345 >= 0) {
/* the run ends at a starter of the canonical decomposition, also
one that was dropped (e.g. a default ignorable with
UTF8PROC_IGNORE) or mapped to a mark (e.g. U+FF9E with
UTF8PROC_COMPAT) */
const utf8proc_property_t *property = unsafe_get_property(uc);
utf8proc_ssize_t end = wpos - decomp_result;
if (property->combining_class ||
(property->decomp_seqindex != UINT16_MAX && !property->decomp_type)) {
while (end < wpos && unsafe_get_property(buffer[end])->combining_class) end++;
if (end == wpos) end = -1;
}
if (end >= 0) {
fold_0345_run(buffer + run_0345, end - run_0345);
run_0345 = -1;
}
}
if (run_0345 < 0 && decomp_result && buffer[wpos-1] == 0x0345)
run_0345 = wpos - 1;
}
}
if (run_0345 >= 0 && bufsize >= wpos)
fold_0345_run(buffer + run_0345, wpos - run_0345);
}
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
utf8proc_ssize_t pos = 0;
Expand Down
Loading