From a180efa0f31a3ba6a91ea505545f1ad1b638cc5a Mon Sep 17 00:00:00 2001 From: Tyge Løvset Date: Mon, 30 May 2022 15:43:27 +0200 Subject: Renamed utf8utils.c => utf8code.c --- examples/regex1.c | 2 +- examples/regex2.c | 2 +- examples/regex_match.c | 2 +- src/utf8code.c | 190 +++++++++++++++++++++++++++++++++++++++++++++++++ src/utf8tabs.c | 1 + src/utf8tabs.py | 2 +- src/utf8utils.c | 190 ------------------------------------------------- 7 files changed, 195 insertions(+), 194 deletions(-) create mode 100644 src/utf8code.c delete mode 100644 src/utf8utils.c diff --git a/examples/regex1.c b/examples/regex1.c index 84a5e28b..919bfd66 100644 --- a/examples/regex1.c +++ b/examples/regex1.c @@ -31,4 +31,4 @@ int main(int argc, char* argv[]) } #include "../src/cregex.c" -#include "../src/utf8utils.c" +#include "../src/utf8code.c" diff --git a/examples/regex2.c b/examples/regex2.c index 19c49ec2..c98c2e21 100644 --- a/examples/regex2.c +++ b/examples/regex2.c @@ -34,4 +34,4 @@ int main() } #include "../src/cregex.c" -#include "../src/utf8utils.c" +#include "../src/utf8code.c" diff --git a/examples/regex_match.c b/examples/regex_match.c index c2499733..4bb8bee5 100644 --- a/examples/regex_match.c +++ b/examples/regex_match.c @@ -29,4 +29,4 @@ int main() } #include "../src/cregex.c" -#include "../src/utf8utils.c" +#include "../src/utf8code.c" diff --git a/src/utf8code.c b/src/utf8code.c new file mode 100644 index 00000000..3b01ae39 --- /dev/null +++ b/src/utf8code.c @@ -0,0 +1,190 @@ +#include +#define i_header +#include + +#include "utf8tabs.h" +#include "utf8tabs.c" + +// https://news.ycombinator.com/item?id=15423674 +// https://gist.github.com/s4y/344a355f8c1f99c6a4cb2347ec4323cc + +void utf8_decode(utf8_decode_t *d, const uint8_t b) +{ + switch (d->state) { + case UTF8_OK: + if (b < 0x80) d->codep = b, d->size = 1; + else if (b < 0xC2) d->state = UTF8_ERROR, d->size = 0; + else if (b < 0xE0) d->state = 1, d->codep = b & 0x1F, d->size = 2; + else if (b < 0xF0) d->state = 2, d->codep = b & 0x0F, d->size = 3; + else if (b < 0xF5) d->state = 3, d->codep = b & 0x07, d->size = 4; + else d->state = UTF8_ERROR, d->size = 0; + break; + case 1: case 2: case 3: + if ((b & 0xC0) == 0x80) { + d->state -= 1; + d->codep = (d->codep << 6) | (b & 0x3F); + } else + d->state = UTF8_ERROR, d->size = 0; + } +} + +unsigned utf8_encode(char *out, uint32_t c) +{ + if (c < 0x80U) { + out[0] = (char) c; + return 1; + } else if (c < 0x0800U) { + out[0] = (char) ((c>>6 & 0x1F) | 0xC0); + out[1] = (char) ((c & 0x3F) | 0x80); + return 2; + } else if (c < 0x010000U) { + if ((c < 0xD800U) | (c >= 0xE000U)) { + out[0] = (char) ((c>>12 & 0x0F) | 0xE0); + out[1] = (char) ((c>>6 & 0x3F) | 0x80); + out[2] = (char) ((c & 0x3F) | 0x80); + return 3; + } + } else if (c < 0x110000U) { + out[0] = (char) ((c>>18 & 0x07) | 0xF0); + out[1] = (char) ((c>>12 & 0x3F) | 0x80); + out[2] = (char) ((c>>6 & 0x3F) | 0x80); + out[3] = (char) ((c & 0x3F) | 0x80); + return 4; + } + return 0; +} + +void utf8_peek(utf8_decode_t* d, const char *s) { + utf8_decode(d, (uint8_t)*s++); + switch (d->size) { + case 4: utf8_decode(d, (uint8_t)*s++); + case 3: utf8_decode(d, (uint8_t)*s++); + case 2: utf8_decode(d, (uint8_t)*s++); + } +} + +bool utf8_valid(const char* s) { + utf8_decode_t d = {UTF8_OK}; + while (*s) + utf8_decode(&d, (uint8_t)*s++); + return d.state == UTF8_OK; +} + +bool utf8_valid_n(const char* s, size_t n) { + utf8_decode_t d = {UTF8_OK}; + while ((n-- != 0) & (*s != 0)) + utf8_decode(&d, (uint8_t)*s++); + return d.state == UTF8_OK; +} + +uint32_t utf8_tolower(uint32_t c) { + for (size_t i=0; i < sizeof casefold/sizeof *casefold; ++i) { + if (c <= casefold[i].c1) { + if (c < casefold[i].c0) return c; + int d = casefold[i].m1 - casefold[i].c1; + if (d == 1) return c + ((casefold[i].c1 & 1) == (c & 1)); + return c + d; + } + } + return c; +} + +uint32_t utf8_toupper(uint32_t c) { + for (size_t i=0; i < sizeof cfold_low/sizeof *cfold_low; ++i) { + struct CaseFold cfold = casefold[cfold_low[i]]; + if (c <= cfold.m1) { + int d = cfold.m1 - cfold.c1; + if (c < (uint32_t)(cfold.c0 + d)) return c; + if (d == 1) return c - ((cfold.m1 & 1) == (c & 1)); + return c - d; + } + } + return c; +} + +bool utf8_isupper(uint32_t c) { + return utf8_tolower(c) != c; +} + +bool utf8_islower(uint32_t c) { + return utf8_toupper(c) != c; +} + +bool utf8_isspace(uint32_t c) { + static uint16_t t[] = {0x20, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x85, 0xA0, + 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000}; + for (size_t i=0; i= 0x2000) & (c <= 0x200A); +} + +bool utf8_isdigit(uint32_t c) { + return ((c >= '0') & (c <= '9')) || + ((c >= 0xFF10) & (c <= 0xFF19)); +} + +bool utf8_isxdigit(uint32_t c) { + static uint16_t t[] = {0x30, 0x39, 0x41, 0x46, 0x61, 0x66, 0xFF10, + 0xFF19, 0xFF21, 0xFF26, 0xFF41, 0xFF46}; + for (size_t i=1; i= t[i - 1]; + return false; +} + +bool utf8_isalnum(uint32_t c) { + if (c < 128) return isalnum(c) != 0; + if ((c >= 0xFF10) & (c <= 0xFF19)) return true; + return utf8_islower(c) || utf8_isupper(c); +} + +bool utf8_isalpha(uint32_t c) { + if (c < 128) return isalpha(c) != 0; + return utf8_islower(c) || utf8_isupper(c); +} + +static struct fnfold { + int (*conv_asc)(int); + uint32_t (*conv_u8)(uint32_t); +} +fn_tolower = {tolower, utf8_tolower}, +fn_toupper = {toupper, utf8_toupper}; + + +static cstr cstr_casefold(const cstr* self, struct fnfold fold) { + csview sv = cstr_sv(self); + cstr out = cstr_null; + char *buf = cstr_reserve(&out, sv.size*3/2); + uint32_t cp; size_t sz = 0; + utf8_decode_t d = {UTF8_OK}; + + for (; *sv.str; sv.str += d.size) { + utf8_peek(&d, sv.str); + switch (d.size) { + case 1: + buf[sz++] = (char)fold.conv_asc(*sv.str); + break; + default: + cp = fold.conv_u8(d.codep); + sz += utf8_encode(buf + sz, cp); + } + } + _cstr_set_size(&out, sz); + cstr_shrink_to_fit(&out); + return out; +} + +cstr cstr_tolower(const cstr* self) { + return cstr_casefold(self, fn_tolower); +} + +cstr cstr_toupper(const cstr* self) { + return cstr_casefold(self, fn_toupper); +} + +void cstr_lowercase(cstr* self) { + cstr_take(self, cstr_casefold(self, fn_tolower)); +} + +void cstr_uppercase(cstr* self) { + cstr_take(self, cstr_casefold(self, fn_toupper)); +} diff --git a/src/utf8tabs.c b/src/utf8tabs.c index 8168f78f..eccf0496 100644 --- a/src/utf8tabs.c +++ b/src/utf8tabs.c @@ -45,6 +45,7 @@ struct CaseFold casefold[] = { {42948, 42948, 42900}, {42949, 42949, 642}, {42950, 42950, 7566}, {42951, 42953, 42954}, {42960, 42962, 42963}, {42968, 42970, 42971}, {43888, 43913, 5049}, {65313, 65338, 65370}, }; // 188 + uint8_t cfold_low[] = { 0, 138, 10, 111, 2, 139, 3, 8, 4, 5, 6, 7, 9, 59, 12, 14, 16, 20, 49, 25, 56, 52, 29, 31, 33, 35, 37, 39, 50, 40, 41, 42, 43, 44, 45, 17, 46, 47, 48, 51, diff --git a/src/utf8tabs.py b/src/utf8tabs.py index 563180e3..9870e181 100644 --- a/src/utf8tabs.py +++ b/src/utf8tabs.py @@ -113,7 +113,7 @@ struct CaseFold casefold[] = {''') print('') count += 1 n += 1 - print('\n}; // %d' % (count)) + print('\n}; // %d\n' % (count)) return table diff --git a/src/utf8utils.c b/src/utf8utils.c deleted file mode 100644 index 3b01ae39..00000000 --- a/src/utf8utils.c +++ /dev/null @@ -1,190 +0,0 @@ -#include -#define i_header -#include - -#include "utf8tabs.h" -#include "utf8tabs.c" - -// https://news.ycombinator.com/item?id=15423674 -// https://gist.github.com/s4y/344a355f8c1f99c6a4cb2347ec4323cc - -void utf8_decode(utf8_decode_t *d, const uint8_t b) -{ - switch (d->state) { - case UTF8_OK: - if (b < 0x80) d->codep = b, d->size = 1; - else if (b < 0xC2) d->state = UTF8_ERROR, d->size = 0; - else if (b < 0xE0) d->state = 1, d->codep = b & 0x1F, d->size = 2; - else if (b < 0xF0) d->state = 2, d->codep = b & 0x0F, d->size = 3; - else if (b < 0xF5) d->state = 3, d->codep = b & 0x07, d->size = 4; - else d->state = UTF8_ERROR, d->size = 0; - break; - case 1: case 2: case 3: - if ((b & 0xC0) == 0x80) { - d->state -= 1; - d->codep = (d->codep << 6) | (b & 0x3F); - } else - d->state = UTF8_ERROR, d->size = 0; - } -} - -unsigned utf8_encode(char *out, uint32_t c) -{ - if (c < 0x80U) { - out[0] = (char) c; - return 1; - } else if (c < 0x0800U) { - out[0] = (char) ((c>>6 & 0x1F) | 0xC0); - out[1] = (char) ((c & 0x3F) | 0x80); - return 2; - } else if (c < 0x010000U) { - if ((c < 0xD800U) | (c >= 0xE000U)) { - out[0] = (char) ((c>>12 & 0x0F) | 0xE0); - out[1] = (char) ((c>>6 & 0x3F) | 0x80); - out[2] = (char) ((c & 0x3F) | 0x80); - return 3; - } - } else if (c < 0x110000U) { - out[0] = (char) ((c>>18 & 0x07) | 0xF0); - out[1] = (char) ((c>>12 & 0x3F) | 0x80); - out[2] = (char) ((c>>6 & 0x3F) | 0x80); - out[3] = (char) ((c & 0x3F) | 0x80); - return 4; - } - return 0; -} - -void utf8_peek(utf8_decode_t* d, const char *s) { - utf8_decode(d, (uint8_t)*s++); - switch (d->size) { - case 4: utf8_decode(d, (uint8_t)*s++); - case 3: utf8_decode(d, (uint8_t)*s++); - case 2: utf8_decode(d, (uint8_t)*s++); - } -} - -bool utf8_valid(const char* s) { - utf8_decode_t d = {UTF8_OK}; - while (*s) - utf8_decode(&d, (uint8_t)*s++); - return d.state == UTF8_OK; -} - -bool utf8_valid_n(const char* s, size_t n) { - utf8_decode_t d = {UTF8_OK}; - while ((n-- != 0) & (*s != 0)) - utf8_decode(&d, (uint8_t)*s++); - return d.state == UTF8_OK; -} - -uint32_t utf8_tolower(uint32_t c) { - for (size_t i=0; i < sizeof casefold/sizeof *casefold; ++i) { - if (c <= casefold[i].c1) { - if (c < casefold[i].c0) return c; - int d = casefold[i].m1 - casefold[i].c1; - if (d == 1) return c + ((casefold[i].c1 & 1) == (c & 1)); - return c + d; - } - } - return c; -} - -uint32_t utf8_toupper(uint32_t c) { - for (size_t i=0; i < sizeof cfold_low/sizeof *cfold_low; ++i) { - struct CaseFold cfold = casefold[cfold_low[i]]; - if (c <= cfold.m1) { - int d = cfold.m1 - cfold.c1; - if (c < (uint32_t)(cfold.c0 + d)) return c; - if (d == 1) return c - ((cfold.m1 & 1) == (c & 1)); - return c - d; - } - } - return c; -} - -bool utf8_isupper(uint32_t c) { - return utf8_tolower(c) != c; -} - -bool utf8_islower(uint32_t c) { - return utf8_toupper(c) != c; -} - -bool utf8_isspace(uint32_t c) { - static uint16_t t[] = {0x20, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x85, 0xA0, - 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000}; - for (size_t i=0; i= 0x2000) & (c <= 0x200A); -} - -bool utf8_isdigit(uint32_t c) { - return ((c >= '0') & (c <= '9')) || - ((c >= 0xFF10) & (c <= 0xFF19)); -} - -bool utf8_isxdigit(uint32_t c) { - static uint16_t t[] = {0x30, 0x39, 0x41, 0x46, 0x61, 0x66, 0xFF10, - 0xFF19, 0xFF21, 0xFF26, 0xFF41, 0xFF46}; - for (size_t i=1; i= t[i - 1]; - return false; -} - -bool utf8_isalnum(uint32_t c) { - if (c < 128) return isalnum(c) != 0; - if ((c >= 0xFF10) & (c <= 0xFF19)) return true; - return utf8_islower(c) || utf8_isupper(c); -} - -bool utf8_isalpha(uint32_t c) { - if (c < 128) return isalpha(c) != 0; - return utf8_islower(c) || utf8_isupper(c); -} - -static struct fnfold { - int (*conv_asc)(int); - uint32_t (*conv_u8)(uint32_t); -} -fn_tolower = {tolower, utf8_tolower}, -fn_toupper = {toupper, utf8_toupper}; - - -static cstr cstr_casefold(const cstr* self, struct fnfold fold) { - csview sv = cstr_sv(self); - cstr out = cstr_null; - char *buf = cstr_reserve(&out, sv.size*3/2); - uint32_t cp; size_t sz = 0; - utf8_decode_t d = {UTF8_OK}; - - for (; *sv.str; sv.str += d.size) { - utf8_peek(&d, sv.str); - switch (d.size) { - case 1: - buf[sz++] = (char)fold.conv_asc(*sv.str); - break; - default: - cp = fold.conv_u8(d.codep); - sz += utf8_encode(buf + sz, cp); - } - } - _cstr_set_size(&out, sz); - cstr_shrink_to_fit(&out); - return out; -} - -cstr cstr_tolower(const cstr* self) { - return cstr_casefold(self, fn_tolower); -} - -cstr cstr_toupper(const cstr* self) { - return cstr_casefold(self, fn_toupper); -} - -void cstr_lowercase(cstr* self) { - cstr_take(self, cstr_casefold(self, fn_tolower)); -} - -void cstr_uppercase(cstr* self) { - cstr_take(self, cstr_casefold(self, fn_toupper)); -} -- cgit v1.2.3