From 8883fc8108428878d3d6291ba8981cf6df72499c Mon Sep 17 00:00:00 2001 From: Tyge Løvset Date: Fri, 10 Jun 2022 11:29:17 +0200 Subject: utf8 fixes and improvements. Some api changes. --- docs/cstr_api.md | 5 +++-- docs/csview_api.md | 10 +++++----- examples/cstr_match.c | 5 +++-- examples/utf8replace_c.c | 2 +- include/stc/cstr.h | 17 ++++++++++------- include/stc/csview.h | 10 +++++----- include/stc/utf8.h | 13 +++++++------ src/utf8code.c | 4 ++-- 8 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/cstr_api.md b/docs/cstr_api.md index cfc807f6..efbb0c5e 100644 --- a/docs/cstr_api.md +++ b/docs/cstr_api.md @@ -87,8 +87,9 @@ bool cstr_getdelim(cstr *self, int delim, FILE *stream); // does no ```c size_t cstr_u8size(cstr s); // number of utf8 codepoints size_t cstr_u8size_n(cstr s, size_t nbytes); // utf8 size within n bytes -const char* cstr_at(const cstr* self, size_t u8idx); // byte position at utf8 index -csview cstr_view_at(const cstr* self, size_t u8idx); // utf8 codepoint at utf8 pos as csview +size_t cstr_bytepos(cstr s, size_t u8idx); // byte pos offset at utf8 index +const char* cstr_at(const cstr* self, size_t u8idx); // char* position at utf8 index +csview cstr_chr(const cstr* self, size_t u8idx); // utf8 character at utf8 pos as csview // iterate utf8 codepoints cstr_iter cstr_begin(const cstr* self); diff --git a/docs/csview_api.md b/docs/csview_api.md index 4baba506..fcdb9f72 100644 --- a/docs/csview_api.md +++ b/docs/csview_api.md @@ -51,24 +51,24 @@ csview csview_token(csview sv, csview sep, size_t* start); // see sp #### UTF8 methods ```c -size_t csview_size_u8(csview sv); -csview csview_substr_u8(csview sv, size_t u8pos, size_t u8len); +size_t csview_u8size(csview sv); +csview csview_u8substr(csview sv, size_t u8pos, size_t u8len); csview_iter csview_begin(const csview* self); csview_iter csview_end(const csview* self); void csview_next(csview_iter* it); // utf8 codepoint step, not byte! // requires linking with src/utf8code.c: -bool csview_valid_u8(csview sv); +bool csview_valid_utf8(csview sv); // from utf8.h/utf8code.c: bool utf8_valid(const char* s); bool utf8_valid_n(const char* s, size_t n); size_t utf8_size(const char *s); -size_t utf8_size_n(const char *s, size_t n); // number of UTF8 codepoints within n bytes +size_t utf8_size_n(const char *s, size_t nbytes); // number of UTF8 codepoints within n bytes const char* utf8_at(const char *s, size_t index); // from UTF8 index to char* position size_t utf8_pos(const char* s, size_t index); // from UTF8 index to byte index position -unsigned utf8_codep_size(const char* s); // 0-4 (0 if s[0] is illegal utf8) +unsigned utf8_chr_size(const char* s); // 0-4 (0 if s[0] is illegal utf8) uint32_t utf8_decode(utf8_decode_t *d, uint8_t byte); // decode next byte to utf8, return state. unsigned utf8_encode(char *out, uint32_t cp); // encode unicode cp into out buffer ``` diff --git a/examples/cstr_match.c b/examples/cstr_match.c index 8130b401..3965b36a 100644 --- a/examples/cstr_match.c +++ b/examples/cstr_match.c @@ -14,8 +14,9 @@ int main() printf("ends_with: %d\n", cstr_ends_with(ss, ".JPG")); cstr s1 = cstr_new("hell😀 w😀rl🐨"); - csview ch1 = cstr_at(&s1, 10); - csview ch2 = cstr_at_u8(&s1, 10); + csview ch1 = cstr_chr(&s1, 7); + csview ch2 = cstr_chr(&s1, 10); + printf("%s\nsize: %" PRIuMAX ", %" PRIuMAX "\n", cstr_str(&s1), cstr_u8size(s1), cstr_size(s1)); printf("ch1: %" c_PRIsv "\n", c_ARGsv(ch1)); printf("ch2: %" c_PRIsv "\n", c_ARGsv(ch2)); } diff --git a/examples/utf8replace_c.c b/examples/utf8replace_c.c index c38b37e4..52eb5bd2 100644 --- a/examples/utf8replace_c.c +++ b/examples/utf8replace_c.c @@ -8,7 +8,7 @@ int main() { cstr_replace_sv( &hello, - csview_substr_u8(cstr_sv(&hello), 7, 1), + csview_u8substr(cstr_sv(&hello), 7, 1), c_sv("🐨") ); printf("%s\n", cstr_str(&hello)); diff --git a/include/stc/cstr.h b/include/stc/cstr.h index 2dc2ccae..41db1cd3 100644 --- a/include/stc/cstr.h +++ b/include/stc/cstr.h @@ -189,21 +189,24 @@ STC_INLINE size_t cstr_u8size(cstr s) STC_INLINE size_t cstr_u8size_n(cstr s, size_t nbytes) { return utf8_size_n(cstr_str(&s), nbytes); } -STC_INLINE csview cstr_view_at(const cstr* self, size_t u8idx) { +STC_INLINE size_t cstr_bytepos(const cstr* self, size_t u8idx) + { return utf8_pos(cstr_str(self), u8idx); } + +STC_INLINE const char* cstr_at(const cstr* self, size_t u8idx) + { return utf8_at(cstr_str(self), u8idx); } + +STC_INLINE csview cstr_chr(const cstr* self, size_t u8idx) { csview sv = cstr_sv(self); sv.str = utf8_at(sv.str, u8idx); - sv.size = utf8_codep_size(sv.str); + sv.size = utf8_chr_size(sv.str); return sv; } -STC_INLINE const char* cstr_at(const cstr* self, size_t u8idx) - { return utf8_at(cstr_str(self), u8idx); } - // utf8 iterator STC_INLINE cstr_iter cstr_begin(const cstr* self) { const char* str = cstr_str(self); - return c_make(cstr_iter){.chr = {str, utf8_codep_size(str)}}; + return c_make(cstr_iter){.chr = {str, utf8_chr_size(str)}}; } STC_INLINE cstr_iter cstr_end(const cstr* self) { csview sv = cstr_sv(self); @@ -211,7 +214,7 @@ STC_INLINE cstr_iter cstr_end(const cstr* self) { } STC_INLINE void cstr_next(cstr_iter* it) { it->ref += it->chr.size; - it->chr.size = utf8_codep_size(it->ref); + it->chr.size = utf8_chr_size(it->ref); } diff --git a/include/stc/csview.h b/include/stc/csview.h index 6cfd6e82..6d12901b 100644 --- a/include/stc/csview.h +++ b/include/stc/csview.h @@ -81,25 +81,25 @@ STC_INLINE csview csview_slice(csview sv, size_t p1, size_t p2) { /* iterator */ STC_INLINE csview_iter csview_begin(const csview* self) - { return c_make(csview_iter){.chr = {self->str, utf8_codep_size(self->str)}}; } + { return c_make(csview_iter){.chr = {self->str, utf8_chr_size(self->str)}}; } STC_INLINE csview_iter csview_end(const csview* self) { return c_make(csview_iter){self->str + self->size}; } STC_INLINE void csview_next(csview_iter* it) - { it->ref += it->chr.size; it->chr.size = utf8_codep_size(it->ref); } + { it->ref += it->chr.size; it->chr.size = utf8_chr_size(it->ref); } /* utf8 */ -STC_INLINE size_t csview_size_u8(csview sv) +STC_INLINE size_t csview_u8size(csview sv) { return utf8_size_n(sv.str, sv.size); } -STC_INLINE csview csview_substr_u8(csview sv, size_t u8pos, size_t u8len) { +STC_INLINE csview csview_u8substr(csview sv, size_t u8pos, size_t u8len) { sv.str = utf8_at(sv.str, u8pos); sv.size = utf8_pos(sv.str, u8len); return sv; } -STC_INLINE bool csview_valid_u8(csview sv) // depends on src/utf8code.c +STC_INLINE bool csview_valid_utf8(csview sv) // depends on src/utf8code.c { return utf8_valid_n(sv.str, sv.size); } diff --git a/include/stc/utf8.h b/include/stc/utf8.h index b7edd2cb..31ea3aa9 100644 --- a/include/stc/utf8.h +++ b/include/stc/utf8.h @@ -37,7 +37,7 @@ uint32_t utf8_casefold(uint32_t c); uint32_t utf8_tolower(uint32_t c); uint32_t utf8_toupper(uint32_t c); bool utf8_valid(const char* s); -bool utf8_valid_n(const char* s, size_t n); +bool utf8_valid_n(const char* s, size_t nbytes); int utf8_icmp_n(size_t u8max, const char* s1, size_t n1, const char* s2, size_t n2); unsigned utf8_encode(char *out, uint32_t c); @@ -60,7 +60,7 @@ STC_INLINE int utf8_icmp(const char* s1, const char* s2) { } /* number of characters in the utf8 codepoint from s */ -STC_INLINE unsigned utf8_codep_size(const char *s) { +STC_INLINE unsigned utf8_chr_size(const char *s) { unsigned b = (uint8_t)*s; if (b < 0x80) return 1; if (b < 0xC2) return 0; @@ -74,14 +74,15 @@ STC_INLINE unsigned utf8_codep_size(const char *s) { STC_INLINE size_t utf8_size(const char *s) { size_t size = 0; while (*s) - size += (*s++ & 0xC0) != 0x80; + size += (*++s & 0xC0) != 0x80; return size; } -STC_INLINE size_t utf8_size_n(const char *s, size_t n) { +STC_INLINE size_t utf8_size_n(const char *s, size_t nbytes) { size_t size = 0; - while ((n-- != 0) & (*s != 0)) - size += (*s++ & 0xC0) != 0x80; + while ((nbytes-- != 0) & (*s != 0)) { + size += (*++s & 0xC0) != 0x80; + } return size; } diff --git a/src/utf8code.c b/src/utf8code.c index a6ecdb65..f64ede70 100644 --- a/src/utf8code.c +++ b/src/utf8code.c @@ -63,9 +63,9 @@ bool utf8_valid(const char* s) { return d.state == 0; } -bool utf8_valid_n(const char* s, size_t n) { +bool utf8_valid_n(const char* s, size_t nbytes) { utf8_decode_t d = {.state=0}; - while ((n-- != 0) & (*s != 0)) + while ((nbytes-- != 0) & (*s != 0)) utf8_decode(&d, (uint8_t)*s++); return d.state == 0; } -- cgit v1.2.3