Final utf8 api, and docs.

3 files changed, 104 insertions, 60 deletions

diff --git a/docs/csview_api.md b/docs/csview_api.md
index a6bd7573..defc29fd 100644
--- a/docs/csview_api.md
+++ b/docs/csview_api.md
@@ -26,68 +26,89 @@ All csview definitions and prototypes are available by including a single header
 ## Methods
 
 ```c
-csview        c_sv(const char literal_only[]);                      // alias for csview_new
-csview        csview_new(const char literal_only[]);                // make csview from literal, no strlen()
-csview        csview_from_s(cstr s);                                // same as cstr_sv()
-csview        csview_from(const char* str);                         // make csview from const char*
-csview        csview_from_n(const char* str, size_t n);             // construct 
-
-size_t        csview_size(csview sv);
-size_t        csview_length(csview sv);
-bool          csview_empty(csview sv);
-char          csview_front(csview sv);
-char          csview_back(csview sv);
-
-void          csview_clear(csview* self);
-
-csview        csview_substr(csview sv, intptr_t pos, size_t n);    // negative pos count from end
-csview        csview_slice(csview sv, intptr_t p1, intptr_t p2);   // negative p1, p2 count from end
-csview        csview_token(csview sv, csview sep, size_t* start);  // see split example below.
-
-bool          csview_equals(csview sv, csview sv2);
-size_t        csview_find(csview sv, csview needle);
-bool          csview_contains(csview sv, csview needle);
-bool          csview_starts_with(csview sv, csview sub);
-bool          csview_ends_with(csview sv, csview sub);
-
-csview_iter   csview_begin(const csview* self);
-csview_iter   csview_end(const csview* self);
-void          csview_next(csview_iter* it);
+csview          c_sv(const char literal_only[]);                      // alias for csview_new
+csview          csview_new(const char literal_only[]);                // make csview from literal, no strlen()
+csview          csview_from_s(cstr s);                                // same as cstr_sv()
+csview          csview_from(const char* str);                         // make csview from const char*
+csview          csview_from_n(const char* str, size_t n);             // construct 
+
+size_t          csview_size(csview sv);
+size_t          csview_length(csview sv);
+bool            csview_empty(csview sv);
+char            csview_front(csview sv);
+char            csview_back(csview sv);
+
+void            csview_clear(csview* self);
+
+csview          csview_substr(csview sv, intptr_t pos, size_t n);    // negative pos count from end
+csview          csview_slice(csview sv, intptr_t p1, intptr_t p2);   // negative p1, p2 count from end
+csview          csview_token(csview sv, csview sep, size_t* start);  // see split example below.
+
+bool            csview_equals(csview sv, csview sv2);
+size_t          csview_find(csview sv, csview needle);
+bool            csview_contains(csview sv, csview needle);
+bool            csview_starts_with(csview sv, csview sub);
+bool            csview_ends_with(csview sv, csview sub);
+
+csview_iter     csview_begin(const csview* self);
+csview_iter     csview_end(const csview* self);
+void            csview_next(csview_iter* it);                        // NB: UTF8 codepoint step, not byte!
 ```
+
+#### UTF8 methods
+```
+bool            utf8_valid_sv(csview sv);
+size_t          utf8_size_sv(csview sv);
+csview          utf8_substr(const char* str, size_t pos, size_t n);
+
+bool            utf8_valid(const char* s);
+size_t          utf8_size(const char *s);
+size_t          utf8_size_n(const char *s, size_t n);           // number of UTF8 codepoints within n bytes
+const char*     utf8_at(const char *s, size_t index);           // from UTF8 index to char* position
+size_t          utf8_pos(const char* s, size_t index);          // from UTF8 index to byte index position
+const char*     utf8_next(const char *s);                       // next codepoint as char*; NULL if *s == 0
+uint32_t        utf8_peek(const char *s);                       // next codepoint as uint32_t
+
+size_t          utf8_codepoint_size(const char* s);             // 1-4 (0 if s[0] is illegal first cp char)
+uint32_t        utf8_decode(uint32_t *state, uint32_t *codep, const uint32_t byte); // decode next utf8 codepoint.
+```
+
 #### Extended cstr methods
 ```c
-cstr          cstr_from_sv(csview sv);                        // construct cstr from csview
-csview        cstr_to_sv(const cstr* self);                   // convert to csview from cstr*
-cstr          cstr_from_replace_all_sv(csview sv, csview find, csview replace);
-
-csview        cstr_sv(cstr s);                                // convert to csview from cstr
-csview        cstr_substr(cstr s, intptr_t pos, size_t n);    // negative pos counts from end
-csview        cstr_slice(cstr s, intptr_t p1, intptr_t p2);   // negative p1, p2 counts from end
-
-cstr*         cstr_assign_sv(cstr* self, csview sv);
-cstr*         cstr_append_sv(cstr* self, csview sv);
-void          cstr_insert_sv(cstr* self, size_t pos, csview sv);
-void          cstr_replace_sv(cstr* self, size_t pos, size_t len, csview sv);
-
-bool          cstr_equals_sv(cstr s, csview sv);
-size_t        cstr_find_sv(cstr s, csview needle);
-bool          cstr_contains_sv(cstr s, csview needle);
-bool          cstr_starts_with_sv(cstr s, csview sub);
-bool          cstr_ends_with_sv(cstr s, csview sub);
+cstr            cstr_from_sv(csview sv);                        // construct cstr from csview
+csview          cstr_to_sv(const cstr* self);                   // convert to csview from cstr*
+cstr            cstr_from_replace_all_sv(csview sv, csview find, csview replace);
+
+csview          cstr_sv(cstr s);                                // convert to csview from cstr
+csview          cstr_substr(cstr s, intptr_t pos, size_t n);    // negative pos counts from end
+csview          cstr_slice(cstr s, intptr_t p1, intptr_t p2);   // negative p1, p2 counts from end
+
+cstr*           cstr_assign_sv(cstr* self, csview sv);
+cstr*           cstr_append_sv(cstr* self, csview sv);
+void            cstr_insert_sv(cstr* self, size_t pos, csview sv);
+void            cstr_replace_sv(cstr* self, size_t pos, size_t len, csview sv);
+
+bool            cstr_equals_sv(cstr s, csview sv);
+size_t          cstr_find_sv(cstr s, csview needle);
+bool            cstr_contains_sv(cstr s, csview needle);
+bool            cstr_starts_with_sv(cstr s, csview sub);
+bool            cstr_ends_with_sv(cstr s, csview sub);
 ```
+
 #### Helper methods
 ```c
-int           csview_cmp(const csview* x, const csview* y);
-bool          csview_eq(const csview* x, const csview* y);
-uint64_t      csview_hash(const csview* x, size_t dummy);
+int             csview_cmp(const csview* x, const csview* y);
+bool            csview_eq(const csview* x, const csview* y);
+uint64_t        csview_hash(const csview* x, size_t dummy);
 ```
+
 ## Types
 
 | Type name       | Type definition                           | Used to represent...     |
 |:----------------|:------------------------------------------|:-------------------------|
 | `csview`        | `struct { const char *str; size_t size }` | The string view type     |
 | `csview_value`  | `char`                                    | The string element type  |
-| `csview_iter`   | `struct { csview_value *ref; }`           | csview iterator          |
+| `csview_iter`   | `struct { csview_value *ref; }`           | UTF8 iterator            |
 
 ## Constants and macros
 
@@ -127,7 +148,31 @@ think live details
 red Apples
 ```
 
-### Example 2: csview tokenizer (string split)
+### Example 2: UTF8 handling
+```c
+#include <stc/cstr.h>
+#include <stc/csview.h>
+
+int main()
+{
+    c_auto (cstr, s1) {
+        s1 = cstr_new("hell😀 w😀rld");
+        cstr_replace_sv(&s1, utf8_substr(s1.str, 7, 1), c_sv("x"));
+        printf("%s\n", s1.str);
+
+        csview sv = csview_from_s(s1);
+        c_foreach (i, csview, sv)
+            printf(c_PRIsv ",", c_ARGsv(i.cp));
+    }
+}
+```
+Output:
+```
+hell😀 wxrld
+h,e,l,l,😀, ,w,x,r,l,d,
+```
+
+### Example 3: csview tokenizer (string split)
 Splits strings into tokens. *print_split()* makes **no** memory allocations or *strlen()* calls,
 and does not depend on null-terminated strings. *string_split()* function returns a vector of cstr.
 ```c
@@ -186,4 +231,3 @@ Output:
 "string"
 "now"
 ""
-
diff --git a/include/stc/csview.h b/include/stc/csview.h
index e65821bb..f3a0ae11 100644
--- a/include/stc/csview.h
+++ b/include/stc/csview.h
@@ -69,18 +69,18 @@ STC_INLINE bool         csview_ends_with(csview sv, csview sub)
                             { if (sub.size > sv.size) return false;

                               return !memcmp(sv.str + sv.size - sub.size, sub.str, sub.size); }

 STC_INLINE csview_iter  csview_begin(const csview* self)

-                            { return c_make(csview_iter){.cp = {self->str, utf8_codepoint_size(*self->str)}}; }

+                            { return c_make(csview_iter){.cp = {self->str, utf8_codepoint_size(self->str)}}; }

 STC_INLINE csview_iter  csview_end(const csview* self)

                             { return c_make(csview_iter){self->str + self->size}; }

 STC_INLINE void         csview_next(csview_iter* it)

-                            { it->ref += it->cp.size; it->cp.size = utf8_codepoint_size(*it->ref); }

+                            { it->ref += it->cp.size; it->cp.size = utf8_codepoint_size(it->ref); }

 

 /* utf8 */

-STC_INLINE bool csview_valid_utf8(csview sv)

-    { return utf8_valid(sv.str); }

+STC_INLINE bool utf8_valid_sv(csview sv)

+    { return utf8_size_n(sv.str, sv.size) != SIZE_MAX; }

 

-STC_INLINE size_t csview_size_utf8(csview sv)

-    { return utf8_size(sv.str); }

+STC_INLINE size_t utf8_size_sv(csview sv)

+    { return utf8_size_n(sv.str, sv.size); }

 

 STC_INLINE csview utf8_substr(const char* str, size_t pos, size_t n) {

     csview sv;

diff --git a/include/stc/utf8.h b/include/stc/utf8.h
index 93f8461a..4270c8c7 100644
--- a/include/stc/utf8.h
+++ b/include/stc/utf8.h
@@ -49,8 +49,8 @@ STC_INLINE uint32_t utf8_peek(const char *s) {
     return codepoint;

 }

 

-STC_INLINE size_t utf8_codepoint_size(char c) {

-    uint8_t u = (uint8_t)c;

+STC_INLINE size_t utf8_codepoint_size(const char* s) {

+    uint8_t u = *(const uint8_t *)s;

     size_t ret = (u & 0xF0) == 0xE0;

     ret += ret << 1;                       // 3

     ret |= u < 0x80;                       // 1

@@ -60,7 +60,7 @@ STC_INLINE size_t utf8_codepoint_size(char c) {
 }

 

 STC_INLINE const char *utf8_next(const char *s) {

-    const char* t = s + utf8_codepoint_size(*s);

+    const char* t = s + utf8_codepoint_size(s);

     

     uintptr_t p = (uintptr_t)t;

     p &= (uintptr_t) -(*s != 0);