Fix `String#byteslice` with `MRB_UTF8_STRING` and some edge cases

Example:

  $ bin/mruby -e '
    p "あa".byteslice(1)
    p "bar".byteslice(3)
    p "bar".byteslice(4..0)
  '

  Before this patch:

    "a"
    ""
    RangeError (4..0 out of range)

  After this patch (same as Ruby):

    "\x81"
    nil
    nil

2 files changed, 74 insertions, 35 deletions

diff --git a/mrbgems/mruby-string-ext/src/string.c b/mrbgems/mruby-string-ext/src/string.c
index d9ebb7392..50a4e5582 100644
--- a/mrbgems/mruby-string-ext/src/string.c
+++ b/mrbgems/mruby-string-ext/src/string.c
@@ -42,44 +42,32 @@ mrb_str_setbyte(mrb_state *mrb, mrb_value str)
 static mrb_value
 mrb_str_byteslice(mrb_state *mrb, mrb_value str)
 {
-  mrb_value a1;
-  mrb_int len;
-
-  if (mrb_get_argc(mrb) == 2) {
-    mrb_int pos;
-    mrb_get_args(mrb, "ii", &pos, &len);
-    return mrb_str_substr(mrb, str, pos, len);
+  mrb_value a1, a2;
+  mrb_int str_len = RSTRING_LEN(str), beg, len;
+  mrb_bool empty = TRUE;
+
+  if (mrb_get_args(mrb, "o|o", &a1, &a2) == 2) {
+    beg = mrb_fixnum(mrb_to_int(mrb, a1));
+    len = mrb_fixnum(mrb_to_int(mrb, a2));
+    goto subseq;
   }
-  mrb_get_args(mrb, "o|i", &a1, &len);
-  switch (mrb_type(a1)) {
-  case MRB_TT_RANGE:
-    {
-      mrb_int beg;
-
-      len = RSTRING_LEN(str);
-      switch (mrb_range_beg_len(mrb, a1, &beg, &len, len, TRUE)) {
-      case MRB_RANGE_TYPE_MISMATCH:
-        break;
-      case MRB_RANGE_OK:
-        return mrb_str_substr(mrb, str, beg, len);
-      case MRB_RANGE_OUT:
-        mrb_raisef(mrb, E_RANGE_ERROR, "%S out of range", a1);
-        break;
-      }
-      return mrb_nil_value();
+  if (mrb_type(a1) == MRB_TT_RANGE) {
+    if (mrb_range_beg_len(mrb, a1, &beg, &len, str_len, TRUE) == MRB_RANGE_OK) {
+      goto subseq;
     }
-#ifndef MRB_WITHOUT_FLOAT
-  case MRB_TT_FLOAT:
-    a1 = mrb_fixnum_value((mrb_int)mrb_float(a1));
-    /* fall through */
-#endif
-  case MRB_TT_FIXNUM:
-    return mrb_str_substr(mrb, str, mrb_fixnum(a1), 1);
-  default:
-    mrb_raise(mrb, E_TYPE_ERROR, "wrong type of argument");
+    return mrb_nil_value();
+  }
+
+  beg = mrb_fixnum(mrb_to_int(mrb, a1));
+  len = 1;
+  empty = FALSE;
+subseq:
+  if (mrb_str_beg_len(str_len, &beg, &len) && (empty || len != 0)) {
+    return mrb_str_byte_subseq(mrb, str, beg, len);
+  }
+  else {
+    return mrb_nil_value();
   }
-  /* not reached */
-  return mrb_nil_value();
 }
 
 /*
diff --git a/mrbgems/mruby-string-ext/test/string.rb b/mrbgems/mruby-string-ext/test/string.rb
index 02777e594..bf633bcef 100644
--- a/mrbgems/mruby-string-ext/test/string.rb
+++ b/mrbgems/mruby-string-ext/test/string.rb
@@ -26,10 +26,61 @@ end
 
 assert('String#byteslice') do
   str1 = "hello"
+  str2 = "\u3042ab"  # "\xE3\x81\x82ab"
+
+  assert_equal("h", str1.byteslice(0))
   assert_equal("e", str1.byteslice(1))
+  assert_equal(nil, str1.byteslice(5))
   assert_equal("o", str1.byteslice(-1))
+  assert_equal(nil, str1.byteslice(-6))
+  assert_equal("\xE3", str2.byteslice(0))
+  assert_equal("\x81", str2.byteslice(1))
+  assert_equal(nil, str2.byteslice(5))
+  assert_equal("b", str2.byteslice(-1))
+  assert_equal(nil, str2.byteslice(-6))
+
+  assert_equal("", str1.byteslice(0, 0))
+  assert_equal(str1, str1.byteslice(0, 6))
+  assert_equal("el", str1.byteslice(1, 2))
+  assert_equal("", str1.byteslice(5, 1))
+  assert_equal("o", str1.byteslice(-1, 6))
+  assert_equal(nil, str1.byteslice(-6, 1))
+  assert_equal(nil, str1.byteslice(0, -1))
+  assert_equal("", str2.byteslice(0, 0))
+  assert_equal(str2, str2.byteslice(0, 6))
+  assert_equal("\x81\x82", str2.byteslice(1, 2))
+  assert_equal("", str2.byteslice(5, 1))
+  assert_equal("b", str2.byteslice(-1, 6))
+  assert_equal(nil, str2.byteslice(-6, 1))
+  assert_equal(nil, str2.byteslice(0, -1))
+
   assert_equal("ell", str1.byteslice(1..3))
   assert_equal("el", str1.byteslice(1...3))
+  assert_equal("h", str1.byteslice(0..0))
+  assert_equal("", str1.byteslice(5..0))
+  assert_equal("o", str1.byteslice(4..5))
+  assert_equal(nil, str1.byteslice(6..0))
+  assert_equal("", str1.byteslice(-1..0))
+  assert_equal("llo", str1.byteslice(-3..5))
+  assert_equal("\x81\x82a", str2.byteslice(1..3))
+  assert_equal("\x81\x82", str2.byteslice(1...3))
+  assert_equal("\xE3", str2.byteslice(0..0))
+  assert_equal("", str2.byteslice(5..0))
+  assert_equal("b", str2.byteslice(4..5))
+  assert_equal(nil, str2.byteslice(6..0))
+  assert_equal("", str2.byteslice(-1..0))
+  assert_equal("\x82ab", str2.byteslice(-3..5))
+
+  assert_raise(ArgumentError) { str1.byteslice }
+  assert_raise(ArgumentError) { str1.byteslice(1, 2, 3) }
+  assert_raise(TypeError) { str1.byteslice("1") }
+  assert_raise(TypeError) { str1.byteslice("1", 2) }
+  assert_raise(TypeError) { str1.byteslice(1, "2") }
+  assert_raise(TypeError) { str1.byteslice(1..2, 3) }
+
+  skip unless Object.const_defined?(:Float)
+  assert_equal("o", str1.byteslice(4.0))
+  assert_equal("\x82ab", str2.byteslice(2.0, 3.0))
 end
 
 assert('String#dump') do