Embed small size array elements in the heap.

It reduces the memory consumption and sometimes improve the
performance as well.  For example, the consumed memory size
of `bench/bm_ao_render.rb` is reduced from 1.2GB to 1GB, and
its total execution time become 18.795 sec from 22.229 sec.

1 files changed, 216 insertions, 169 deletions

diff --git a/src/array.c b/src/array.c
index 7c07295c7..9124ff950 100644
--- a/src/array.c
+++ b/src/array.c
@@ -28,9 +28,15 @@ ary_new_capa(mrb_state *mrb, mrb_int capa)
   blen = capa * sizeof(mrb_value);
 
   a = (struct RArray*)mrb_obj_alloc(mrb, MRB_TT_ARRAY, mrb->array_class);
-  a->ptr = (mrb_value *)mrb_malloc(mrb, blen);
-  a->aux.capa = capa;
-  a->len = 0;
+  if (capa <= MRB_ARY_EMBED_LEN_MAX) {
+    ARY_SET_EMBED_FLAG(a);
+    /* ARY_SET_EMBED_LEN(a, 0); */
+  }
+  else {
+    a->as.heap.ptr = (mrb_value *)mrb_malloc(mrb, blen);
+    a->as.heap.aux.capa = capa;
+    a->as.heap.len = 0;
+  }
 
   return a;
 }
@@ -77,8 +83,8 @@ mrb_ary_new_from_values(mrb_state *mrb, mrb_int size, const mrb_value *vals)
 {
   struct RArray *a = ary_new_capa(mrb, size);
 
-  array_copy(a->ptr, vals, size);
-  a->len = size;
+  array_copy(ARY_PTR(a), vals, size);
+  ARY_SET_LEN(a, size);
 
   return mrb_obj_value(a);
 }
@@ -89,9 +95,9 @@ mrb_assoc_new(mrb_state *mrb, mrb_value car, mrb_value cdr)
   struct RArray *a;
 
   a = ary_new_capa(mrb, 2);
-  a->ptr[0] = car;
-  a->ptr[1] = cdr;
-  a->len = 2;
+  ARY_PTR(a)[0] = car;
+  ARY_PTR(a)[1] = cdr;
+  ARY_SET_LEN(a, 2);
   return mrb_obj_value(a);
 }
 
@@ -119,25 +125,25 @@ ary_modify(mrb_state *mrb, struct RArray *a)
   ary_modify_check(mrb, a);
 
   if (ARY_SHARED_P(a)) {
-    mrb_shared_array *shared = a->aux.shared;
+    mrb_shared_array *shared = a->as.heap.aux.shared;
 
-    if (shared->refcnt == 1 && a->ptr == shared->ptr) {
-      a->ptr = shared->ptr;
-      a->aux.capa = a->len;
+    if (shared->refcnt == 1 && a->as.heap.ptr == shared->ptr) {
+      a->as.heap.ptr = shared->ptr;
+      a->as.heap.aux.capa = a->as.heap.len;
       mrb_free(mrb, shared);
     }
     else {
       mrb_value *ptr, *p;
       mrb_int len;
 
-      p = a->ptr;
-      len = a->len * sizeof(mrb_value);
+      p = a->as.heap.ptr;
+      len = a->as.heap.len * sizeof(mrb_value);
       ptr = (mrb_value *)mrb_malloc(mrb, len);
       if (p) {
-        array_copy(ptr, p, a->len);
+        array_copy(ptr, p, a->as.heap.len);
       }
-      a->ptr = ptr;
-      a->aux.capa = a->len;
+      a->as.heap.ptr = ptr;
+      a->as.heap.aux.capa = a->as.heap.len;
       mrb_ary_decref(mrb, shared);
     }
     ARY_UNSET_SHARED_FLAG(a);
@@ -154,18 +160,20 @@ mrb_ary_modify(mrb_state *mrb, struct RArray* a)
 static void
 ary_make_shared(mrb_state *mrb, struct RArray *a)
 {
-  if (!ARY_SHARED_P(a)) {
+  if (!ARY_SHARED_P(a) && !ARY_EMBED_P(a)) {
     mrb_shared_array *shared = (mrb_shared_array *)mrb_malloc(mrb, sizeof(mrb_shared_array));
+    mrb_value *ptr = a->as.heap.ptr;
+    mrb_int len = a->as.heap.len;
 
     shared->refcnt = 1;
-    if (a->aux.capa > a->len) {
-      a->ptr = shared->ptr = (mrb_value *)mrb_realloc(mrb, a->ptr, sizeof(mrb_value)*a->len+1);
+    if (a->as.heap.aux.capa > len) {
+      a->as.heap.ptr = shared->ptr = (mrb_value *)mrb_realloc(mrb, ptr, sizeof(mrb_value)*len+1);
     }
     else {
-      shared->ptr = a->ptr;
+      shared->ptr = ptr;
     }
-    shared->len = a->len;
-    a->aux.shared = shared;
+    shared->len = len;
+    a->as.heap.aux.shared = shared;
     ARY_SET_SHARED_FLAG(a);
   }
 }
@@ -173,14 +181,14 @@ ary_make_shared(mrb_state *mrb, struct RArray *a)
 static void
 ary_expand_capa(mrb_state *mrb, struct RArray *a, mrb_int len)
 {
-  mrb_int capa = a->aux.capa;
+  mrb_int capa = ARY_CAPA(a);
 
   if (len > ARY_MAX_SIZE || len < 0) {
   size_error:
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
   }
 
-  if (capa == 0) {
+  if (capa < ARY_DEFAULT_LEN) {
     capa = ARY_DEFAULT_LEN;
   }
   while (capa < len) {
@@ -195,21 +203,36 @@ ary_expand_capa(mrb_state *mrb, struct RArray *a, mrb_int len)
     goto size_error;
   }
 
-  if (capa > a->aux.capa) {
-    mrb_value *expanded_ptr = (mrb_value *)mrb_realloc(mrb, a->ptr, sizeof(mrb_value)*capa);
+  if (ARY_EMBED_P(a)) {
+    mrb_value *ptr = ARY_EMBED_PTR(a);
+    mrb_int len = ARY_EMBED_LEN(a);
+    mrb_value *expanded_ptr = (mrb_value *)mrb_malloc(mrb, sizeof(mrb_value)*capa);
+
+    ARY_UNSET_EMBED_FLAG(a);
+    array_copy(expanded_ptr, ptr, len);
+    a->as.heap.len = len;
+    a->as.heap.aux.capa = capa;
+    a->as.heap.ptr = expanded_ptr;
+  }
+  else if (capa > a->as.heap.aux.capa) {
+    mrb_value *expanded_ptr = (mrb_value *)mrb_realloc(mrb, a->as.heap.ptr, sizeof(mrb_value)*capa);
 
-    a->aux.capa = capa;
-    a->ptr = expanded_ptr;
+    a->as.heap.aux.capa = capa;
+    a->as.heap.ptr = expanded_ptr;
   }
 }
 
 static void
 ary_shrink_capa(mrb_state *mrb, struct RArray *a)
 {
-  mrb_int capa = a->aux.capa;
+  
+  mrb_int capa;
+
+  if (ARY_EMBED_P(a)) return;
 
+  capa = a->as.heap.aux.capa;
   if (capa < ARY_DEFAULT_LEN * 2) return;
-  if (capa <= a->len * ARY_SHRINK_RATIO) return;
+  if (capa <= a->as.heap.len * ARY_SHRINK_RATIO) return;
 
   do {
     capa /= 2;
@@ -217,11 +240,11 @@ ary_shrink_capa(mrb_state *mrb, struct RArray *a)
       capa = ARY_DEFAULT_LEN;
       break;
     }
-  } while (capa > a->len * ARY_SHRINK_RATIO);
+  } while (capa > a->as.heap.len * ARY_SHRINK_RATIO);
 
-  if (capa > a->len && capa < a->aux.capa) {
-    a->aux.capa = capa;
-    a->ptr = (mrb_value *)mrb_realloc(mrb, a->ptr, sizeof(mrb_value)*capa);
+  if (capa > a->as.heap.len && capa < a->as.heap.aux.capa) {
+    a->as.heap.aux.capa = capa;
+    a->as.heap.ptr = (mrb_value *)mrb_realloc(mrb, a->as.heap.ptr, sizeof(mrb_value)*capa);
   }
 }
 
@@ -234,13 +257,13 @@ mrb_ary_resize(mrb_state *mrb, mrb_value ary, mrb_int new_len)
   ary_modify(mrb, a);
   old_len = RARRAY_LEN(ary);
   if (old_len != new_len) {
-    a->len = new_len;
+    ARY_SET_LEN(a, new_len);
     if (new_len < old_len) {
       ary_shrink_capa(mrb, a);
     }
     else {
       ary_expand_capa(mrb, a, new_len);
-      ary_fill_with_nil(a->ptr + old_len, new_len - old_len);
+      ary_fill_with_nil(ARY_PTR(a) + old_len, new_len - old_len);
     }
   }
 
@@ -268,18 +291,18 @@ ary_concat(mrb_state *mrb, struct RArray *a, struct RArray *a2)
 {
   mrb_int len;
 
-  if (a2->len > ARY_MAX_SIZE - a->len) {
+  if (ARY_LEN(a2) > ARY_MAX_SIZE - ARY_LEN(a)) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
   }
-  len = a->len + a2->len;
+  len = ARY_LEN(a) + ARY_LEN(a2);
 
   ary_modify(mrb, a);
-  if (a->aux.capa < len) {
+  if (ARY_CAPA(a) < len) {
     ary_expand_capa(mrb, a, len);
   }
-  array_copy(a->ptr+a->len, a2->ptr, a2->len);
+  array_copy(ARY_PTR(a)+ARY_LEN(a), ARY_PTR(a2), ARY_LEN(a2));
   mrb_write_barrier(mrb, (struct RBasic*)a);
-  a->len = len;
+  ARY_SET_LEN(a, len);
 }
 
 MRB_API void
@@ -306,16 +329,17 @@ mrb_ary_plus(mrb_state *mrb, mrb_value self)
   struct RArray *a1 = mrb_ary_ptr(self);
   struct RArray *a2;
   mrb_value *ptr;
-  mrb_int blen;
+  mrb_int blen, len1;
 
   mrb_get_args(mrb, "a", &ptr, &blen);
-  if (ARY_MAX_SIZE - blen < a1->len) {
+  if (ARY_MAX_SIZE - blen < ARY_LEN(a1)) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
   }
-  a2 = ary_new_capa(mrb, a1->len + blen);
-  array_copy(a2->ptr, a1->ptr, a1->len);
-  array_copy(a2->ptr + a1->len, ptr, blen);
-  a2->len = a1->len + blen;
+  len1 = ARY_LEN(a1);
+  a2 = ary_new_capa(mrb, len1 + blen);
+  array_copy(ARY_PTR(a2), ARY_PTR(a1), len1);
+  array_copy(ARY_PTR(a2) + len1, ptr, blen);
+  ARY_SET_LEN(a2, len1+blen);
 
   return mrb_obj_value(a2);
 }
@@ -324,11 +348,11 @@ static void
 ary_replace(mrb_state *mrb, struct RArray *a, mrb_value *argv, mrb_int len)
 {
   ary_modify(mrb, a);
-  if (a->aux.capa < len)
+  if (ARY_CAPA(a) < len)
     ary_expand_capa(mrb, a, len);
-  array_copy(a->ptr, argv, len);
+  array_copy(ARY_PTR(a), argv, len);
   mrb_write_barrier(mrb, (struct RBasic*)a);
-  a->len = len;
+  ARY_SET_LEN(a, len);
 }
 
 MRB_API void
@@ -338,7 +362,7 @@ mrb_ary_replace(mrb_state *mrb, mrb_value self, mrb_value other)
   struct RArray *a2 = mrb_ary_ptr(other);
 
   if (a1 != a2) {
-    ary_replace(mrb, a1, a2->ptr, a2->len);
+    ary_replace(mrb, a1, ARY_PTR(a2), ARY_LEN(a2));
   }
 }
 
@@ -359,22 +383,23 @@ mrb_ary_times(mrb_state *mrb, mrb_value self)
   struct RArray *a1 = mrb_ary_ptr(self);
   struct RArray *a2;
   mrb_value *ptr;
-  mrb_int times;
+  mrb_int times, len1;
 
   mrb_get_args(mrb, "i", &times);
   if (times < 0) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "negative argument");
   }
   if (times == 0) return mrb_ary_new(mrb);
-  if (ARY_MAX_SIZE / times < a1->len) {
+  if (ARY_MAX_SIZE / times < ARY_LEN(a1)) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
   }
-  a2 = ary_new_capa(mrb, a1->len * times);
-  ptr = a2->ptr;
+  len1 = ARY_LEN(a1);
+  a2 = ary_new_capa(mrb, len1 * times);
+  ARY_SET_LEN(a2, len1 * times);
+  ptr = ARY_PTR(a2);
   while (times--) {
-    array_copy(ptr, a1->ptr, a1->len);
-    ptr += a1->len;
-    a2->len += a1->len;
+    array_copy(ptr, ARY_PTR(a1), len1);
+    ptr += len1;
   }
 
   return mrb_obj_value(a2);
@@ -384,13 +409,14 @@ static mrb_value
 mrb_ary_reverse_bang(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
+  mrb_int len = ARY_LEN(a);
 
-  if (a->len > 1) {
+  if (len > 1) {
     mrb_value *p1, *p2;
 
     ary_modify(mrb, a);
-    p1 = a->ptr;
-    p2 = a->ptr + a->len - 1;
+    p1 = ARY_PTR(a);
+    p2 = p1 + len - 1;
 
     while (p1 < p2) {
       mrb_value tmp = *p1;
@@ -404,18 +430,19 @@ mrb_ary_reverse_bang(mrb_state *mrb, mrb_value self)
 static mrb_value
 mrb_ary_reverse(mrb_state *mrb, mrb_value self)
 {
-  struct RArray *a = mrb_ary_ptr(self), *b = ary_new_capa(mrb, a->len);
+  struct RArray *a = mrb_ary_ptr(self), *b = ary_new_capa(mrb, ARY_LEN(a));
+  mrb_int len = ARY_LEN(a);
 
-  if (a->len > 0) {
+  if (len > 0) {
     mrb_value *p1, *p2, *e;
 
-    p1 = a->ptr;
-    e  = p1 + a->len;
-    p2 = b->ptr + a->len - 1;
+    p1 = ARY_PTR(a);
+    e  = p1 + len;
+    p2 = ARY_PTR(b) + len - 1;
     while (p1 < e) {
       *p2-- = *p1++;
     }
-    b->len = a->len;
+    ARY_SET_LEN(b, len);
   }
   return mrb_obj_value(b);
 }
@@ -424,11 +451,13 @@ MRB_API void
 mrb_ary_push(mrb_state *mrb, mrb_value ary, mrb_value elem)
 {
   struct RArray *a = mrb_ary_ptr(ary);
+  mrb_int len = ARY_LEN(a);
 
   ary_modify(mrb, a);
-  if (a->len == a->aux.capa)
-    ary_expand_capa(mrb, a, a->len + 1);
-  a->ptr[a->len++] = elem;
+  if (len == ARY_CAPA(a))
+    ary_expand_capa(mrb, a, len + 1);
+  ARY_PTR(a)[len] = elem;
+  ARY_SET_LEN(a, len+1);
   mrb_field_write_barrier_value(mrb, (struct RBasic*)a, elem);
 }
 
@@ -450,10 +479,12 @@ MRB_API mrb_value
 mrb_ary_pop(mrb_state *mrb, mrb_value ary)
 {
   struct RArray *a = mrb_ary_ptr(ary);
+  mrb_int len = ARY_LEN(a);
 
   ary_modify_check(mrb, a);
-  if (a->len == 0) return mrb_nil_value();
-  return a->ptr[--a->len];
+  if (len == 0) return mrb_nil_value();
+  ARY_SET_LEN(a, len-1);
+  return ARY_PTR(a)[len-1];
 }
 
 #define ARY_SHIFT_SHARED_MIN 10
@@ -462,31 +493,32 @@ MRB_API mrb_value
 mrb_ary_shift(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
+  mrb_int len = ARY_LEN(a);
   mrb_value val;
 
   ary_modify_check(mrb, a);
-  if (a->len == 0) return mrb_nil_value();
+  if (len == 0) return mrb_nil_value();
   if (ARY_SHARED_P(a)) {
   L_SHIFT:
-    val = a->ptr[0];
-    a->ptr++;
-    a->len--;
+    val = a->as.heap.ptr[0];
+    a->as.heap.ptr++;
+    a->as.heap.len--;
     return val;
   }
-  if (a->len > ARY_SHIFT_SHARED_MIN) {
+  if (len > ARY_SHIFT_SHARED_MIN) {
     ary_make_shared(mrb, a);
     goto L_SHIFT;
   }
   else {
-    mrb_value *ptr = a->ptr;
-    mrb_int size = a->len;
+    mrb_value *ptr = ARY_PTR(a);
+    mrb_int size = len;
 
     val = *ptr;
     while (--size) {
       *ptr = *(ptr+1);
       ++ptr;
     }
-    --a->len;
+    ARY_SET_LEN(a, len-1);
   }
   return val;
 }
@@ -499,21 +531,25 @@ MRB_API mrb_value
 mrb_ary_unshift(mrb_state *mrb, mrb_value self, mrb_value item)
 {
   struct RArray *a = mrb_ary_ptr(self);
+  mrb_int len = ARY_LEN(a);
 
   if (ARY_SHARED_P(a)
-      && a->aux.shared->refcnt == 1 /* shared only referenced from this array */
-      && a->ptr - a->aux.shared->ptr >= 1) /* there's room for unshifted item */ {
-    a->ptr--;
-    a->ptr[0] = item;
+      && a->as.heap.aux.shared->refcnt == 1 /* shared only referenced from this array */
+      && a->as.heap.ptr - a->as.heap.aux.shared->ptr >= 1) /* there's room for unshifted item */ {
+    a->as.heap.ptr--;
+    a->as.heap.ptr[0] = item;
   }
   else {
+    mrb_value *ptr;
+
     ary_modify(mrb, a);
-    if (a->aux.capa < a->len + 1)
-      ary_expand_capa(mrb, a, a->len + 1);
-    value_move(a->ptr + 1, a->ptr, a->len);
-    a->ptr[0] = item;
+    if (ARY_CAPA(a) < len + 1)
+      ary_expand_capa(mrb, a, len + 1);
+    ptr = ARY_PTR(a);
+    value_move(ptr + 1, ptr, len);
+    ptr[0] = item;
   }
-  a->len++;
+  ARY_SET_LEN(a, len+1);
   mrb_field_write_barrier_value(mrb, (struct RBasic*)a, item);
 
   return self;
@@ -523,28 +559,31 @@ static mrb_value
 mrb_ary_unshift_m(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
-  mrb_value *vals;
-  mrb_int len;
+  mrb_value *vals, *ptr;
+  mrb_int alen, len;
 
-  mrb_get_args(mrb, "*!", &vals, &len);
-  if (len > ARY_MAX_SIZE - a->len) {
+  mrb_get_args(mrb, "*!", &vals, &alen);
+  len = ARY_LEN(a);
+  if (alen > ARY_MAX_SIZE - len) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
   }
   if (ARY_SHARED_P(a)
-      && a->aux.shared->refcnt == 1 /* shared only referenced from this array */
-      && a->ptr - a->aux.shared->ptr >= len) /* there's room for unshifted item */ {
+      && a->as.heap.aux.shared->refcnt == 1 /* shared only referenced from this array */
+      && a->as.heap.ptr - a->as.heap.aux.shared->ptr >= alen) /* there's room for unshifted item */ {
     ary_modify_check(mrb, a);
-    a->ptr -= len;
+    a->as.heap.ptr -= len;
+    ptr = a->as.heap.ptr;
   }
   else {
     ary_modify(mrb, a);
-    if (len == 0) return self;
-    if (a->aux.capa < a->len + len)
-      ary_expand_capa(mrb, a, a->len + len);
-    value_move(a->ptr + len, a->ptr, a->len);
-  }
-  array_copy(a->ptr, vals, len);
-  a->len += len;
+    if (alen == 0) return self;
+    if (ARY_CAPA(a) < len + alen)
+      ary_expand_capa(mrb, a, len + alen);
+    ptr = ARY_PTR(a);
+    value_move(ptr + alen, ptr, len);
+  }
+  array_copy(ptr, vals, alen);
+  ARY_SET_LEN(a, len+alen);
   while (len--) {
     mrb_field_write_barrier_value(mrb, (struct RBasic*)a, vals[len]);
   }
@@ -556,44 +595,47 @@ MRB_API mrb_value
 mrb_ary_ref(mrb_state *mrb, mrb_value ary, mrb_int n)
 {
   struct RArray *a = mrb_ary_ptr(ary);
+  mrb_int len = ARY_LEN(a);
 
   /* range check */
-  if (n < 0) n += a->len;
-  if (n < 0 || a->len <= n) return mrb_nil_value();
+  if (n < 0) n += len;
+  if (n < 0 || len <= n) return mrb_nil_value();
 
-  return a->ptr[n];
+  return ARY_PTR(a)[n];
 }
 
 MRB_API void
 mrb_ary_set(mrb_state *mrb, mrb_value ary, mrb_int n, mrb_value val)
 {
   struct RArray *a = mrb_ary_ptr(ary);
+  mrb_int len = ARY_LEN(a);
 
   ary_modify(mrb, a);
   /* range check */
   if (n < 0) {
-    n += a->len;
+    n += len;
     if (n < 0) {
-      mrb_raisef(mrb, E_INDEX_ERROR, "index %S out of array", mrb_fixnum_value(n - a->len));
+      mrb_raisef(mrb, E_INDEX_ERROR, "index %S out of array", mrb_fixnum_value(n - len));
     }
   }
-  if (a->len <= n) {
-    if (a->aux.capa <= n)
+  if (len <= n) {
+    if (ARY_CAPA(a) <= n)
       ary_expand_capa(mrb, a, n + 1);
-    ary_fill_with_nil(a->ptr + a->len, n + 1 - a->len);
-    a->len = n + 1;
+    ary_fill_with_nil(ARY_PTR(a) + len, n + 1 - len);
+    ARY_SET_LEN(a, n+1);
   }
 
-  a->ptr[n] = val;
+  ARY_PTR(a)[n] = val;
   mrb_field_write_barrier_value(mrb, (struct RBasic*)a, val);
 }
 
 static struct RArray*
 ary_dup(mrb_state *mrb, struct RArray *a)
 {
-  struct RArray *d = ary_new_capa(mrb, a->len);
+  mrb_int len = ARY_LEN(a);
+  struct RArray *d = ary_new_capa(mrb, len);
 
-  ary_replace(mrb, d, a->ptr, a->len);
+  ary_replace(mrb, d, ARY_PTR(a), len);
   return d;
 }
 
@@ -601,6 +643,7 @@ MRB_API mrb_value
 mrb_ary_splice(mrb_state *mrb, mrb_value ary, mrb_int head, mrb_int len, mrb_value rpl)
 {
   struct RArray *a = mrb_ary_ptr(ary);
+  mrb_int alen = ARY_LEN(a);
   const mrb_value *argv;
   mrb_int argc;
   mrb_int tail;
@@ -612,66 +655,67 @@ mrb_ary_splice(mrb_state *mrb, mrb_value ary, mrb_int head, mrb_int len, mrb_val
 
   /* range check */
   if (head < 0) {
-    head += a->len;
+    head += alen;
     if (head < 0) {
       mrb_raise(mrb, E_INDEX_ERROR, "index is out of array");
     }
   }
   tail = head + len;
-  if (a->len < len || a->len < tail) {
-    len = a->len - head;
+  if (alen < len || alen < tail) {
+    len = alen - head;
   }
 
   /* size check */
   if (mrb_array_p(rpl)) {
     argc = RARRAY_LEN(rpl);
     argv = RARRAY_PTR(rpl);
-    if (argv == a->ptr) {
+    if (argv == ARY_PTR(a)) {
       struct RArray *r;
 
       if (argc > 32767) {
         mrb_raise(mrb, E_ARGUMENT_ERROR, "too big recursive splice");
       }
       r = ary_dup(mrb, a);
-      argv = r->ptr;
+      argv = ARY_PTR(r);
     }
   }
   else {
     argc = 1;
     argv = &rpl;
   }
-  if (head >= a->len) {
+  if (head >= alen) {
     if (head > ARY_MAX_SIZE - argc) {
       mrb_raisef(mrb, E_INDEX_ERROR, "index %S too big", mrb_fixnum_value(head));
     }
     len = head + argc;
-    if (len > a->aux.capa) {
+    if (len > ARY_CAPA(a)) {
       ary_expand_capa(mrb, a, head + argc);
     }
-    ary_fill_with_nil(a->ptr + a->len, head - a->len);
+    ary_fill_with_nil(ARY_PTR(a) + alen, head - alen);
     if (argc > 0) {
-      array_copy(a->ptr + head, argv, argc);
+      array_copy(ARY_PTR(a) + head, argv, argc);
     }
-    a->len = len;
+    ARY_SET_LEN(a, len);
   }
   else {
-    mrb_int alen;
+    mrb_int newlen;
 
-    if (a->len - len > ARY_MAX_SIZE - argc) {
-      mrb_raisef(mrb, E_INDEX_ERROR, "index %S too big", mrb_fixnum_value(a->len + argc - len));
+    if (alen - len > ARY_MAX_SIZE - argc) {
+      mrb_raisef(mrb, E_INDEX_ERROR, "index %S too big", mrb_fixnum_value(alen + argc - len));
     }
-    alen = a->len + argc - len;
-    if (alen > a->aux.capa) {
-      ary_expand_capa(mrb, a, alen);
+    newlen = alen + argc - len;
+    if (newlen > ARY_CAPA(a)) {
+      ary_expand_capa(mrb, a, newlen);
     }
 
     if (len != argc) {
+      mrb_value *ptr = ARY_PTR(a);
       tail = head + len;
-      value_move(a->ptr + head + argc, a->ptr + tail, a->len - tail);
-      a->len = alen;
+      value_move(ptr + head + argc, ptr + tail, alen - tail);
+      ARY_SET_LEN(a, newlen);
     }
     if (argc > 0) {
-      value_move(a->ptr + head, argv, argc);
+      value_move(ARY_PTR(a) + head, argv, argc);
     }
   }
   mrb_write_barrier(mrb, (struct RBasic*)a);
@@ -693,12 +737,15 @@ ary_subseq(mrb_state *mrb, struct RArray *a, mrb_int beg, mrb_int len)
 {
   struct RArray *b;
 
+  if (!ARY_SHARED_P(a) && len <= ARY_SHIFT_SHARED_MIN) {
+    return mrb_ary_new_from_values(mrb, len, ARY_PTR(a)+beg);
+  }
   ary_make_shared(mrb, a);
   b  = (struct RArray*)mrb_obj_alloc(mrb, MRB_TT_ARRAY, mrb->array_class);
-  b->ptr = a->ptr + beg;
-  b->len = len;
-  b->aux.shared = a->aux.shared;
-  b->aux.shared->refcnt++;
+  b->as.heap.ptr = a->as.heap.ptr + beg;
+  b->as.heap.len = len;
+  b->as.heap.aux.shared = a->as.heap.aux.shared;
+  b->as.heap.aux.shared->refcnt++;
   ARY_SET_SHARED_FLAG(b);
 
   return mrb_obj_value(b);
@@ -753,14 +800,14 @@ static mrb_value
 mrb_ary_aget(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
-  mrb_int i, len;
+  mrb_int i, len, alen = ARY_LEN(a);
   mrb_value index;
 
   if (mrb_get_args(mrb, "o|i", &index, &len) == 1) {
     switch (mrb_type(index)) {
       /* a[n..m] */
     case MRB_TT_RANGE:
-      if (mrb_range_beg_len(mrb, index, &i, &len, a->len, TRUE) == 1) {
+      if (mrb_range_beg_len(mrb, index, &i, &len, alen, TRUE) == 1) {
         return ary_subseq(mrb, a, i, len);
       }
       else {
@@ -774,11 +821,11 @@ mrb_ary_aget(mrb_state *mrb, mrb_value self)
   }
 
   i = aget_index(mrb, index);
-  if (i < 0) i += a->len;
-  if (i < 0 || a->len < i) return mrb_nil_value();
+  if (i < 0) i += alen;
+  if (i < 0 || alen < i) return mrb_nil_value();
   if (len < 0) return mrb_nil_value();
-  if (a->len == i) return mrb_ary_new(mrb);
-  if (len > a->len - i) len = a->len - i;
+  if (alen == i) return mrb_ary_new(mrb);
+  if (len > alen - i) len = alen - i;
 
   return ary_subseq(mrb, a, i, len);
 }
@@ -853,22 +900,23 @@ mrb_ary_delete_at(mrb_state *mrb, mrb_value self)
   mrb_int   index;
   mrb_value val;
   mrb_value *ptr;
-  mrb_int len;
+  mrb_int len, alen = ARY_LEN(a);
 
   mrb_get_args(mrb, "i", &index);
-  if (index < 0) index += a->len;
-  if (index < 0 || a->len <= index) return mrb_nil_value();
+  if (index < 0) index += alen;
+  if (index < 0 || alen <= index) return mrb_nil_value();
 
   ary_modify(mrb, a);
-  val = a->ptr[index];
+  ptr = ARY_PTR(a);
+  val = ptr[index];
 
-  ptr = a->ptr + index;
-  len = a->len - index;
+  ptr += index;
+  len = alen - index;
   while (--len) {
     *ptr = *(ptr+1);
     ++ptr;
   }
-  --a->len;
+  ARY_SET_LEN(a, alen-1);
 
   ary_shrink_capa(mrb, a);
 
@@ -879,39 +927,39 @@ static mrb_value
 mrb_ary_first(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
-  mrb_int size;
+  mrb_int size, alen = ARY_LEN(a);
 
   if (mrb_get_args(mrb, "|i", &size) == 0) {
-    return (a->len > 0)? a->ptr[0]: mrb_nil_value();
+    return (alen > 0)? ARY_PTR(a)[0]: mrb_nil_value();
   }
   if (size < 0) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "negative array size");
   }
 
-  if (size > a->len) size = a->len;
+  if (size > alen) size = alen;
   if (ARY_SHARED_P(a)) {
     return ary_subseq(mrb, a, 0, size);
   }
-  return mrb_ary_new_from_values(mrb, size, a->ptr);
+  return mrb_ary_new_from_values(mrb, size, ARY_PTR(a));
 }
 
 static mrb_value
 mrb_ary_last(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
-  mrb_int size;
+  mrb_int size, alen = ARY_LEN(a);
 
   if (mrb_get_args(mrb, "|i", &size) == 0)
-    return (a->len > 0)? a->ptr[a->len - 1]: mrb_nil_value();
+    return (alen > 0)? ARY_PTR(a)[alen - 1]: mrb_nil_value();
 
   if (size < 0) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "negative array size");
   }
-  if (size > a->len) size = a->len;
+  if (size > alen) size = alen;
   if (ARY_SHARED_P(a) || size > ARY_DEFAULT_LEN) {
-    return ary_subseq(mrb, a, a->len - size, size);
+    return ary_subseq(mrb, a, alen - size, size);
   }
-  return mrb_ary_new_from_values(mrb, size, a->ptr + a->len - size);
+  return mrb_ary_new_from_values(mrb, size, ARY_PTR(a) + alen - size);
 }
 
 static mrb_value
@@ -984,7 +1032,7 @@ mrb_ary_size(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
 
-  return mrb_fixnum_value(a->len);
+  return mrb_fixnum_value(ARY_LEN(a));
 }
 
 MRB_API mrb_value
@@ -994,15 +1042,14 @@ mrb_ary_clear(mrb_state *mrb, mrb_value self)
 
   ary_modify(mrb, a);
   if (ARY_SHARED_P(a)) {
-    mrb_ary_decref(mrb, a->aux.shared);
+    mrb_ary_decref(mrb, a->as.heap.aux.shared);
     ARY_UNSET_SHARED_FLAG(a);
   }
-  else {
-    mrb_free(mrb, a->ptr);
+  else if (!ARY_EMBED_P(a)){
+    mrb_free(mrb, a->as.heap.ptr);
   }
-  a->len = 0;
-  a->aux.capa = 0;
-  a->ptr = 0;
+  ARY_SET_EMBED_FLAG(a);
+  ARY_SET_EMBED_LEN(a, 0);
 
   return self;
 }
@@ -1012,7 +1059,7 @@ mrb_ary_empty_p(mrb_state *mrb, mrb_value self)
 {
   struct RArray *a = mrb_ary_ptr(self);
 
-  return mrb_bool_value(a->len == 0);
+  return mrb_bool_value(ARY_LEN(a) == 0);
 }
 
 MRB_API mrb_value