Squashed commit of the following:

commit 2d7d545c4c4bfce7fdcbcbe9baaeb437915742f0
Merge: 625a1249 b178914b
Author: Yukihiro "Matz" Matsumoto <[email protected]>
Date:   Fri Jun 5 14:35:13 2020 +0900

    Merge branch 'fix-mrb_open-with-nomem' of https://github.com/dearblue/mruby into dearblue-fix-mrb_open-with-nomem

commit b178914b111dda79a8f36ec4eb3e9d37b76f982e
Author: dearblue <[email protected]>
Date:   Sat Jan 19 22:22:44 2019 +0900

    Fix invalid pointer free inside other heap's block

     1. `e = mrb_obj_alloc(...)`
     2. `e->stack = mrb->c->stack` (`mrb->c->stack` is anywhere in the range `stbase...stend`)
     3. And raised exception by `mrb_malloc()`!
     4. `mrb_free(e->stack)` by GC part (wrong free)

commit 52e3d5d8585daf86af3ed12db5ab0efefbc9b956
Author: dearblue <[email protected]>
Date:   Sat Jan 19 21:55:36 2019 +0900

    Fix memory leak for temporary symbols when out of memory

commit 4c5499b88e47cc6012ad7d7379cb6bc74c6a0b60
Author: dearblue <[email protected]>
Date:   Sun Jan 20 11:42:07 2019 +0900

    Fix uninitialized pointer dereference for debug section

commit 8e993167dec62a9709d6faacd517729ddcedf4f9
Author: dearblue <[email protected]>
Date:   Sun Jan 20 11:41:09 2019 +0900

    Fix memory leak for temporary filenames when out of memory

commit 8b422577e6eae68a28121b88421d937e8707b487
Author: dearblue <[email protected]>
Date:   Sun Jan 20 10:57:51 2019 +0900

    Fix memory leak for irep when out of memory

commit 6b35ebf49a0aa3edb6bbda770ed58681e9c2e6af
Author: dearblue <[email protected]>
Date:   Sun Jan 20 10:55:50 2019 +0900

    Fix uninitialized pointer dereference when do not finished initializing irep

commit 2531f2631e67e0462749618e2344c733a29238f0
Author: dearblue <[email protected]>
Date:   Sun Jan 20 10:48:15 2019 +0900

    Fix NULL pointer dereference when do not finished initializing irep

commit e2d6896ebad13694800af49c2625e106b8440ddf
Author: dearblue <[email protected]>
Date:   Sat Jan 19 12:54:19 2019 +0900

    Fix memory leak for irep when out of memory by `mrb_proc_new()`

commit b6214ff8a0a1c73bc9554e39053878ac50bb683f
Author: dearblue <[email protected]>
Date:   Sat Jan 19 12:53:07 2019 +0900

    Fix memory leak for `khash_t` in `kh_init_size()` when out of memory by `kh_alloc()`

commit 19162dd6c11f0093d0011e7cab83b8f9e84c2c07
Author: dearblue <[email protected]>
Date:   Sun Jan 20 02:15:07 2019 +0900

    Fix memory leak for symbol string when out of memory in `kh_put()`

commit 15e67297ff54bc14ef359d6d1e745d760a4a255a
Author: dearblue <[email protected]>
Date:   Sun Jan 20 02:12:24 2019 +0900

    Fix keep wrong symbol index when out of memory

commit 3f8e2b375244f5441e8d62efa13c6e6a9afecb14
Author: dearblue <[email protected]>
Date:   Sun Jan 20 02:08:13 2019 +0900

    Fix keep wrong symbol capacity when out of memory

commit a3cfe755ab3e758046c3f4e30938ac8d567ed046
Author: dearblue <[email protected]>
Date:   Sat Jan 19 10:11:37 2019 +0900

    Fix NULL pointer dereference `mrb->c` by `mark_context()`

commit d9c7b6be6eb54630b64eea5c35be241e551676e5
Author: dearblue <[email protected]>
Date:   Sun Jan 20 15:25:09 2019 +0900

    Fix protect exception for print error message

commit 100642750e4d549f2e8050f8d6cabdf8825d4495
Author: dearblue <[email protected]>
Date:   Sun Jan 20 11:59:02 2019 +0900

    Protect exception for mruby core initialization

commit 7a0418304ec70764fa215bef3599f5f735222075
Author: dearblue <[email protected]>
Date:   Fri Jan 18 20:38:27 2019 +0900

    Fix memory leak for string object when out of memory

    The `mrb_str_pool()` function has a path to call `malloc()` twice.

    If occurs `NoMemoryError` exception in second `malloc()`,
    first `malloc()` pointer is not freed.

commit fef1c152ce4e52b9e4a34dc23aca5b02907ac639
Author: dearblue <[email protected]>
Date:   Sat Jan 19 13:05:09 2019 +0900

    Fix stack overflow when out of memory

    As a result of this change, no backtrace information is set
    for NoMemoryError (`mrb->nomem_err`).

    Detailes:

    When generating a backtrace, called `mrb_intern_lit()`,
    `mrb_str_new_cstr()` and `mrb_obj_iv_set()` function with
    `exc_debug_info()` function in `src/error.c`.

    If a `NoMemoryError` exception occurs at this time,
    the `exc_debug_info()` function will be called again,
    and in the same way `NoMemoryError` exception raised will result
    in an infinite loop to occurs stack overflow (and SIGSEGV).

commit da7d7f881bbbad9988a3a2b7bad8f2b72ff06bc6
Author: dearblue <[email protected]>
Date:   Sun Jan 20 12:00:38 2019 +0900

    Fix NULL pointer dereference `mrb->nomem_err` when not initialized

    Add internal functions (not `static`):

      * `mrb_raise_nomemory()`
      * `mrb_core_init_abort()`

1 files changed, 34 insertions, 11 deletions

diff --git a/src/state.c b/src/state.c
index 3e5ebb483..533bdaa0b 100644
--- a/src/state.c
+++ b/src/state.c
@@ -19,11 +19,25 @@ void mrb_init_mrbgems(mrb_state*);
 void mrb_gc_init(mrb_state*, mrb_gc *gc);
 void mrb_gc_destroy(mrb_state*, mrb_gc *gc);
 
+int mrb_core_init_protect(mrb_state *mrb, void (*body)(mrb_state *, void *), void *opaque);
+
+static void
+init_gc_and_core(mrb_state *mrb, void *opaque)
+{
+  static const struct mrb_context mrb_context_zero = { 0 };
+
+  mrb_gc_init(mrb, &mrb->gc);
+  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
+  *mrb->c = mrb_context_zero;
+  mrb->root_c = mrb->c;
+
+  mrb_init_core(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_core(mrb_allocf f, void *ud)
 {
   static const mrb_state mrb_state_zero = { 0 };
-  static const struct mrb_context mrb_context_zero = { 0 };
   mrb_state *mrb;
 
   if (f == NULL) f = mrb_default_allocf;
@@ -35,12 +49,10 @@ mrb_open_core(mrb_allocf f, void *ud)
   mrb->allocf = f;
   mrb->atexit_stack_len = 0;
 
-  mrb_gc_init(mrb, &mrb->gc);
-  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
-  *mrb->c = mrb_context_zero;
-  mrb->root_c = mrb->c;
-
-  mrb_init_core(mrb);
+  if (mrb_core_init_protect(mrb, init_gc_and_core, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
 
   return mrb;
 }
@@ -65,6 +77,12 @@ mrb_open(void)
   return mrb;
 }
 
+static void
+init_mrbgems(mrb_state *mrb, void *opaque)
+{
+  mrb_init_mrbgems(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_allocf(mrb_allocf f, void *ud)
 {
@@ -75,7 +93,10 @@ mrb_open_allocf(mrb_allocf f, void *ud)
   }
 
 #ifndef DISABLE_GEMS
-  mrb_init_mrbgems(mrb);
+  if (mrb_core_init_protect(mrb, init_mrbgems, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
   mrb_gc_arena_restore(mrb, 0);
 #endif
   return mrb;
@@ -131,9 +152,11 @@ mrb_irep_free(mrb_state *mrb, mrb_irep *irep)
   }
   mrb_free(mrb, irep->pool);
   mrb_free(mrb, irep->syms);
-  for (i=0; i<irep->rlen; i++) {
-    if (irep->reps[i])
-      mrb_irep_decref(mrb, irep->reps[i]);
+  if (irep->reps) {
+    for (i=0; i<irep->rlen; i++) {
+      if (irep->reps[i])
+        mrb_irep_decref(mrb, irep->reps[i]);
+    }
   }
   mrb_free(mrb, irep->reps);
   mrb_free(mrb, irep->lv);