From 69fd1a592560d321061790c94f93532db93dccb9 Mon Sep 17 00:00:00 2001 From: KOBAYASHI Shuji Date: Mon, 4 Feb 2019 18:11:22 +0900 Subject: Fix `Symbol#size` for multi-byte characters with `MRB_UTF8_STRING` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: p :あ.size #=> 3 After: p :あ.size #=> 1 --- mrbgems/mruby-symbol-ext/src/symbol.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'mrbgems/mruby-symbol-ext/src') diff --git a/mrbgems/mruby-symbol-ext/src/symbol.c b/mrbgems/mruby-symbol-ext/src/symbol.c index a992dbfce..215226502 100644 --- a/mrbgems/mruby-symbol-ext/src/symbol.c +++ b/mrbgems/mruby-symbol-ext/src/symbol.c @@ -1,6 +1,7 @@ #include #include #include +#include typedef struct symbol_name { size_t len; @@ -45,7 +46,13 @@ static mrb_value mrb_sym_length(mrb_state *mrb, mrb_value self) { mrb_int len; +#ifdef MRB_UTF8_STRING + mrb_int byte_len; + const char *name = mrb_sym2name_len(mrb, mrb_symbol(self), &byte_len); + len = mrb_utf8_len(name, byte_len); +#else mrb_sym2name_len(mrb, mrb_symbol(self), &len); +#endif return mrb_fixnum_value(len); } -- cgit v1.2.3 From 06ba5905f019ca1a8d8e498fe6dce7a94702c9ad Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Wed, 6 Feb 2019 22:01:45 +0900 Subject: Implement inline packed symbols. Small symbols with all alphanumeric characters (<5) are packed in 32bit symbol integer a la base64. This means those small symbols are not listed in `Symbol.all_symbols`. --- include/mruby.h | 3 ++ mrbgems/mruby-symbol-ext/src/symbol.c | 3 +- src/symbol.c | 93 +++++++++++++++++++++++++++++++++-- 3 files changed, 95 insertions(+), 4 deletions(-) (limited to 'mrbgems/mruby-symbol-ext/src') diff --git a/include/mruby.h b/include/mruby.h index 2f2d98677..e9fd08102 100644 --- a/include/mruby.h +++ b/include/mruby.h @@ -242,6 +242,9 @@ typedef struct mrb_state { struct symbol_name *symtbl; /* symbol table */ mrb_sym symhash[256]; size_t symcapa; +#ifndef MRB_ENABLE_SYMBOLL_ALL + char symbuf[8]; /* buffer for small symbol names */ +#endif #ifdef MRB_ENABLE_DEBUG_HOOK void (*code_fetch_hook)(struct mrb_state* mrb, struct mrb_irep *irep, mrb_code *pc, mrb_value *regs); diff --git a/mrbgems/mruby-symbol-ext/src/symbol.c b/mrbgems/mruby-symbol-ext/src/symbol.c index 215226502..05e7d5b91 100644 --- a/mrbgems/mruby-symbol-ext/src/symbol.c +++ b/mrbgems/mruby-symbol-ext/src/symbol.c @@ -30,7 +30,8 @@ mrb_sym_all_symbols(mrb_state *mrb, mrb_value self) mrb_value ary = mrb_ary_new_capa(mrb, mrb->symidx); for (i=1, lim=mrb->symidx+1; i 6) return 0; /* too long */ + for (i=0; i 27) lower = 0; + sym |= bits<<(i*6+2); + } + if (lower) { + sym = 0; + for (i=0; i>(i*5+2) & 31; + if (bits == 0) break; + c = pack_table[bits-1]; + buf[i] = c; + } + buf[i] = '\0'; + return buf; + } + + for (i=0; i<5; i++) { + uint32_t bits; + char c; + + bits = sym>>(i*6+2) & 63; + if (bits == 0) break; + c = pack_table[bits-1]; + buf[i] = c; + } + buf[i] = '\0'; + return buf; +} + uint8_t symhash(const char *key, size_t len) { @@ -50,19 +123,23 @@ find_symbol(mrb_state *mrb, const char *name, uint16_t len, uint8_t hash) mrb_sym i; symbol_name *sname; + /* inline symbol */ + i = sym_inline_pack(name, len); + if (i > 0) return i; + i = mrb->symhash[hash]; if (i == 0) return 0; do { sname = &mrb->symtbl[i]; if (sname->len == len && memcmp(sname->name, name, len) == 0) { - return i; + return i<<1; } if (sname->prev == 0xff) { i -= 0xff; sname = &mrb->symtbl[i]; while (mrb->symtbl < sname) { if (sname->len == len && memcmp(sname->name, name, len) == 0) { - return (mrb_sym)(sname - mrb->symtbl); + return (mrb_sym)(sname - mrb->symtbl)<<1; } sname--; } @@ -117,7 +194,7 @@ sym_intern(mrb_state *mrb, const char *name, size_t len, mrb_bool lit) } mrb->symhash[hash] = sym; - return sym; + return sym<<1; } MRB_API mrb_sym @@ -170,6 +247,13 @@ mrb_check_intern_str(mrb_state *mrb, mrb_value str) MRB_API const char* mrb_sym2name_len(mrb_state *mrb, mrb_sym sym, mrb_int *lenp) { + if (sym & 1) { /* inline packed symbol */ + sym_inline_unpack(sym, mrb->symbuf); + if (lenp) *lenp = strlen(mrb->symbuf); + return mrb->symbuf; + } + + sym >>= 1; if (sym == 0 || mrb->symidx < sym) { if (lenp) *lenp = 0; return NULL; @@ -446,6 +530,9 @@ mrb_sym2str(mrb_state *mrb, mrb_sym sym) const char *name = mrb_sym2name_len(mrb, sym, &len); if (!name) return mrb_undef_value(); /* can't happen */ + if (sym&1) { /* inline symbol */ + return mrb_str_new(mrb, name, len); + } return mrb_str_new_static(mrb, name, len); } -- cgit v1.2.3 From 37bc343e0ad6b837de672c8c0cf6bf00876f746b Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Wed, 6 Feb 2019 23:35:26 +0900 Subject: Disable `Symbol.all_symbols`. --- include/mrbconf.h | 1 + mrbgems/mruby-symbol-ext/src/symbol.c | 9 ++++----- mrbgems/mruby-symbol-ext/test/symbol.rb | 11 +++++++---- src/symbol.c | 6 ++++++ 4 files changed, 18 insertions(+), 9 deletions(-) (limited to 'mrbgems/mruby-symbol-ext/src') diff --git a/include/mrbconf.h b/include/mrbconf.h index b1aec8334..08e69d3aa 100644 --- a/include/mrbconf.h +++ b/include/mrbconf.h @@ -120,6 +120,7 @@ /* -DMRB_ENABLE_XXXX to enable following features */ //#define MRB_ENABLE_DEBUG_HOOK /* hooks for debugger */ +//#define MRB_ENABLE_ALL_SYMBOLS /* Symbols.all_symbols */ /* end of configuration */ diff --git a/mrbgems/mruby-symbol-ext/src/symbol.c b/mrbgems/mruby-symbol-ext/src/symbol.c index 05e7d5b91..ccb2971dc 100644 --- a/mrbgems/mruby-symbol-ext/src/symbol.c +++ b/mrbgems/mruby-symbol-ext/src/symbol.c @@ -3,11 +3,6 @@ #include #include -typedef struct symbol_name { - size_t len; - const char *name; -} symbol_name; - /* * call-seq: * Symbol.all_symbols => array @@ -23,6 +18,7 @@ typedef struct symbol_name { * :Tms, :getwd, :$=, :ThreadGroup, * :wait2, :$>] */ +#ifdef MRB_ENABLE_ALL_SYMBOLS static mrb_value mrb_sym_all_symbols(mrb_state *mrb, mrb_value self) { @@ -36,6 +32,7 @@ mrb_sym_all_symbols(mrb_state *mrb, mrb_value self) return ary; } +#endif /* * call-seq: @@ -61,7 +58,9 @@ void mrb_mruby_symbol_ext_gem_init(mrb_state* mrb) { struct RClass *s = mrb->symbol_class; +#ifdef MRB_ENABLE_ALL_SYMBOLS mrb_define_class_method(mrb, s, "all_symbols", mrb_sym_all_symbols, MRB_ARGS_NONE()); +#endif mrb_define_method(mrb, s, "length", mrb_sym_length, MRB_ARGS_NONE()); mrb_define_method(mrb, s, "size", mrb_sym_length, MRB_ARGS_NONE()); } diff --git a/mrbgems/mruby-symbol-ext/test/symbol.rb b/mrbgems/mruby-symbol-ext/test/symbol.rb index 2c7a62b0c..61ecad247 100644 --- a/mrbgems/mruby-symbol-ext/test/symbol.rb +++ b/mrbgems/mruby-symbol-ext/test/symbol.rb @@ -1,10 +1,13 @@ +# coding: utf-8 ## # Symbol(Ext) Test -assert('Symbol.all_symbols') do - foo = [:__symbol_test_1, :__symbol_test_2, :__symbol_test_3].sort - symbols = Symbol.all_symbols.select{|sym|sym.to_s.include? '__symbol_test'}.sort - assert_equal foo, symbols +if Symbol.respond_to?(:all_symbols) + assert('Symbol.all_symbols') do + foo = [:__symbol_test_1, :__symbol_test_2, :__symbol_test_3].sort + symbols = Symbol.all_symbols.select{|sym|sym.to_s.include? '__symbol_test'}.sort + assert_equal foo, symbols + end end %w[size length].each do |n| diff --git a/src/symbol.c b/src/symbol.c index ab7504772..9cc566245 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -28,6 +28,7 @@ sym_validate_len(mrb_state *mrb, size_t len) } } +#ifndef MRB_ENABLE_ALL_SYMBOLS static char pack_table[] = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; static mrb_sym @@ -100,6 +101,7 @@ sym_inline_unpack(mrb_sym sym, char *buf) buf[i] = '\0'; return buf; } +#endif uint8_t symhash(const char *key, size_t len) @@ -123,9 +125,11 @@ find_symbol(mrb_state *mrb, const char *name, uint16_t len, uint8_t hash) mrb_sym i; symbol_name *sname; +#ifndef MRB_ENABLE_ALL_SYMBOLS /* inline symbol */ i = sym_inline_pack(name, len); if (i > 0) return i; +#endif i = mrb->symhash[hash]; if (i == 0) return 0; @@ -247,11 +251,13 @@ mrb_check_intern_str(mrb_state *mrb, mrb_value str) MRB_API const char* mrb_sym2name_len(mrb_state *mrb, mrb_sym sym, mrb_int *lenp) { +#ifndef MRB_ENABLE_ALL_SYMBOLS if (sym & 1) { /* inline packed symbol */ sym_inline_unpack(sym, mrb->symbuf); if (lenp) *lenp = strlen(mrb->symbuf); return mrb->symbuf; } +#endif sym >>= 1; if (sym == 0 || mrb->symidx < sym) { -- cgit v1.2.3