From c3ac33d9cd532aa80b8595d460eac745712b4042 Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Wed, 20 Jun 2018 18:17:38 +0900 Subject: Make minirake parallel. --- minirake | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 116 insertions(+), 13 deletions(-) (limited to 'minirake') diff --git a/minirake b/minirake index 542c37a79..ae3d53ca0 100755 --- a/minirake +++ b/minirake @@ -6,6 +6,12 @@ require 'getoptlong' require 'fileutils' +require 'fiber' + +$rake_fiber_table = {} +$rake_jobs = 1 +$rake_failed = [] +$rake_root_fiber = Fiber.current class String def ext(newext='') @@ -86,14 +92,31 @@ module MiniRake @name.to_s end + def done?; @done end + def running?; @running end + # Invoke the task if it is needed. Prerequites are invoked first. def invoke puts "Invoke #{name} (already=[#{@already_invoked}], needed=[#{needed?}])" if $trace return if @already_invoked - @already_invoked = true prerequisites = @prerequisites.collect{ |n| n.is_a?(Proc) ? n.call(name) : n }.flatten - prerequisites.each { |n| Task[n].invoke } - execute if needed? + prerequisites.each do |n| + t = Task[n] + unless t.done? + return prerequisites.select{|v| v = Task[v]; v && (!v.done? || !v.running?) } + end + end + + @already_invoked = true + + if needed? + @running = true + return Fiber.new do + self.execute + end + end + + @done = true end # Execute the actions associated with this task. @@ -103,6 +126,8 @@ module MiniRake unless $dryrun @actions.each { |act| act.call(self) } end + @done = true + @running = false end # Is this task needed? @@ -281,7 +306,19 @@ module MiniRake # Run the system command +cmd+. def sh(cmd) puts cmd if $verbose - system(cmd) or fail "Command Failed: [#{cmd}]" + + if $rake_jobs == 1 || Fiber.current == $rake_root_fiber + system(cmd) or fail "Command Failed: [#{cmd}]" + return + end + + pid = Process.spawn(cmd) + $rake_fiber_table[pid] = { + fiber: Fiber.current, + command: cmd, + process_waiter: Process.detach(pid) + } + Fiber.yield end def desc(text) @@ -329,7 +366,9 @@ class RakeApp ['--verbose', '-v', GetoptLong::NO_ARGUMENT, "Log message to standard output."], ['--directory', '-C', GetoptLong::REQUIRED_ARGUMENT, - "Change executing directory of rakefiles."] + "Change executing directory of rakefiles."], + ['--jobs', '-j', GetoptLong::REQUIRED_ARGUMENT, + 'Execute rake with parallel jobs.'] ] # Create a RakeApp object. @@ -422,6 +461,8 @@ class RakeApp exit when '--directory' Dir.chdir value + when '--jobs' + $rake_jobs = [value.to_i, 1].max else fail "Unknown option: #{opt}" end @@ -447,12 +488,12 @@ class RakeApp end here = Dir.pwd end - tasks = [] + root_tasks = [] ARGV.each do |task_name| if /^(\w+)=(.*)/.match(task_name) ENV[$1] = $2 else - tasks << task_name + root_tasks << task_name end end puts "(in #{Dir.pwd})" @@ -461,20 +502,82 @@ class RakeApp if $show_tasks display_tasks else - tasks.push("default") if tasks.size == 0 - tasks.each do |task_name| - MiniRake::Task[task_name].invoke + root_tasks.push("default") if root_tasks.empty? + # revese tasks for popping + root_tasks.reverse! + + tasks = [] + until root_tasks.empty? + root_name = root_tasks.pop + tasks << root_name + until tasks.empty? + task_name = tasks.pop + t = MiniRake::Task[task_name] + f = t.invoke + + # append additional tasks to task queue + if f.kind_of?(Array) + tasks.push(*f) + tasks.uniq! + end + + unless f.kind_of? Fiber + tasks.insert 0, task_name unless t.done? + if root_name == task_name + wait_process + end + next + end + + f.resume + end end + + wait_process until $rake_fiber_table.empty? + end + rescue Exception => e + begin + $rake_failed << e + wait_process until $rake_fiber_table.empty? + rescue Exception => next_e + e = next_e + retry end - rescue Exception => ex - puts "rake aborted!" + end + + return if $rake_failed.empty? + + puts "rake aborted!" + $rake_failed.each do |ex| puts ex.message if $trace puts ex.backtrace.join("\n") else puts ex.backtrace.find {|str| str =~ /#{@rakefile}/ } || "" end - exit 1 + end + exit 1 + end + + def wait_process + sleep 0.1 + + exited = [] + $rake_fiber_table.each do |pid, v| + exited << pid unless v[:process_waiter].alive? + end + + exited.each do |pid| + ent = $rake_fiber_table.delete pid + st = ent[:process_waiter].value + + # ignore process that isn't created by `sh` method + return if ent.nil? + + if st.exitstatus != 0 + raise "Command Failed: [#{ent[:command]}]" + end + ent[:fiber].resume end end end -- cgit v1.2.3 From 6f35b07fabb150ead959889d0833cf9b10c9712c Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Thu, 21 Jun 2018 11:26:55 +0900 Subject: Use Fiber#transfer instead in minirake. --- minirake | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'minirake') diff --git a/minirake b/minirake index ae3d53ca0..43a669f8e 100755 --- a/minirake +++ b/minirake @@ -113,6 +113,7 @@ module MiniRake @running = true return Fiber.new do self.execute + $rake_root_fiber.transfer end end @@ -318,7 +319,7 @@ module MiniRake command: cmd, process_waiter: Process.detach(pid) } - Fiber.yield + $rake_root_fiber.transfer end def desc(text) @@ -529,7 +530,7 @@ class RakeApp next end - f.resume + f.transfer end end @@ -577,7 +578,7 @@ class RakeApp if st.exitstatus != 0 raise "Command Failed: [#{ent[:command]}]" end - ent[:fiber].resume + ent[:fiber].transfer end end end -- cgit v1.2.3 From c90dbd12ef41f3451e3e1dadaed8269d30df7da3 Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Thu, 21 Jun 2018 11:29:50 +0900 Subject: Show minirake backtrace in verbose mode. --- minirake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'minirake') diff --git a/minirake b/minirake index ae3d53ca0..6f772cb3d 100755 --- a/minirake +++ b/minirake @@ -550,7 +550,7 @@ class RakeApp puts "rake aborted!" $rake_failed.each do |ex| puts ex.message - if $trace + if $trace || $verbose puts ex.backtrace.join("\n") else puts ex.backtrace.find {|str| str =~ /#{@rakefile}/ } || "" -- cgit v1.2.3 From 7c246cb9db375c30490cc62da1c7138d54e6d441 Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Thu, 21 Jun 2018 12:45:55 +0900 Subject: Fix job waiter. --- minirake | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'minirake') diff --git a/minirake b/minirake index ae3d53ca0..343978d77 100755 --- a/minirake +++ b/minirake @@ -529,6 +529,8 @@ class RakeApp next end + wait_process while $rake_fiber_table.size >= $rake_jobs + f.resume end end @@ -577,6 +579,9 @@ class RakeApp if st.exitstatus != 0 raise "Command Failed: [#{ent[:command]}]" end + + wait_process while $rake_fiber_table.size >= $rake_jobs + ent[:fiber].resume end end -- cgit v1.2.3 From 61e8f831dd0d8936087b2d3be9a135e0802d3d64 Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Fri, 22 Jun 2018 09:29:59 +0900 Subject: Use `Thread.pass` instead of sleeping. --- minirake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'minirake') diff --git a/minirake b/minirake index 09592980f..00675ba39 100755 --- a/minirake +++ b/minirake @@ -562,7 +562,7 @@ class RakeApp end def wait_process - sleep 0.1 + Thread.pass exited = [] $rake_fiber_table.each do |pid, v| -- cgit v1.2.3 From 48ba7c7bb0ccf146921e585c6ce2f5a0e1afff0f Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Fri, 22 Jun 2018 11:41:12 +0900 Subject: Increase sleep duration exponentially instead of `Thread.pass`. --- minirake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'minirake') diff --git a/minirake b/minirake index e16dd5bcf..8b6bfa296 100755 --- a/minirake +++ b/minirake @@ -562,8 +562,9 @@ class RakeApp exit 1 end - def wait_process - Thread.pass + def wait_process(count = 0) + dur = [0.0001 * (10 ** count), 1].min + sleep dur exited = [] $rake_fiber_table.each do |pid, v| @@ -581,10 +582,12 @@ class RakeApp raise "Command Failed: [#{ent[:command]}]" end - wait_process while $rake_fiber_table.size >= $rake_jobs + fail 'task scheduling bug!' if $rake_fiber_table.size >= $rake_jobs ent[:fiber].transfer end + + wait_process(count + 1) if !$rake_fiber_table.empty? && exited.empty? end end -- cgit v1.2.3 From b03f1f78e3f1bab579b21219bbecffb6eb4b7020 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Sat, 11 Aug 2018 00:28:32 +0900 Subject: Fix misspelling words in comments --- doc/guides/mrbgems.md | 4 ++-- include/mruby.h | 4 ++-- minirake | 2 +- mrbgems/mruby-bin-debugger/bintest/print.rb | 2 +- mrbgems/mruby-compiler/core/codegen.c | 2 +- mrbgems/mruby-socket/src/socket.c | 2 +- test/t/literals.rb | 2 +- test/t/module.rb | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) (limited to 'minirake') diff --git a/doc/guides/mrbgems.md b/doc/guides/mrbgems.md index 8dac0dc86..0fcc936ed 100644 --- a/doc/guides/mrbgems.md +++ b/doc/guides/mrbgems.md @@ -179,11 +179,11 @@ Version requirement supports following operators: When more than one version requirements is passed, the dependency must satisfy all of it. -You can have default gem to use as depedency when it's not defined in *build_config.rb*. +You can have default gem to use as dependency when it's not defined in *build_config.rb*. When the last argument of `add_dependency` call is `Hash`, it will be treated as default gem information. Its format is same as argument of method `MRuby::Build#gem`, expect that it can't be treated as path gem location. -When a special version of depedency is required, +When a special version of dependency is required, use `MRuby::Build#gem` in *build_config.rb* to override default gem. If you have conflicting GEMs use the following method: diff --git a/include/mruby.h b/include/mruby.h index 93fc0f010..bc6d72330 100644 --- a/include/mruby.h +++ b/include/mruby.h @@ -782,7 +782,7 @@ MRB_API struct RClass * mrb_define_module_under(mrb_state *mrb, struct RClass *o #define MRB_ARGS_REQ(n) ((mrb_aspec)((n)&0x1f) << 18) /** - * Funtion takes n optional arguments + * Function takes n optional arguments * * @param n * The number of optional arguments. @@ -790,7 +790,7 @@ MRB_API struct RClass * mrb_define_module_under(mrb_state *mrb, struct RClass *o #define MRB_ARGS_OPT(n) ((mrb_aspec)((n)&0x1f) << 13) /** - * Funtion takes n1 mandatory arguments and n2 optional arguments + * Function takes n1 mandatory arguments and n2 optional arguments * * @param n1 * The number of required arguments. diff --git a/minirake b/minirake index 8b6bfa296..ad8276a6d 100755 --- a/minirake +++ b/minirake @@ -95,7 +95,7 @@ module MiniRake def done?; @done end def running?; @running end - # Invoke the task if it is needed. Prerequites are invoked first. + # Invoke the task if it is needed. Prerequisites are invoked first. def invoke puts "Invoke #{name} (already=[#{@already_invoked}], needed=[#{needed?}])" if $trace return if @already_invoked diff --git a/mrbgems/mruby-bin-debugger/bintest/print.rb b/mrbgems/mruby-bin-debugger/bintest/print.rb index 0d4aad011..6675392b8 100644 --- a/mrbgems/mruby-bin-debugger/bintest/print.rb +++ b/mrbgems/mruby-bin-debugger/bintest/print.rb @@ -317,7 +317,7 @@ TestConstNameSubClass.new.m() bp = nil SRC - # todo: wait for 'break' to be implimented + # todo: wait for 'break' to be implemented tc = [] 9.times { tc << {:cmd=>"s"} } tc << {:cmd=>"p CONST", :exp=>"super class"} diff --git a/mrbgems/mruby-compiler/core/codegen.c b/mrbgems/mruby-compiler/core/codegen.c index fec747f0c..945ddcc1a 100644 --- a/mrbgems/mruby-compiler/core/codegen.c +++ b/mrbgems/mruby-compiler/core/codegen.c @@ -200,7 +200,7 @@ genop_peep(codegen_scope *s, mrb_code i, int val) switch (c0) { case OP_MOVE: if (GETARG_A(i) == GETARG_A(i0)) { - /* skip overriden OP_MOVE */ + /* skip overridden OP_MOVE */ s->pc--; s->iseq[s->pc] = i; } diff --git a/mrbgems/mruby-socket/src/socket.c b/mrbgems/mruby-socket/src/socket.c index b3ca8b1c9..b44371544 100644 --- a/mrbgems/mruby-socket/src/socket.c +++ b/mrbgems/mruby-socket/src/socket.c @@ -300,7 +300,7 @@ mrb_basicsocket_getpeereid(mrb_state *mrb, mrb_value self) mrb_ary_push(mrb, ary, mrb_fixnum_value((mrb_int)egid)); return ary; #else - mrb_raise(mrb, E_RUNTIME_ERROR, "getpeereid is not avaialble on this system"); + mrb_raise(mrb, E_RUNTIME_ERROR, "getpeereid is not available on this system"); return mrb_nil_value(); #endif } diff --git a/test/t/literals.rb b/test/t/literals.rb index 51a37c32d..6344219aa 100644 --- a/test/t/literals.rb +++ b/test/t/literals.rb @@ -22,7 +22,7 @@ assert('Literals Numerical', '8.7.6.2') do # decimal assert_equal 999, 0d999 assert_equal 999, 0D999 - # decimal seperator + # decimal separator assert_equal 10000000, 10_000_000 assert_equal 10, 1_0 # integer with exponent diff --git a/test/t/module.rb b/test/t/module.rb index 5a46c24ff..fb82fc934 100644 --- a/test/t/module.rb +++ b/test/t/module.rb @@ -699,7 +699,7 @@ end end end - # these assertions will not run without a #assert_seperately method + # these assertions will not run without a #assert_separately method #assert 'test_prepend_optmethod' do # bug7983 = '[ruby-dev:47124] [Bug #7983]' # assert_separately [], %{ @@ -807,7 +807,7 @@ end assert_equal([m, c2, m, c1], c2.ancestors[0, 4], "should accesisble prepended module in superclass") end - # requires #assert_seperately + # requires #assert_separately #assert 'Module#prepend call super' do # assert_separately([], <<-'end;') #do # bug10847 = '[ruby-core:68093] [Bug #10847]' -- cgit v1.2.3 From e6357f3baa465412c521e7d4b527c868f73ec62e Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Tue, 4 Sep 2018 16:16:25 +0900 Subject: Fix indent. --- minirake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'minirake') diff --git a/minirake b/minirake index ad8276a6d..2c93502f2 100755 --- a/minirake +++ b/minirake @@ -590,7 +590,7 @@ class RakeApp wait_process(count + 1) if !$rake_fiber_table.empty? && exited.empty? end - end +end if __FILE__ == $0 then RakeApp.new.run -- cgit v1.2.3 From 901dccb2df17cd6ebd5b9e5c7ef2734af54f0588 Mon Sep 17 00:00:00 2001 From: take-cheeze Date: Tue, 4 Sep 2018 16:16:33 +0900 Subject: Avoid root fiber initialization for CRuby workaround. closes #4085. --- minirake | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'minirake') diff --git a/minirake b/minirake index 2c93502f2..86c42c999 100755 --- a/minirake +++ b/minirake @@ -6,12 +6,10 @@ require 'getoptlong' require 'fileutils' -require 'fiber' $rake_fiber_table = {} $rake_jobs = 1 $rake_failed = [] -$rake_root_fiber = Fiber.current class String def ext(newext='') @@ -111,9 +109,13 @@ module MiniRake if needed? @running = true - return Fiber.new do + if $rake_root_fiber + return Fiber.new do + self.execute + $rake_root_fiber.transfer + end + else self.execute - $rake_root_fiber.transfer end end @@ -308,7 +310,7 @@ module MiniRake def sh(cmd) puts cmd if $verbose - if $rake_jobs == 1 || Fiber.current == $rake_root_fiber + if !$rake_root_fiber || Fiber.current == $rake_root_fiber system(cmd) or fail "Command Failed: [#{cmd}]" return end @@ -480,6 +482,12 @@ class RakeApp # Run the +rake+ application. def run handle_options + + if $rake_root_fiber + require 'fiber' + $rake_root_fiber = Fiber.current + end + begin here = Dir.pwd while ! have_rakefile -- cgit v1.2.3 From 5b5e54d38e241a1cf49662f4395fb5bbc26696ea Mon Sep 17 00:00:00 2001 From: Takeshi Watanabe Date: Tue, 25 Dec 2018 14:37:32 +0900 Subject: Fix `$rake_root_fiber` checking. https://github.com/mruby/mruby/pull/4101#pullrequestreview-187752352 --- minirake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'minirake') diff --git a/minirake b/minirake index 86c42c999..cd2d85854 100755 --- a/minirake +++ b/minirake @@ -483,7 +483,7 @@ class RakeApp def run handle_options - if $rake_root_fiber + unless $rake_root_fiber require 'fiber' $rake_root_fiber = Fiber.current end -- cgit v1.2.3