diff --git a/.gitignore b/.gitignore index 963a6a6..3eb2aef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,11 @@ Gemfile.lock tmp bin/enterprise_script_service +bin/enterprise_script_service_no_gen_gc bin/enterprise_script_service_tests +bin/enterprise_script_service_no_gen_gc_tests bin/enterprise_script_service.dSYM +bin/enterprise_script_service_no_gen_gc.dSYM bin/enterprise_script_service_tests.dSYM +bin/enterprise_script_service_no_gen_gc_tests.dSYM spec/examples.txt diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..712bd5a --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.3.1 \ No newline at end of file diff --git a/README.adoc b/README.adoc index ee1b753..e887333 100644 --- a/README.adoc +++ b/README.adoc @@ -60,6 +60,29 @@ expect(result.stdout).to eq("hello") <2> two "scripts" to be executed, one sets the `@stdout_buffer` to a value, the second puts the value associated to the key `:result` of the map used passed in in <1> <3> give the ESS a 1000 second time quota to spawn, init, inject, eval and finally output the result back. +=== Turning off generational garbage collection + +In some cases, you may want to run a script in an mruby environment with the generational garbage collector turned off. To do that, you can run: + +[source, ruby] +---- +result = EnterpriseScriptService.run( + input: {result: [26803196617, 0.475]}, + sources: [ + ["stdout", "@stdout_buffer = 'hello'"], + ["foo", "@output = @input[:result]"], + ], + instructions: nil, + timeout: 10.0, + instruction_quota: 100000, + instruction_quota_start: 1, + memory_quota: 8 << 20, + generational_gc: false +) +---- + +Take note of the last argument of `generational_gc: false`. + == Where are things? === C++ sources diff --git a/ext/enterprise_script_service/Rakefile b/ext/enterprise_script_service/Rakefile index d8685cf..7a9fdc2 100644 --- a/ext/enterprise_script_service/Rakefile +++ b/ext/enterprise_script_service/Rakefile @@ -15,6 +15,7 @@ MESSAGE ROOT = Pathname.new(__dir__).join("../..") SERVICE_EXECUTABLE_DIR = ROOT.join("bin") SERVICE_EXECUTABLE = SERVICE_EXECUTABLE_DIR.join("enterprise_script_service").to_s +SERVICE_EXECUTABLE_NO_GENERATIONAL_GC = SERVICE_EXECUTABLE_DIR.join("enterprise_script_service_no_gen_gc").to_s SERVICE_SOURCES = Dir.glob("*.cpp").map(&:to_s) Dir.chdir("#{ROOT}/tests") do SERVICE_TESTS = Dir.glob("*_test.cpp").map { |f| "#{Dir.pwd}/#{f.to_s}"} @@ -22,10 +23,13 @@ Dir.chdir("#{ROOT}/tests") do SERVICE_TESTS << "#{GOOGLE_TEST_DIR}/src/gtest-all.cc" SERVICE_TESTS << "#{GOOGLE_TEST_DIR}/src/gtest_main.cc" SERVICE_TESTS_EXECUTABLE = SERVICE_EXECUTABLE_DIR.join("enterprise_script_service_tests").to_s + SERVICE_NO_GENERATIONAL_GC_TESTS_EXECUTABLE = SERVICE_EXECUTABLE_DIR.join("enterprise_script_service_no_gen_gc_tests").to_s end MRUBY_LIB_DIR = MRUBY_DIR.join("build/sandbox/lib") +MRUBY_NO_GENERATIONAL_GC_LIB_DIR = MRUBY_DIR.join("build/sandbox_no_gen_gc/lib") MRUBY_LIB = MRUBY_LIB_DIR.join("libmruby.a") +MRUBY_NO_GENERATIONAL_GC_LIB = MRUBY_NO_GENERATIONAL_GC_LIB_DIR.join("libmruby.a") LIBSECCOMP_DIR = Pathname.new(__dir__).join("libseccomp") LIBSECCOMP_LIB_DIR = LIBSECCOMP_DIR.join("src/.libs") @@ -67,6 +71,29 @@ file(SERVICE_EXECUTABLE => [ ) end +file(SERVICE_EXECUTABLE_NO_GENERATIONAL_GC => [ + SERVICE_EXECUTABLE_DIR, + *SERVICE_SOURCES, + __FILE__, + MRUBY_NO_GENERATIONAL_GC_LIB, +]) do + sh( + CXX, + "--std=c++11", + "-Wall", + "-Wextra", + "-Imsgpack/include", + "-Imruby/include", + "-L#{MRUBY_NO_GENERATIONAL_GC_LIB_DIR}", + *Flags.cflags, + *Flags.defines_with_no_gc.map { |define| "-D#{define}" }, + "-o", SERVICE_EXECUTABLE_NO_GENERATIONAL_GC, + *SERVICE_SOURCES, + "-lmruby", + *LIBSECCOMP_CFLAGS, + ) +end + SERVICE_SOURCES_NO_MAIN = SERVICE_SOURCES.select { |f| f != "ext.cpp"} file(SERVICE_TESTS_EXECUTABLE => [ @@ -98,26 +125,58 @@ file(SERVICE_TESTS_EXECUTABLE => [ ) end +file(SERVICE_NO_GENERATIONAL_GC_TESTS_EXECUTABLE => [ + SERVICE_EXECUTABLE_DIR, + *SERVICE_SOURCES_NO_MAIN, + *SERVICE_TESTS, + __FILE__, + MRUBY_NO_GENERATIONAL_GC_LIB, +]) do + sh( + CXX, + "--std=c++11", + "-Wall", + "-Wextra", + "-Imsgpack/include", + "-Imruby/include", + "-I#{GOOGLE_TEST_DIR}/include", + "-I#{GOOGLE_TEST_DIR}", + "-I.", + "-L#{MRUBY_NO_GENERATIONAL_GC_LIB_DIR}", + *Flags.cflags, + *Flags.defines_with_no_gc.map { |define| "-D#{define}" }, + "-o", SERVICE_NO_GENERATIONAL_GC_TESTS_EXECUTABLE, + *SERVICE_SOURCES_NO_MAIN, + *SERVICE_TESTS, + "-lmruby", + "-lpthread", + *LIBSECCOMP_CFLAGS, + ) +end + file(MRUBY_LIB => [:"mruby:compile", :"libseccomp:compile"]) +file(MRUBY_NO_GENERATIONAL_GC_LIB => [:"mruby:compile_no_gen_gc", :"libseccomp:compile"]) task(clean: [:"mruby:mrproper", :"libseccomp:mrproper"]) do sh("rm", SERVICE_EXECUTABLE) + sh("rm", SERVICE_EXECUTABLE_NO_GENERATIONAL_GC) end task(mrproper: [:clean, :"mruby:mrproper", :"libseccomp:mrproper"]) -task(default: [SERVICE_EXECUTABLE, :test]) +task(default: [SERVICE_EXECUTABLE, SERVICE_EXECUTABLE_NO_GENERATIONAL_GC, :test]) -task(test: [SERVICE_TESTS_EXECUTABLE]) do +task(test: [SERVICE_TESTS_EXECUTABLE, SERVICE_NO_GENERATIONAL_GC_TESTS_EXECUTABLE]) do sh(SERVICE_TESTS_EXECUTABLE) + sh(SERVICE_NO_GENERATIONAL_GC_TESTS_EXECUTABLE) end namespace(:mruby) do - def within_mruby + def within_mruby(config: '../mruby_config.rb') Dir.chdir(MRUBY_DIR) do original_mruby_config = ENV['MRUBY_CONFIG'] begin - ENV['MRUBY_CONFIG'] = '../mruby_config.rb' + ENV['MRUBY_CONFIG'] = config yield ensure ENV['MRUBY_CONFIG'] = original_mruby_config @@ -131,6 +190,15 @@ namespace(:mruby) do end end + task(:compile_no_gen_gc) do + within_mruby(config: '../mruby_no_gen_gc_config.rb') do + extra_args = [] + extra_args << '' if RUBY_PLATFORM.match?(/darwin/i) + sh('sed', '-i', *extra_args, 's/{ :verbose => $verbose }/verbose: $verbose/', 'Rakefile') + sh("ruby", "./minirake") + end + end + task(:clean) do within_mruby do sh("ruby", "./minirake", "clean") diff --git a/ext/enterprise_script_service/flags.rb b/ext/enterprise_script_service/flags.rb index 1948699..16dc78d 100644 --- a/ext/enterprise_script_service/flags.rb +++ b/ext/enterprise_script_service/flags.rb @@ -27,8 +27,16 @@ def io_safe_defines ) end + def io_safe_defines_with_no_gc + io_safe_defines + %w(MRB_GC_TURN_OFF_GENERATIONAL) + end + def defines io_safe_defines + %w(MRB_DISABLE_STDIO) end + + def defines_with_no_gc + defines + %w(MRB_GC_TURN_OFF_GENERATIONAL) + end end end diff --git a/ext/enterprise_script_service/mruby_config.rb b/ext/enterprise_script_service/mruby_config.rb index 7f8a9e4..20121d7 100644 --- a/ext/enterprise_script_service/mruby_config.rb +++ b/ext/enterprise_script_service/mruby_config.rb @@ -1,37 +1,5 @@ require_relative("./flags") +require_relative("./mruby_shared_config") -mruby_engine_gembox_path = Pathname.new(__FILE__).dirname.join("mruby_engine") - -MRuby::Build.new do |conf| - toolchain(:gcc) - - enable_debug - - conf.gembox(mruby_engine_gembox_path) - conf.gem(core: "mruby-bin-mirb") - conf.gem(core: 'mruby-bin-mruby') - - conf.bins = ["mrbc", "mruby"] - - conf.cc do |cc| - cc.flags += %w(-fPIC) - cc.flags += Flags.cflags - cc.defines += Flags.io_safe_defines - end -end - -MRuby::CrossBuild.new("sandbox") do |conf| - toolchain(:gcc) - - enable_debug - - conf.gembox(mruby_engine_gembox_path) - - conf.bins = [] - - conf.cc do |cc| - cc.flags += %w(-fPIC) - cc.flags += Flags.cflags - cc.defines += Flags.defines - end -end +build(Flags.io_safe_defines) +crossbuild("sandbox", Flags.defines) diff --git a/ext/enterprise_script_service/mruby_no_gen_gc_config.rb b/ext/enterprise_script_service/mruby_no_gen_gc_config.rb new file mode 100644 index 0000000..965e05d --- /dev/null +++ b/ext/enterprise_script_service/mruby_no_gen_gc_config.rb @@ -0,0 +1,5 @@ +require_relative("./flags") +require_relative("./mruby_shared_config") + +build(Flags.io_safe_defines_with_no_gc) +crossbuild("sandbox_no_gen_gc", Flags.defines_with_no_gc) diff --git a/ext/enterprise_script_service/mruby_shared_config.rb b/ext/enterprise_script_service/mruby_shared_config.rb new file mode 100644 index 0000000..ef790d2 --- /dev/null +++ b/ext/enterprise_script_service/mruby_shared_config.rb @@ -0,0 +1,41 @@ +require_relative("./flags") + +MRUBY_ENGINE_GEMBOX_PATH = Pathname.new(__FILE__).dirname.join("mruby_engine") + +def build(defines) + MRuby::Build.new do |conf| + toolchain(:gcc) + + enable_debug + + conf.gembox(MRUBY_ENGINE_GEMBOX_PATH) + conf.gem(core: "mruby-bin-mirb") + conf.gem(core: 'mruby-bin-mruby') + + conf.bins = ["mrbc", "mruby"] + + conf.cc do |cc| + cc.flags += %w(-fPIC) + cc.flags += Flags.cflags + cc.defines += defines + end + end +end + +def crossbuild(directory, defines) + MRuby::CrossBuild.new(directory) do |conf| + toolchain(:gcc) + + enable_debug + + conf.gembox(MRUBY_ENGINE_GEMBOX_PATH) + + conf.bins = [] + + conf.cc do |cc| + cc.flags += %w(-fPIC) + cc.flags += Flags.cflags + cc.defines += defines + end + end +end diff --git a/lib/enterprise_script_service.rb b/lib/enterprise_script_service.rb index 353b066..9e4dda8 100644 --- a/lib/enterprise_script_service.rb +++ b/lib/enterprise_script_service.rb @@ -14,7 +14,7 @@ module EnterpriseScriptService class << self - def run(input:, sources:, instructions: nil, timeout: 1, instruction_quota: 100000, instruction_quota_start: 0) + def run(input:, sources:, instructions: nil, timeout: 1, instruction_quota: 100000, instruction_quota_start: 0, generational_gc: true) packer = EnterpriseScriptService::Protocol.packer_factory.packer payload = {input: input, sources: sources} @@ -26,7 +26,7 @@ def run(input:, sources:, instructions: nil, timeout: 1, instruction_quota: 1000 spawner = EnterpriseScriptService::Spawner.new service_process = EnterpriseScriptService::ServiceProcess.new( - service_path, + generational_gc ? service_path : service_path_no_generational_gc, spawner, instruction_quota, instruction_quota_start, @@ -47,5 +47,12 @@ def service_path base_path.join("bin/enterprise_script_service".freeze).to_s end end + + def service_path_no_generational_gc + @service_path_no_generational_gc ||= begin + base_path = Pathname.new(__dir__).parent + base_path.join("bin/enterprise_script_service_no_gen_gc".freeze).to_s + end + end end end diff --git a/script/mkmruby b/script/mkmruby index 65f1b15..0899c00 100755 --- a/script/mkmruby +++ b/script/mkmruby @@ -28,10 +28,6 @@ def within_mruby end case ARGV[0] -when "compile" - within_mruby do - sh("ruby", "./minirake") - end when "clean" within_mruby do sh("ruby", "./minirake", "clean") @@ -41,5 +37,5 @@ when "clobber" sh("ruby", "./minirake", "deep_clean") end else - puts("#{__FILE__} compile|clean|clobber") + puts("#{__FILE__} clean|clobber") end diff --git a/spec/script_service_spec.rb b/spec/script_service_spec.rb index 6dd02d4..006a1a3 100644 --- a/spec/script_service_spec.rb +++ b/spec/script_service_spec.rb @@ -282,6 +282,21 @@ def foo assert_roundtrip_input("😅") end + it "handles generational_gc being false" do + result = EnterpriseScriptService.run( + input: {result: [26803196617, 0.475]}, + sources: [ + ["stdout", "@stdout_buffer = 'hello'"], + ["foo", "@output = @input[:result]"], + ], + timeout: 1000, + generational_gc: false + ) + expect(result.success?).to be(true) + expect(result.output).to eq([26803196617, 0.475]) + expect(result.stdout).to eq("hello") + end + private def assert_roundtrip_input(input)