diff --git a/.gitignore b/.gitignore index cec3cb5..0a55386 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.gem +dump.rdb Gemfile.lock diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..ed95882 --- /dev/null +++ b/Rakefile @@ -0,0 +1,9 @@ +require "rake/testtask" + +Rake::TestTask.new do |t| + t.libs << "test" + t.test_files = FileList['test/test*.rb'] + t.verbose = true +end + +task default: :test diff --git a/rspecq.gemspec b/rspecq.gemspec index e9a90aa..41b1fe4 100644 --- a/rspecq.gemspec +++ b/rspecq.gemspec @@ -15,6 +15,8 @@ Gem::Specification.new do |s| s.add_dependency "rspec-core" s.add_dependency "redis" - s.add_development_dependency "minitest", "~> 5.14" s.add_development_dependency "rake" + s.add_development_dependency "pry-byebug" + s.add_development_dependency "minitest" + s.add_development_dependency "rspec" end diff --git a/test/sample_suites/README b/test/sample_suites/README new file mode 100644 index 0000000..64b2119 --- /dev/null +++ b/test/sample_suites/README @@ -0,0 +1 @@ +Sample RSpec suites, used as fixtures in the tests. diff --git a/test/sample_suites/different_spec_path/mytests/foo_spec.rb b/test/sample_suites/different_spec_path/mytests/foo_spec.rb new file mode 100644 index 0000000..3333d33 --- /dev/null +++ b/test/sample_suites/different_spec_path/mytests/foo_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe do + it "I should not be executed!" do + expect(1).to eq 2 + end +end diff --git a/test/sample_suites/different_spec_path/mytests/qwe_spec.rb b/test/sample_suites/different_spec_path/mytests/qwe_spec.rb new file mode 100644 index 0000000..5b01dfe --- /dev/null +++ b/test/sample_suites/different_spec_path/mytests/qwe_spec.rb @@ -0,0 +1,13 @@ +RSpec.describe do + context "foo" do + describe "abc" do + it { expect(false).to be false } + end + end + + context "bar" do + describe "dfg" do + it { expect(true).to be true } + end + end +end diff --git a/test/sample_suites/failing_suite/spec/bar_spec.rb b/test/sample_suites/failing_suite/spec/bar_spec.rb new file mode 100644 index 0000000..47f8ecf --- /dev/null +++ b/test/sample_suites/failing_suite/spec/bar_spec.rb @@ -0,0 +1,4 @@ +RSpec.describe do + it { expect(false).to be false } + it { expect(1).to be 2 } +end diff --git a/test/sample_suites/failing_suite/spec/foo_spec.rb b/test/sample_suites/failing_suite/spec/foo_spec.rb new file mode 100644 index 0000000..04208a3 --- /dev/null +++ b/test/sample_suites/failing_suite/spec/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe do + it { expect(true).to be true } +end diff --git a/test/sample_suites/flakey_suite/spec/foo_spec.rb b/test/sample_suites/flakey_suite/spec/foo_spec.rb new file mode 100644 index 0000000..b8e0d8a --- /dev/null +++ b/test/sample_suites/flakey_suite/spec/foo_spec.rb @@ -0,0 +1,8 @@ +RSpec.describe do + it do + $tries ||= 0 + $tries += 1 + + expect($tries).to eq 3 + end +end diff --git a/test/sample_suites/non_example_error/spec/bar_spec.rb b/test/sample_suites/non_example_error/spec/bar_spec.rb new file mode 100644 index 0000000..04208a3 --- /dev/null +++ b/test/sample_suites/non_example_error/spec/bar_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe do + it { expect(true).to be true } +end diff --git a/test/sample_suites/non_example_error/spec/foo_spec.rb b/test/sample_suites/non_example_error/spec/foo_spec.rb new file mode 100644 index 0000000..873b996 --- /dev/null +++ b/test/sample_suites/non_example_error/spec/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe IDONTEXISTZ do + it { expect(true).to be true } +end diff --git a/test/sample_suites/passing_suite/spec/foo_spec.rb b/test/sample_suites/passing_suite/spec/foo_spec.rb new file mode 100644 index 0000000..04208a3 --- /dev/null +++ b/test/sample_suites/passing_suite/spec/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe do + it { expect(true).to be true } +end diff --git a/test/sample_suites/scheduling/spec/bar_spec.rb b/test/sample_suites/scheduling/spec/bar_spec.rb new file mode 100644 index 0000000..318063c --- /dev/null +++ b/test/sample_suites/scheduling/spec/bar_spec.rb @@ -0,0 +1,9 @@ +RSpec.describe do + it do + expect(true).to be true + end + + it do + expect(true).to be true + end +end diff --git a/test/sample_suites/scheduling/spec/foo_spec.rb b/test/sample_suites/scheduling/spec/foo_spec.rb new file mode 100644 index 0000000..e8d3261 --- /dev/null +++ b/test/sample_suites/scheduling/spec/foo_spec.rb @@ -0,0 +1,13 @@ +RSpec.describe "slow spec file (will be split)" do + it do + sleep 0.1 + expect(true).to be true + end + + context "foo" do + it do + sleep 0.2 + expect(true).to be true + end + end +end diff --git a/test/sample_suites/spec_file_splitting/spec/fast_spec.rb b/test/sample_suites/spec_file_splitting/spec/fast_spec.rb new file mode 100644 index 0000000..ae8a773 --- /dev/null +++ b/test/sample_suites/spec_file_splitting/spec/fast_spec.rb @@ -0,0 +1,4 @@ +RSpec.describe do + it { expect(true).to be true } + it { expect(true).to be true } +end diff --git a/test/sample_suites/spec_file_splitting/spec/slow_spec.rb b/test/sample_suites/spec_file_splitting/spec/slow_spec.rb new file mode 100644 index 0000000..3ad6695 --- /dev/null +++ b/test/sample_suites/spec_file_splitting/spec/slow_spec.rb @@ -0,0 +1,13 @@ +RSpec.describe do + it do + sleep 0.6 + expect(true).to be true + end + + context "foo" do + it do + sleep 0.6 + expect(true).to be true + end + end +end diff --git a/test/sample_suites/timings/spec/fast_spec.rb b/test/sample_suites/timings/spec/fast_spec.rb new file mode 100644 index 0000000..cb0d23b --- /dev/null +++ b/test/sample_suites/timings/spec/fast_spec.rb @@ -0,0 +1,6 @@ +RSpec.describe do + it do + sleep 0.1 + expect(true).to be true + end +end diff --git a/test/sample_suites/timings/spec/medium_spec.rb b/test/sample_suites/timings/spec/medium_spec.rb new file mode 100644 index 0000000..61fab93 --- /dev/null +++ b/test/sample_suites/timings/spec/medium_spec.rb @@ -0,0 +1,6 @@ +RSpec.describe do + it do + sleep 0.2 + expect(true).to be true + end +end diff --git a/test/sample_suites/timings/spec/slow_spec.rb b/test/sample_suites/timings/spec/slow_spec.rb new file mode 100644 index 0000000..a49aea9 --- /dev/null +++ b/test/sample_suites/timings/spec/slow_spec.rb @@ -0,0 +1,6 @@ +RSpec.describe do + it do + sleep 0.3 + expect(true).to be true + end +end diff --git a/test/sample_suites/timings/spec/very_fast_spec.rb b/test/sample_suites/timings/spec/very_fast_spec.rb new file mode 100644 index 0000000..d8b632c --- /dev/null +++ b/test/sample_suites/timings/spec/very_fast_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe do + it do + expect(true).to be true + end +end diff --git a/test/sample_suites/timings/spec/very_slow_spec.rb b/test/sample_suites/timings/spec/very_slow_spec.rb new file mode 100644 index 0000000..b63fdf4 --- /dev/null +++ b/test/sample_suites/timings/spec/very_slow_spec.rb @@ -0,0 +1,6 @@ +RSpec.describe do + it do + sleep 0.4 + expect(true).to be true + end +end diff --git a/test/test_e2e.rb b/test/test_e2e.rb new file mode 100644 index 0000000..7b21bfe --- /dev/null +++ b/test/test_e2e.rb @@ -0,0 +1,96 @@ +require "test_helpers" + +class TestEndToEnd < RSpecQTest + def test_suite_with_legit_failures + queue = exec_build("failing_suite") + + refute queue.build_successful? + + assert_processed_jobs [ + "./spec/foo_spec.rb", + "./spec/bar_spec.rb", + "./spec/bar_spec.rb[1:2]", + ], queue + + assert_equal 3+RSpecQ::MAX_REQUEUES, queue.example_count + + assert_equal({ "./spec/bar_spec.rb[1:2]" => "3" }, queue.requeued_jobs) + end + + def test_passing_suite + queue = exec_build("passing_suite") + + assert queue.build_successful? + assert_build_not_flakey(queue) + assert_equal 1, queue.example_count + assert_equal ["./spec/foo_spec.rb"], queue.processed_jobs + end + + def test_flakey_suite + queue = exec_build("flakey_suite") + + assert queue.build_successful? + assert_processed_jobs [ + "./spec/foo_spec.rb", + "./spec/foo_spec.rb[1:1]", + ], queue + + assert_equal({ "./spec/foo_spec.rb[1:1]" => "2" }, queue.requeued_jobs) + end + + def test_scheduling_by_file_and_custom_spec_path + queue = exec_build("different_spec_path", "mytests/qwe_spec.rb") + + assert queue.build_successful? + assert_build_not_flakey(queue) + assert_equal 2, queue.example_count + assert_processed_jobs ["./mytests/qwe_spec.rb"], queue + end + + def test_non_example_error + queue = exec_build("non_example_error") + + refute queue.build_successful? + assert_build_not_flakey(queue) + assert_equal 1, queue.example_count + assert_processed_jobs ["./spec/foo_spec.rb", "./spec/bar_spec.rb"], queue + assert_equal ["./spec/foo_spec.rb"], queue.non_example_errors.keys + end + + def test_timings_update + queue = exec_build("timings", "--update-timings") + + assert queue.build_successful? + + assert_equal [ + "./spec/very_fast_spec.rb", + "./spec/fast_spec.rb", + "./spec/medium_spec.rb", + "./spec/slow_spec.rb", + "./spec/very_slow_spec.rb", + ], queue.timings.sort_by { |k,v| v }.map(&:first) + end + + def test_timings_no_update + queue = exec_build("timings") + + assert queue.build_successful? + assert_empty queue.timings + end + + def test_spec_file_splitting + queue = exec_build( "spec_file_splitting", "--update-timings") + assert queue.build_successful? + refute_empty queue.timings + + queue = exec_build( "spec_file_splitting", "--file-split-threshold 1") + + assert queue.build_successful? + refute_empty queue.timings + assert_processed_jobs([ + "./spec/slow_spec.rb[1:2:1]", + "./spec/slow_spec.rb[1:1]", + "./spec/fast_spec.rb", + ], queue) + end +end diff --git a/test/test_helpers.rb b/test/test_helpers.rb new file mode 100644 index 0000000..fe11854 --- /dev/null +++ b/test/test_helpers.rb @@ -0,0 +1,70 @@ +require "minitest/autorun" +require "securerandom" +require "rspecq" +require "pry-byebug" + +module TestHelpers + REDIS_HOST = "127.0.0.1".freeze + + def rand_id + SecureRandom.hex(4) + end + + def new_worker(path) + RSpecQ::Worker.new( + build_id: rand_id, + worker_id: rand_id, + redis_host: REDIS_HOST, + files_or_dirs_to_run: suite_path(path), + ) + end + + def exec_build(path, args="") + worker_id = rand_id + build_id = rand_id + + Dir.chdir(suite_path(path)) do + out = `bundle exec rspecq --worker #{worker_id} --build #{build_id} #{args}` + puts out if ENV["RSPECQ_DEBUG"] + end + + assert_equal 0, $?.exitstatus + + queue = RSpecQ::Queue.new(build_id, worker_id, REDIS_HOST) + assert_queue_well_formed(queue) + + return queue + end + + def assert_queue_well_formed(queue, msg=nil) + redis = queue.redis + heartbeats = redis.zrange( + queue.send(:key_worker_heartbeats), 0, -1, withscores: true) + + assert queue.published? + assert queue.exhausted? + assert_operator heartbeats.size, :>=, 0 + assert heartbeats.all? { |hb| Time.at(hb.last) <= Time.now } + end + + def assert_build_not_flakey(queue) + assert_empty queue.requeued_jobs + end + + def assert_processed_jobs(exp, queue) + assert_equal exp.sort, queue.processed_jobs.sort + end + + def suite_path(path) + File.join("test", "sample_suites", path) + end +end + +# To be subclassed from all test cases. +class RSpecQTest < Minitest::Test + include TestHelpers + + def setup + Redis.new(host: REDIS_HOST).flushdb + end +end diff --git a/test/test_scheduling.rb b/test/test_scheduling.rb new file mode 100644 index 0000000..c5db554 --- /dev/null +++ b/test/test_scheduling.rb @@ -0,0 +1,52 @@ +require "test_helpers" + +class TestScheduling < RSpecQTest + def test_scheduling_with_timings_simple + worker = new_worker("timings") + worker.populate_timings = true + worker.work + + worker = new_worker("timings") + # worker.populate_timings is false by default + queue = worker.queue + worker.try_publish_queue!(queue) + + assert_equal [ + "./test/sample_suites/timings/spec/very_slow_spec.rb", + "./test/sample_suites/timings/spec/slow_spec.rb", + "./test/sample_suites/timings/spec/medium_spec.rb", + "./test/sample_suites/timings/spec/fast_spec.rb", + "./test/sample_suites/timings/spec/very_fast_spec.rb" + ], queue.unprocessed_jobs + end + + def test_scheduling_with_timings_and_splitting + worker = new_worker("scheduling") + worker.populate_timings = true + worker.work + + # 1st run with timings, the slow file will be split + worker = new_worker("scheduling") + worker.populate_timings = true + worker.file_split_threshold = 0.2 + worker.work + + assert_processed_jobs([ + "./test/sample_suites/scheduling/spec/bar_spec.rb", + "./test/sample_suites/scheduling/spec/foo_spec.rb[1:1]", + "./test/sample_suites/scheduling/spec/foo_spec.rb[1:2:1]", + ], worker.queue) + + # 2nd run with timings; individual example jobs will also have timings now + worker = new_worker("scheduling") + worker.populate_timings = true + worker.file_split_threshold = 0.2 + worker.try_publish_queue!(worker.queue) + + assert_equal [ + "./test/sample_suites/scheduling/spec/foo_spec.rb[1:2:1]", + "./test/sample_suites/scheduling/spec/foo_spec.rb[1:1]", + "./test/sample_suites/scheduling/spec/bar_spec.rb", + ], worker.queue.unprocessed_jobs + end +end