diff --git a/Gemfile b/Gemfile index 0f3159535b..be639a7326 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ gem "sass-rails", "~> 6.0" gem "sentry-rails" gem "sentry-ruby" gem "shell-spinner" -gem "sidekiq", "<7" +gem "sidekiq", "<8" gem "slim-rails", "~> 3.6" gem "sprockets", "~> 4.2.1" diff --git a/Gemfile.lock b/Gemfile.lock index 2191239121..fbd79c82ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -288,6 +288,7 @@ GEM kaminari-core (1.2.2) language_server-protocol (3.17.0.3) libreconv (0.9.5) + logger (1.6.4) lograge (0.12.0) actionpack (>= 4) activesupport (>= 4) @@ -435,7 +436,8 @@ GEM rdoc (6.7.0) psych (>= 4.0.0) recursive-open-struct (1.2.2) - redis (4.8.1) + redis-client (0.23.0) + connection_pool regexp_parser (2.9.2) reline (0.5.9) io-console (~> 0.5) @@ -539,10 +541,11 @@ GEM colorize shoulda-matchers (5.1.0) activesupport (>= 5.2.0) - sidekiq (6.5.12) - connection_pool (>= 2.2.5, < 3) - rack (~> 2.0) - redis (>= 4.5.0, < 5) + sidekiq (7.3.7) + connection_pool (>= 2.3.0) + logger + rack (>= 2.2.4) + redis-client (>= 0.22.2) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -688,7 +691,7 @@ DEPENDENCIES sentry-ruby shell-spinner shoulda-matchers (~> 5.1) - sidekiq (< 7) + sidekiq (< 8) simplecov simplecov-json site_prism (< 5.0) diff --git a/app/controllers/heartbeat_controller.rb b/app/controllers/heartbeat_controller.rb index 52c529b258..40e986888e 100644 --- a/app/controllers/heartbeat_controller.rb +++ b/app/controllers/heartbeat_controller.rb @@ -37,7 +37,7 @@ def healthcheck private def redis_alive? - Sidekiq.redis_info + Sidekiq.redis(&:info) true rescue StandardError false diff --git a/app/services/stats/base_closed_cases_report.rb b/app/services/stats/base_closed_cases_report.rb index eec0f34c6e..7ffe21c3c3 100644 --- a/app/services/stats/base_closed_cases_report.rb +++ b/app/services/stats/base_closed_cases_report.rb @@ -30,10 +30,9 @@ def process(report_guid:) report = Report.find_by(guid: report_guid) # Put the generated report into Redis for consumption by web app - redis = Redis.new data = nil File.open(etl_handler.results_filepath, "r") { |f| data = f.read } - redis.set(report_guid, data, ex: 7.days) + Sidekiq.redis { |r| r.set(report_guid, data, ex: 7.days.to_i) } if report report.report_data = { @@ -74,11 +73,10 @@ def run(**args) end def report_details(report) - redis = Redis.new - if redis.exists?(report.guid) + if Sidekiq.redis { |r| r.exists(report.guid).positive? } report.status = Stats::BaseReport::COMPLETE report.save! - redis.get(report.guid) + Sidekiq.redis { |r| r.get(report.guid) } end end end diff --git a/app/services/stats/base_monthly_performance_report.rb b/app/services/stats/base_monthly_performance_report.rb index d97e136100..0d9729ce4d 100644 --- a/app/services/stats/base_monthly_performance_report.rb +++ b/app/services/stats/base_monthly_performance_report.rb @@ -66,8 +66,7 @@ def process(offset, report_job_guid: nil, record_limit: ROWS_PER_FRAGMENT) .each { |kase| analyse_case(kase) } unless report_job_guid.nil? - redis = Redis.new - redis.set(report_job_guid, @stats.stats.to_json, ex: 7.days) + Sidekiq.redis { |r| r.set(report_job_guid, @stats.stats.to_json, ex: 7.days.to_i) } end end @@ -102,11 +101,10 @@ def to_csv # This function is only when the report is done via ETL (tasks) def report_details(report) - redis = Redis.new data_collector = [] report.job_ids.each do |job_id| - if redis.exists?(job_id) - data_collector << redis.get(job_id) + if Sidekiq.redis { |r| r.exists(job_id).positive? } + data_collector << Sidekiq.redis { |r| r.get(job_id) } end end diff --git a/lib/db/database_anonymizer_tasks.rb b/lib/db/database_anonymizer_tasks.rb index 789aeddfca..1b399c75c9 100644 --- a/lib/db/database_anonymizer_tasks.rb +++ b/lib/db/database_anonymizer_tasks.rb @@ -26,7 +26,6 @@ def execute_task(task_name, task_arguments) end def store_anonymise_status(task_arguments, tasks) - redis = Redis.new tasks_ids = tasks.map { |task| task[:task_id] } anonymizer_job_info = { "start_time": task_arguments[:timestamp], @@ -35,7 +34,7 @@ def store_anonymise_status(task_arguments, tasks) "tasks": tasks_ids, } anonymizer_job_id = "anonymizer_job_#{task_arguments[:tag]}_#{Time.zone.today.strftime('%Y%m%d')}" - redis.set(anonymizer_job_id, anonymizer_job_info.to_json, ex: 7.days) + Sidekiq.redis { |r| r.set(anonymizer_job_id, anonymizer_job_info.to_json, ex: 7.days.to_i) } end private diff --git a/sidekiq_admin.ru b/sidekiq_admin.ru deleted file mode 100644 index 447add9eec..0000000000 --- a/sidekiq_admin.ru +++ /dev/null @@ -1,9 +0,0 @@ -require "sidekiq" - -Sidekiq.configure_client do |config| - config.redis = { size: 1 } -end - -require "sidekiq/web" -use Rack::Session::Cookie, secret: ENV.fetch("SESSION_SECRET_KEY") -run Sidekiq::Web diff --git a/spec/controllers/heartbeat_controller_spec.rb b/spec/controllers/heartbeat_controller_spec.rb index 7f79b90ba9..18f07cea79 100644 --- a/spec/controllers/heartbeat_controller_spec.rb +++ b/spec/controllers/heartbeat_controller_spec.rb @@ -17,6 +17,8 @@ end describe "#healthcheck" do + let(:redis) { double("Redis") } # rubocop:disable RSpec/VerifiedDoubles + before do retry_set = instance_double(Sidekiq::RetrySet, size: 0) allow(Sidekiq::RetrySet).to receive(:new).and_return(retry_set) @@ -28,7 +30,8 @@ allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set) dead_set = instance_double(Sidekiq::DeadSet, size: 1) allow(Sidekiq::DeadSet).to receive(:new).and_return(dead_set) - allow(Sidekiq).to receive(:redis_info).and_raise(Errno::ECONNREFUSED) + allow(Sidekiq).to receive(:redis).and_yield(redis) + allow(redis).to receive(:info).and_raise(Errno::ECONNREFUSED) allow(ActiveRecord::Base.connection) .to receive(:execute).and_raise(PG::ConnectionBad) @@ -55,7 +58,8 @@ allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set) dead_set = instance_double(Sidekiq::DeadSet, size: 0) allow(Sidekiq::DeadSet).to receive(:new).and_return(dead_set) - allow(Sidekiq).to receive(:redis_info).and_return({}) + allow(Sidekiq).to receive(:redis).and_yield(redis) + allow(redis).to receive(:info).and_return({}) get :healthcheck end diff --git a/spec/services/stats/base_monthly_performance_report_spec.rb b/spec/services/stats/base_monthly_performance_report_spec.rb index d205ca447a..035c3413e6 100644 --- a/spec/services/stats/base_monthly_performance_report_spec.rb +++ b/spec/services/stats/base_monthly_performance_report_spec.rb @@ -126,100 +126,103 @@ def case_scope end end - describe "#process" do - it "creates data in Redis with an expiry" do - guid = SecureRandom.uuid - redis_double = instance_double(Redis) - allow(Redis).to receive(:new).and_return(redis_double) - expect(redis_double).to receive(:set).with(guid, anything, ex: 7.days) - - new_report = DummyPerformanceReport.new( - report_type_id: find_or_create(:report_type, :r205).id, - period_start: @period_start, - period_end: @period_end, - ) - new_report.process(0, report_job_guid: guid) + context "with redis" do + let(:redis) { double("Redis") } # rubocop:disable RSpec/VerifiedDoubles + + before do + allow(Sidekiq).to receive(:redis).and_yield(redis) + allow(redis).to receive(:exists).and_return(1) end - end - describe "#report_details" do - it "more months compared with period from stats data" do - redis_double = instance_double(Redis) - allow(Redis).to receive(:new).and_return(redis_double) - - allow(redis_double).to receive(:exists?).and_return(true) - allow(redis_double).to receive(:get).and_return( - '{"201811": - {"month":0, - "non_trigger_performance":0, - "non_trigger_total":0, - "non_trigger_responded_in_time":0, - "non_trigger_responded_late":0, - "non_trigger_open_in_time":30, - "non_trigger_open_late":5, - "trigger_performance":0, - "trigger_total":0, - "trigger_responded_in_time":0, - "trigger_responded_late":0, - "trigger_open_in_time":0, - "trigger_open_late":0, - "overall_performance":0, - "overall_total":0, - "overall_responded_in_time":0, - "overall_responded_late":0, - "overall_open_in_time":30, - "overall_open_late":5}, - "201812": - { "month":0, - "non_trigger_performance":0, - "non_trigger_total":0, - "non_trigger_responded_in_time":0, - "non_trigger_responded_late":0, - "non_trigger_open_in_time":20, - "non_trigger_open_late":10, - "trigger_performance":0, - "trigger_total":0, - "trigger_open_in_time":0, - "trigger_open_late":0, - "trigger_responded_in_time":0, - "trigger_responded_late":0, - "overall_performance":0, - "overall_total":0, - "overall_responded_in_time":0, - "overall_responded_late":0, - "overall_open_in_time":20, - "overall_open_late":10}, - "total": - { "month":0, - "non_trigger_performance":0, - "non_trigger_total":0, - "non_trigger_responded_in_time":0, - "non_trigger_responded_late":0, - "non_trigger_open_in_time":0, - "non_trigger_open_late":0, - "trigger_performance":0, - "trigger_total":0, - "trigger_open_in_time":0, - "trigger_open_late":0, - "trigger_responded_in_time":0, - "trigger_responded_late":0, - "overall_performance":0, - "overall_total":0, - "overall_responded_in_time":0, - "overall_responded_late":0, - "overall_open_in_time":50, - "overall_open_late":15}}', - ) - new_report = Report.new( - report_type_id: find_or_create(:report_type, :r205).id, - period_start: @period_start, - period_end: @period_end, - ) - new_report.job_ids = %w[job1] - result_data = JSON.parse(@report.report_details(new_report)) - expect(result_data.key?("201812")).to eq true - expect(result_data.key?("total")).to eq true - expect(result_data.key?("201811")).to eq false + describe "#process" do + it "creates data in Redis with an expiry" do + guid = SecureRandom.uuid + expect(redis).to receive(:set).with(guid, anything, ex: 7.days.to_i) + + new_report = DummyPerformanceReport.new( + report_type_id: find_or_create(:report_type, :r205).id, + period_start: @period_start, + period_end: @period_end, + ) + new_report.process(0, report_job_guid: guid) + end + end + + describe "#report_details" do + it "more months compared with period from stats data" do + allow(redis).to receive(:get).and_return( + '{"201811": + {"month":0, + "non_trigger_performance":0, + "non_trigger_total":0, + "non_trigger_responded_in_time":0, + "non_trigger_responded_late":0, + "non_trigger_open_in_time":30, + "non_trigger_open_late":5, + "trigger_performance":0, + "trigger_total":0, + "trigger_responded_in_time":0, + "trigger_responded_late":0, + "trigger_open_in_time":0, + "trigger_open_late":0, + "overall_performance":0, + "overall_total":0, + "overall_responded_in_time":0, + "overall_responded_late":0, + "overall_open_in_time":30, + "overall_open_late":5}, + "201812": + { "month":0, + "non_trigger_performance":0, + "non_trigger_total":0, + "non_trigger_responded_in_time":0, + "non_trigger_responded_late":0, + "non_trigger_open_in_time":20, + "non_trigger_open_late":10, + "trigger_performance":0, + "trigger_total":0, + "trigger_open_in_time":0, + "trigger_open_late":0, + "trigger_responded_in_time":0, + "trigger_responded_late":0, + "overall_performance":0, + "overall_total":0, + "overall_responded_in_time":0, + "overall_responded_late":0, + "overall_open_in_time":20, + "overall_open_late":10}, + "total": + { "month":0, + "non_trigger_performance":0, + "non_trigger_total":0, + "non_trigger_responded_in_time":0, + "non_trigger_responded_late":0, + "non_trigger_open_in_time":0, + "non_trigger_open_late":0, + "trigger_performance":0, + "trigger_total":0, + "trigger_open_in_time":0, + "trigger_open_late":0, + "trigger_responded_in_time":0, + "trigger_responded_late":0, + "overall_performance":0, + "overall_total":0, + "overall_responded_in_time":0, + "overall_responded_late":0, + "overall_open_in_time":50, + "overall_open_late":15}}', + ) + new_report = Report.new( + report_type_id: find_or_create(:report_type, :r205).id, + period_start: @period_start, + period_end: @period_end, + ) + new_report.job_ids = %w[job1] + result_data = JSON.parse(@report.report_details(new_report)) + expect(result_data.key?("201812")).to eq true + expect(result_data.key?("total")).to eq true + expect(result_data.key?("201811")).to eq false + end end end end diff --git a/spec/services/stats/r007_closed_cases_report_spec.rb b/spec/services/stats/r007_closed_cases_report_spec.rb index 0c4cf6d6a9..c8ebb6e238 100644 --- a/spec/services/stats/r007_closed_cases_report_spec.rb +++ b/spec/services/stats/r007_closed_cases_report_spec.rb @@ -96,18 +96,18 @@ module Stats end describe "#report_details" do - let(:redis) { instance_double(Redis) } + let(:redis) { double("redis") } # rubocop:disable RSpec/VerifiedDoubles let(:report) { create(:report) } let(:data) { "some data" } before do - allow(Redis).to receive(:new).and_return(redis) - allow(redis).to receive(:exists?).with(report.guid).and_return(exists) + allow(Sidekiq).to receive(:redis).and_yield(redis) + allow(redis).to receive(:exists).with(report.guid).and_return(exists) allow(redis).to receive(:get).with(report.guid).and_return(data) end context "when data is in redis" do - let(:exists) { true } + let(:exists) { 1 } it "sets the report as complete" do expect { @@ -121,7 +121,7 @@ module Stats end context "when data is not in redis" do - let(:exists) { false } + let(:exists) { 0 } it "doesn't change the report status" do expect {