From 6753ec56ea9bb8beb559b97731a9ba61da37a204 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Fri, 1 Mar 2024 15:08:09 -0800 Subject: [PATCH 1/7] Add Health Endpoint Work towards #1202 --- Gemfile | 1 + Gemfile.lock | 3 +++ app/checks/solr_status.rb | 10 ++++++++++ config/initializers/health_monitor.rb | 17 +++++++++++++++++ config/routes.rb | 1 + spec/checks/solr_status_spec.rb | 8 ++++++++ spec/requests/health_check_spec.rb | 11 +++++++++++ 7 files changed, 51 insertions(+) create mode 100644 app/checks/solr_status.rb create mode 100644 config/initializers/health_monitor.rb create mode 100644 spec/checks/solr_status_spec.rb create mode 100644 spec/requests/health_check_spec.rb diff --git a/Gemfile b/Gemfile index ae6c9a7e..fd84f4c3 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ source "https://rubygems.org" gem "autoprefixer-rails" gem "ddtrace", "~> 0.54" +gem "health-monitor-rails" gem "jquery-rails" gem "lograge" gem "logstash-event" diff --git a/Gemfile.lock b/Gemfile.lock index 36e3970b..5d6142f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -320,6 +320,8 @@ GEM tilt (>= 1.2) hashdiff (1.0.1) hashie (5.0.0) + health-monitor-rails (11.3.0) + railties (>= 6.1) honeybadger (5.0.0) httpclient (2.8.3) i18n (1.14.1) @@ -683,6 +685,7 @@ DEPENDENCIES geoblacklight (= 3.7.0) geoblacklight_sidecar_images google-cloud-storage (~> 1.11) + health-monitor-rails honeybadger jquery-rails lograge diff --git a/app/checks/solr_status.rb b/app/checks/solr_status.rb new file mode 100644 index 00000000..6f44230d --- /dev/null +++ b/app/checks/solr_status.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +class SolrStatus < HealthMonitor::Providers::Base + def check! + uri = Blacklight.default_index.connection.uri + status_uri = URI(uri.to_s.gsub(uri.path, "/solr/admin/cores?action=STATUS")) + response = Net::HTTP.get(status_uri) + json = JSON.parse(response) + raise "The solr has an invalid status #{status_uri}" if json["responseHeader"]["status"] != 0 + end +end diff --git a/config/initializers/health_monitor.rb b/config/initializers/health_monitor.rb new file mode 100644 index 00000000..cff56238 --- /dev/null +++ b/config/initializers/health_monitor.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +Rails.application.config.after_initialize do + HealthMonitor.configure do |config| + config.cache + config.redis + + config.add_custom_provider(SolrStatus) + + # Make this health check available at /health + config.path = :health + + config.error_callback = proc do |e| + Rails.logger.error "Health check failed with: #{e.message}" + Honeybadger.notify(e) + end + end +end diff --git a/config/routes.rb b/config/routes.rb index d0fffff0..d8cc8bcb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true Rails.application.routes.draw do + mount HealthMonitor::Engine, at: "/" concern :range_searchable, BlacklightRangeLimit::Routes::RangeSearchable.new mount Blacklight::Engine => "/" mount BlacklightAdvancedSearch::Engine => "/" diff --git a/spec/checks/solr_status_spec.rb b/spec/checks/solr_status_spec.rb new file mode 100644 index 00000000..d3623115 --- /dev/null +++ b/spec/checks/solr_status_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +require "rails_helper" + +RSpec.describe SolrStatus do + it "doesn't error when Solr's fine" do + expect { described_class.new.check! }.not_to raise_error + end +end diff --git a/spec/requests/health_check_spec.rb b/spec/requests/health_check_spec.rb new file mode 100644 index 00000000..e2787ba0 --- /dev/null +++ b/spec/requests/health_check_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true +require "rails_helper" + +RSpec.describe "Health Check", type: :request do + describe "GET /health" do + it "has a health check" do + get "/health" + expect(response).to be_successful + end + end +end From 99e5d011f678cfa6e3c89ffa20a08b4e21d063df Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Fri, 1 Mar 2024 15:22:13 -0800 Subject: [PATCH 2/7] Support basic auth for Solr. --- app/checks/solr_status.rb | 6 ++++-- spec/checks/solr_status_spec.rb | 8 -------- spec/requests/health_check_spec.rb | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 spec/checks/solr_status_spec.rb diff --git a/app/checks/solr_status.rb b/app/checks/solr_status.rb index 6f44230d..d8001590 100644 --- a/app/checks/solr_status.rb +++ b/app/checks/solr_status.rb @@ -3,8 +3,10 @@ class SolrStatus < HealthMonitor::Providers::Base def check! uri = Blacklight.default_index.connection.uri status_uri = URI(uri.to_s.gsub(uri.path, "/solr/admin/cores?action=STATUS")) - response = Net::HTTP.get(status_uri) - json = JSON.parse(response) + req = Net::HTTP::Get.new(status_uri) + req.basic_auth(uri.user, uri.password) if uri.user && uri.password + response = Net::HTTP.start(uri.hostname, uri.port) { |http| http.request(req) } + json = JSON.parse(response.body) raise "The solr has an invalid status #{status_uri}" if json["responseHeader"]["status"] != 0 end end diff --git a/spec/checks/solr_status_spec.rb b/spec/checks/solr_status_spec.rb deleted file mode 100644 index d3623115..00000000 --- a/spec/checks/solr_status_spec.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true -require "rails_helper" - -RSpec.describe SolrStatus do - it "doesn't error when Solr's fine" do - expect { described_class.new.check! }.not_to raise_error - end -end diff --git a/spec/requests/health_check_spec.rb b/spec/requests/health_check_spec.rb index e2787ba0..c6768bd8 100644 --- a/spec/requests/health_check_spec.rb +++ b/spec/requests/health_check_spec.rb @@ -4,7 +4,7 @@ RSpec.describe "Health Check", type: :request do describe "GET /health" do it "has a health check" do - get "/health" + get "/health.json" expect(response).to be_successful end end From 2e50baecb581447de0adf84091137362e134eda1 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Fri, 1 Mar 2024 15:28:48 -0800 Subject: [PATCH 3/7] Don't check redis in test. --- config/initializers/health_monitor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/health_monitor.rb b/config/initializers/health_monitor.rb index cff56238..0e990156 100644 --- a/config/initializers/health_monitor.rb +++ b/config/initializers/health_monitor.rb @@ -2,7 +2,7 @@ Rails.application.config.after_initialize do HealthMonitor.configure do |config| config.cache - config.redis + config.redis unless Rails.env.test? config.add_custom_provider(SolrStatus) From 8401ea6e52c2cd83d1ced85816dc2b3bb0be2dc2 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Mon, 4 Mar 2024 15:21:19 -0800 Subject: [PATCH 4/7] Add health spec. --- spec/requests/health_check_spec.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spec/requests/health_check_spec.rb b/spec/requests/health_check_spec.rb index c6768bd8..107ca901 100644 --- a/spec/requests/health_check_spec.rb +++ b/spec/requests/health_check_spec.rb @@ -7,5 +7,19 @@ get "/health.json" expect(response).to be_successful end + + it "errors when a service is down" do + allow(Blacklight.default_index.connection).to receive(:uri).and_return(URI("http://example.com/bla")) + stub_request(:get, "http://example.com/solr/admin/cores?action=STATUS").to_return( + body: { responseHeader: { status: 500 } }.to_json, headers: { "Content-Type" => "text/json" } + ) + + get "/health.json" + + expect(response).not_to be_successful + expect(response.status).to eq 503 + solr_response = JSON.parse(response.body)["results"].find { |x| x["name"] == "SolrStatus" } + expect(solr_response["message"]).to start_with "The solr has an invalid status" + end end end From 64cfddbf6c25ec09b7c960afca4c9972e77b53a0 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Thu, 7 Mar 2024 11:32:25 -0800 Subject: [PATCH 5/7] Add Webmock --- Gemfile | 1 + Gemfile.lock | 9 ++++++++- spec/spec_helper.rb | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index fd84f4c3..d97a5190 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ end group :test do gem "axe-core-rspec" + gem "webmock" end gem "blacklight", "7.31.0" diff --git a/Gemfile.lock b/Gemfile.lock index 5d6142f5..e921aabf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -148,6 +148,8 @@ GEM deep_merge (~> 1.2, >= 1.2.1) dry-validation (~> 1.0, >= 1.0.0) connection_pool (2.4.1) + crack (0.4.5) + rexml crass (1.0.6) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) @@ -644,6 +646,10 @@ GEM nokogiri (~> 1.6) rubyzip (>= 1.3.0) selenium-webdriver (~> 4.0, < 4.11) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) webrick (1.7.0) websocket (1.2.10) websocket-driver (0.7.6) @@ -721,7 +727,8 @@ DEPENDENCIES vite_rails web-console webdrivers + webmock whenever BUNDLED WITH - 2.3.18 + 2.4.10 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 03e3b00f..361b2a08 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true ENV["RACK_ENV"] = "test" +require "webmock/rspec" require "simplecov" require "capybara/rspec" require "capybara-screenshot/rspec" @@ -32,3 +33,8 @@ mocks.verify_partial_doubles = true end end + +WebMock.disable_net_connect!(allow_localhost: true, + net_http_connect_on_start: true, + allow: ["chromedriver.storage.googleapis.com", "googlechromelabs.github.io", + "edgedl.me.gvt1.com"]) From 2484c12be151948fd6a6c65ff6eef6958ef0d1a9 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Thu, 7 Mar 2024 11:34:31 -0800 Subject: [PATCH 6/7] Add exception. --- spec/spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 361b2a08..359805ea 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -37,4 +37,5 @@ WebMock.disable_net_connect!(allow_localhost: true, net_http_connect_on_start: true, allow: ["chromedriver.storage.googleapis.com", "googlechromelabs.github.io", + "https://storage.googleapis.com/chrome-for-testing-public", "edgedl.me.gvt1.com"]) From d8ca3d6a34418d318743270f2cb2e8e35382f56f Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Thu, 7 Mar 2024 11:35:48 -0800 Subject: [PATCH 7/7] Add exception for downloading Chrome. --- spec/spec_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 359805ea..84098fce 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -37,5 +37,5 @@ WebMock.disable_net_connect!(allow_localhost: true, net_http_connect_on_start: true, allow: ["chromedriver.storage.googleapis.com", "googlechromelabs.github.io", - "https://storage.googleapis.com/chrome-for-testing-public", + "storage.googleapis.com", "edgedl.me.gvt1.com"])