Merge branch 'develop' into remove_redis-activesupport

.github/workflows/ruby-unit-tests.yml

@@ -7,6 +7,7 @@ on:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
         backend: ['api', 'api-agraph'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend
     runs-on: ubuntu-latest

Capfile

@@ -22,6 +22,6 @@ require 'capistrano/bundler'
 # require 'capistrano/rails/assets'
 # require 'capistrano/rails/migrations'
 require 'capistrano/locally'
-
+require 'new_relic/recipes' # announce deployments in NewRelic
 # Loads custom tasks from `lib/capistrano/tasks' if you have any defined.
 Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r }

Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 gem 'activesupport', '~> 3.2'
 # see https://github.com/ncbo/ontologies_api/issues/69
 gem 'bigdecimal', '1.4.2'
-gem 'faraday', '~> 2.0'
 gem 'json-schema', '~> 2.0'
 gem 'multi_json', '~> 1.0'
 gem 'oj', '~> 3.0'
@@ -31,7 +30,7 @@ gem 'redis-store', '~>1.10'
 
 # Monitoring
 gem 'cube-ruby', require: 'cube'
-gem 'newrelic_rpm'
+gem 'newrelic_rpm', group: [:default, :deployment]
 
 # HTTP server
 gem 'unicorn'
@@ -50,15 +49,19 @@ gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'de
 gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop'
 
 group :development do
+  # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
+  gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo'
+  gem 'rubocop'
+end
+
+group :deployment do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
   gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
   gem 'capistrano', '~> 3', require: false
   gem 'capistrano-bundler', require: false
   gem 'capistrano-locally', require: false
   gem 'capistrano-rbenv', require: false
   gem 'ed25519', '>= 1.2', '< 2.0', require: false
-  gem 'pry'
-  gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo'
 end
 
 group :profiling do

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/goo.git
-  revision: fb203b0396d03c1df61abfcdbc4070787010f052
+  revision: 83425ba6c05d051d86c6f5775540727ce4238443
   branch: develop
   specs:
     goo (0.0.2)
@@ -15,7 +15,7 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/ncbo_annotator.git
-  revision: 3ae6bfb56dc59a670b5bc1a513ff4929f8cf3756
+  revision: 067104ae94c0e9d058cfbf419364fbf03f34de43
   branch: develop
   specs:
     ncbo_annotator (0.0.1)
@@ -26,7 +26,7 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/ncbo_cron.git
-  revision: 8c8ddb6c7f9f1d2584a5fb3220f1d2aa37b66cb2
+  revision: ed14911ccb28375298c63e7ca1b388ed0c638abb
   branch: develop
   specs:
     ncbo_cron (0.0.1)
@@ -42,7 +42,7 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/ncbo_ontology_recommender.git
-  revision: 818be7ee0d6b21f52a13dc1d6068505e6e17e0c5
+  revision: e6d4449d8b854f17bb54af6de142bc64bff22ab3
   branch: develop
   specs:
     ncbo_ontology_recommender (0.0.1)
@@ -53,7 +53,7 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/ontologies_linked_data.git
-  revision: 13bb138aa40ca72bfafc91a1a7416c162ba8d325
+  revision: 711ebf27722355a25a0e53a8204b4ae472902bd0
   branch: develop
   specs:
     ontologies_linked_data (0.0.1)
@@ -103,11 +103,13 @@ GEM
     activesupport (3.2.22.5)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
-    addressable (2.8.4)
+    addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
-    airbrussh (1.4.1)
+    airbrussh (1.4.2)
       sshkit (>= 1.6.1, != 1.7.0)
+    ast (2.4.2)
     backports (3.24.1)
+    base64 (0.1.1)
     bcrypt (3.1.19)
     bcrypt_pbkdf (1.1.0)
     bigdecimal (1.4.2)
@@ -141,7 +143,7 @@ GEM
     faraday-retry (2.2.0)
       faraday (~> 2.0)
     ffi (1.15.5)
-    gapic-common (0.19.1)
+    gapic-common (0.20.0)
       faraday (>= 1.9, < 3.a)
       faraday-retry (>= 1.0, < 3.a)
       google-protobuf (~> 3.14)
@@ -154,7 +156,7 @@ GEM
     google-analytics-data (0.4.0)
       google-analytics-data-v1beta (>= 0.7, < 2.a)
       google-cloud-core (~> 1.6)
-    google-analytics-data-v1beta (0.8.0)
+    google-analytics-data-v1beta (0.8.1)
       gapic-common (>= 0.19.1, < 2.a)
       google-cloud-errors (~> 1.0)
     google-cloud-core (1.6.0)
@@ -163,30 +165,28 @@ GEM
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
     google-cloud-errors (1.3.1)
-    google-protobuf (3.23.4-aarch64-linux)
-    google-protobuf (3.23.4-arm64-darwin)
-    google-protobuf (3.23.4-x86_64-darwin)
-    google-protobuf (3.23.4-x86_64-linux)
+    google-protobuf (3.24.3-aarch64-linux)
+    google-protobuf (3.24.3-x86_64-darwin)
+    google-protobuf (3.24.3-x86_64-linux)
     googleapis-common-protos (1.4.0)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (~> 1.2)
       grpc (~> 1.27)
-    googleapis-common-protos-types (1.7.0)
-      google-protobuf (~> 3.14)
-    googleauth (1.7.0)
+    googleapis-common-protos-types (1.8.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.8.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
-      memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.56.2)
+    grpc (1.58.0)
       google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.56.2-x86_64-darwin)
+    grpc (1.58.0-x86_64-darwin)
       google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.56.2-x86_64-linux)
+    grpc (1.58.0-x86_64-linux)
       google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
     haml (5.2.2)
@@ -203,6 +203,7 @@ GEM
     json_pure (2.6.3)
     jwt (2.7.1)
     kgio (2.11.4)
+    language_server-protocol (3.17.0.3)
     libxml-ruby (4.1.1)
     logger (1.5.3)
     macaddr (1.7.2)
@@ -212,19 +213,18 @@ GEM
       net-imap
       net-pop
       net-smtp
-    memoist (0.16.2)
     method_source (1.0.0)
-    mime-types (3.4.1)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.0218.1)
-    mini_mime (1.1.2)
+    mime-types-data (3.2023.0808)
+    mini_mime (1.1.5)
     minitest (4.7.5)
     minitest-stub_any_instance (1.0.3)
     mlanett-redis-lock (0.2.7)
       redis
     multi_json (1.15.0)
     net-http-persistent (2.9.4)
-    net-imap (0.3.6)
+    net-imap (0.3.7)
       date
       net-protocol
     net-pop (0.1.2)
@@ -235,20 +235,25 @@ GEM
       net-ssh (>= 2.6.5, < 8.0.0)
     net-smtp (0.3.3)
       net-protocol
-    net-ssh (7.1.0)
+    net-ssh (7.2.0)
     netrc (0.11.0)
-    newrelic_rpm (9.3.1)
-    oj (3.15.0)
+    newrelic_rpm (9.4.2)
+    oj (3.16.1)
     omni_logger (0.1.4)
       logger
     os (1.1.4)
+    parallel (1.23.0)
     parseconfig (1.1.2)
+    parser (3.2.2.3)
+      ast (~> 2.4.1)
+      racc
     pony (1.13.1)
       mail (>= 2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.3)
+    racc (1.7.1)
     rack (1.6.13)
     rack-accept (0.4.5)
       rack (>= 0.4)
@@ -258,13 +263,14 @@ GEM
       rack (>= 0.4)
     rack-cors (1.0.6)
       rack (>= 1.6.0)
-    rack-mini-profiler (3.1.0)
+    rack-mini-profiler (3.1.1)
       rack (>= 1.2.0)
     rack-protection (1.5.5)
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
     rack-timeout (0.6.3)
+    rainbow (3.1.1)
     raindrops (0.20.1)
     rake (10.5.0)
     rdf (1.0.8)
@@ -284,16 +290,31 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    rexml (3.2.5)
+    rexml (3.2.6)
     rsolr (2.5.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
+    rubocop (1.56.2)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.29.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
     ruby-xxHash (0.4.0.2)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
     rufus-scheduler (2.0.24)
       tzinfo (>= 0.3.22)
-    signet (0.17.0)
+    signet (0.18.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
@@ -332,6 +353,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.2)
+    unicode-display_width (2.4.2)
     unicorn (6.1.0)
       kgio (~> 2.6)
       raindrops (~> 0.7)
@@ -343,8 +365,6 @@ GEM
 
 PLATFORMS
   aarch64-linux
-  arm64-darwin-22
-  x86_64-darwin-18
   x86_64-darwin-21
   x86_64-linux
 
@@ -358,7 +378,6 @@ DEPENDENCIES
   capistrano-rbenv
   cube-ruby
   ed25519 (>= 1.2, < 2.0)
-  faraday (~> 2.0)
   ffi
   goo!
   haml (~> 5.2.2)
@@ -373,7 +392,6 @@ DEPENDENCIES
   oj (~> 3.0)
   ontologies_linked_data!
   parseconfig
-  pry
   rack
   rack-accept (~> 0.4)
   rack-attack (~> 6.6.1)
@@ -388,6 +406,7 @@ DEPENDENCIES
   redis
   redis-rack-cache (~> 2.0)
   redis-store (~> 1.10)
+  rubocop
   shotgun!
   simplecov
   simplecov-cobertura

config/deploy.rb

@@ -108,6 +108,7 @@
   after :publishing, :get_config
   after :get_config, :restart
   # after :deploy, :smoke_test
+  after :restart, "newrelic:notice_deployment"
 
   after :restart, :clear_cache do
     on roles(:web), in: :groups, limit: 3, wait: 10 do

config/environments/test.rb

@@ -5,7 +5,7 @@
 GOO_PATH_DATA    = ENV.include?("GOO_PATH_DATA")    ? ENV["GOO_PATH_DATA"]    : "/data/"
 GOO_PATH_QUERY   = ENV.include?("GOO_PATH_QUERY")   ? ENV["GOO_PATH_QUERY"]   : "/sparql/"
 GOO_PATH_UPDATE  = ENV.include?("GOO_PATH_UPDATE")  ? ENV["GOO_PATH_UPDATE"]  : "/update/"
-GOO_PORT         = ENV.include?("GOO_PORT")         ? ENV["GOO_PORT"]         : 9000
+GOO_PORT         = ENV.include?("GOO_PORT")         ? ENV["GOO_PORT"]         : 8080
 MGREP_DICTIONARY_FILE = ENV.include?("MGREP_DICTIONARY_FILE") ? ENV["MGREP_DICTIONARY_FILE"] : "./test/data/dictionary.txt"
 MGREP_HOST       = ENV.include?("MGREP_HOST")       ? ENV["MGREP_HOST"]       : "localhost"
 MGREP_PORT       = ENV.include?("MGREP_PORT")       ? ENV["MGREP_PORT"]       : 55555

config/rack_attack.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-puts "(API) >> Throttling enabled at #{LinkedData::OntologiesAPI.settings.req_per_second_per_ip} req/sec"
+limit_req_ip = LinkedData::OntologiesAPI.settings.req_per_second_per_ip
+limit_req_ip_heavy = limit_req_ip / 5
+puts "(API) >> Throttling enabled at #{limit_req_ip} req/sec"
 
 require 'rack/attack'
 use Rack::Attack
@@ -27,11 +29,12 @@
   request.env['REMOTE_USER']&.admin?
 end
 
-Rack::Attack.throttle('requests by ip',
-  limit: LinkedData::OntologiesAPI.settings.req_per_second_per_ip,
-  period: 1.second
-) do |request|
-  request.ip
+Rack::Attack.throttle('req/ip/heavy', limit: limit_req_ip_heavy, period: 1.second) do |req|
+  req.ip if req.path.include?('/recommender') || req.path.include?('/annotator')
+end
+
+Rack::Attack.throttle('req/ip', limit: limit_req_ip, period: 1.second) do |req|
+  req.ip
 end
 
 Rack::Attack.throttled_responder = lambda do |request|

controllers/metrics_controller.rb

@@ -85,7 +85,6 @@ class MetricsController < ApplicationController
   get "/ontologies/:ontology/metrics" do
     check_last_modified_collection(LinkedData::Models::Metric)
     ont, sub = get_ontology_and_submission
-    ont = Ontology.find(params["ontology"]).first
     error 404, "Ontology #{params['ontology']} not found" unless ont
     sub.bring(ontology: [:acronym], metrics: LinkedData::Models::Metric.goo_attrs_to_load(includes_param))
     reply sub.metrics || {}
@@ -106,10 +105,9 @@ class MetricsController < ApplicationController
     # reply {}
   end
 
-  get "/ontologies/:ontology/submissions/:submissionId/metrics" do
+  get "/ontologies/:ontology/submissions/:ontology_submission_id/metrics" do
     check_last_modified_collection(LinkedData::Models::Metric)
     ont, sub = get_ontology_and_submission
-    ont = Ontology.find(params["ontology"]).first
     error 404, "Ontology #{params['ontology']} not found" unless ont
     sub.bring(ontology: [:acronym], metrics: LinkedData::Models::Metric.goo_attrs_to_load(includes_param))
     reply sub.metrics || {}