From 141cbcedac92386711e0112849ad0c57ee271d3e Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 20 Mar 2024 09:05:46 +0100 Subject: [PATCH 01/46] update minitest, bigdecimal, and activesupport to newer versions --- Gemfile | 15 +++++++++------ Gemfile.lock | 48 +++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/Gemfile b/Gemfile index ec1f76f3..288250fa 100644 --- a/Gemfile +++ b/Gemfile @@ -1,11 +1,11 @@ source 'https://rubygems.org' - -gem 'activesupport', '~> 3.2' +gem 'activesupport', '~> 5' # see https://github.com/ncbo/ontologies_api/issues/69 -gem 'bigdecimal', '1.4.2' +gem 'bigdecimal' +gem 'faraday', '~> 1.9' gem 'json-schema', '~> 2.0' -gem 'multi_json', '~> 1.0' -gem 'oj', '~> 3.0' +gem 'multi_json' +gem 'oj' gem 'parseconfig' gem 'rack' gem 'rake', '~> 10.0' @@ -69,9 +69,12 @@ group :profiling do end group :test do - gem 'minitest', '~> 4.0' + gem 'minitest', '~> 5.0' gem 'minitest-stub_any_instance' gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io + gem 'crack', '0.4.5' + gem 'webmock' + gem "minitest-hooks", "~> 1.5" end diff --git a/Gemfile.lock b/Gemfile.lock index 37f2a36d..c17a0941 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -100,9 +100,11 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (3.2.22.5) - i18n (~> 0.6, >= 0.6.4) - multi_json (~> 1.0) + activesupport (5.2.8.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) airbrussh (1.5.1) @@ -112,7 +114,7 @@ GEM base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) - bigdecimal (1.4.2) + bigdecimal (3.1.6) builder (3.2.4) capistrano (3.18.1) airbrussh (>= 1.0.0) @@ -128,7 +130,8 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.3) - connection_pool (2.4.1) + crack (0.4.5) + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.4) @@ -168,7 +171,6 @@ GEM google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) - google-protobuf (3.25.3-x86_64-linux) googleapis-common-protos (1.5.0) google-protobuf (~> 3.18) googleapis-common-protos-types (~> 1.7) @@ -191,16 +193,14 @@ GEM grpc (1.62.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) - i18n (0.9.5) + httpclient (2.8.3) + i18n (1.14.1) concurrent-ruby (~> 1.0) json (2.7.1) json-schema (2.8.1) @@ -224,7 +224,9 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) mini_mime (1.1.5) - minitest (4.7.5) + minitest (5.22.2) + minitest-hooks (1.5.1) + minitest (> 5.3) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis @@ -247,7 +249,8 @@ GEM net-ssh (7.2.1) netrc (0.11.0) newrelic_rpm (9.7.1) - oj (3.16.1) + oj (3.16.3) + bigdecimal (>= 3.0) omni_logger (0.1.4) logger os (1.1.4) @@ -357,11 +360,13 @@ GEM net-ssh (>= 2.8.0) systemu (2.6.5) temple (0.10.3) + thread_safe (0.3.6) tilt (2.3.0) timeout (0.4.1) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) + trailblazer-option (0.1.2) + tzinfo (1.2.11) + thread_safe (~> 0.1) + uber (0.1.0) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) @@ -377,30 +382,31 @@ PLATFORMS arm64-darwin-23 x86_64-darwin-18 x86_64-darwin-23 - x86_64-linux DEPENDENCIES - activesupport (~> 3.2) + activesupport (~> 5) bcrypt_pbkdf (>= 1.0, < 2.0) - bigdecimal (= 1.4.2) + bigdecimal capistrano (~> 3) capistrano-bundler capistrano-locally capistrano-rbenv + crack (= 0.4.5) cube-ruby ed25519 (>= 1.2, < 2.0) ffi goo! haml (~> 5.2.2) json-schema (~> 2.0) - minitest (~> 4.0) + minitest (~> 5.0) + minitest-hooks (~> 1.5) minitest-stub_any_instance - multi_json (~> 1.0) + multi_json ncbo_annotator! ncbo_cron! ncbo_ontology_recommender! newrelic_rpm - oj (~> 3.0) + oj ontologies_linked_data! parseconfig rack From b3c5eb795875ff884c5c9b1ceb12affb5d67f5d0 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 20 Mar 2024 09:06:05 +0100 Subject: [PATCH 02/46] update the tests files to use the new minitest framework --- .env | 6 ++ Gemfile | 12 +-- Gemfile.lock | 85 ++++++++++++------- test/controllers/test_annotator_controller.rb | 4 +- test/controllers/test_batch_controller.rb | 2 +- test/controllers/test_classes_controller.rb | 2 +- test/controllers/test_instances_controller.rb | 2 +- test/controllers/test_mappings_controller.rb | 4 +- test/controllers/test_metrics_controller.rb | 2 +- test/controllers/test_notes_controller.rb | 6 +- .../controllers/test_ontologies_controller.rb | 10 +-- .../test_ontology_analytics_controller.rb | 8 +- .../test_ontology_submissions_controller.rb | 9 +- .../controllers/test_properties_controller.rb | 4 +- .../test_properties_search_controller.rb | 6 +- .../test_provisional_classes_controller.rb | 8 +- .../test_provisional_relations_controller.rb | 8 +- .../test_recommender_controller.rb | 2 +- .../test_recommender_v1_controller.rb | 2 +- test/controllers/test_replies_controller.rb | 4 +- test/controllers/test_search_controller.rb | 5 +- test/controllers/test_slices_controller.rb | 13 ++- test/controllers/test_users_controller.rb | 2 +- test/helpers/test_access_control_helper.rb | 11 +-- test/helpers/test_application_helper.rb | 2 +- test/helpers/test_http_cache_helper.rb | 8 +- test/helpers/test_slices_helper.rb | 11 ++- test/helpers/test_users_helper.rb | 9 +- test/middleware/test_rack_attack.rb | 6 +- test/test_case.rb | 36 +++++--- 30 files changed, 167 insertions(+), 122 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..2b53fceb --- /dev/null +++ b/.env @@ -0,0 +1,6 @@ +API_URL=http://localhost:9393 +ONTOLOGIES_LINKED_DATA_PATH= +GOO_PATH= +SPARQL_CLIENT_PATH= +ONTOPORTAL_KB=ontoportal_kb +REDIS_HOST=redis-ut diff --git a/Gemfile b/Gemfile index 288250fa..3048681d 100644 --- a/Gemfile +++ b/Gemfile @@ -41,12 +41,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index c17a0941..a64ad257 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 75436fe8e387febc53e34ee31ff0e6dd837a9d3f - branch: master + revision: 87e991b0e720c580a71e2c2c10f9ab44183db322 + branch: develop specs: goo (0.0.2) addressable (~> 2.8) @@ -15,8 +15,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + revision: fe0783c542479ed0a12da5663ef28e1673e80eba + branch: develop specs: ncbo_annotator (0.0.1) goo @@ -26,8 +26,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: b5764cc314758dbfdbb855b2b63c1c53b1325f6f - branch: master + revision: 2f8b1c1cff9a00da06ff68c1449f049e3f59c5ce + branch: develop specs: ncbo_cron (0.0.1) dante @@ -42,8 +42,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 - branch: master + revision: d047de0a9650df321db47300eba8ff8c195e0854 + branch: develop specs: ncbo_ontology_recommender (0.0.1) goo @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: b6df920d03af15ee35a3a7c31078c6ea263ba41d - branch: master + revision: a466d530877cb48cdfb811d0b61ecaabd2b835eb + branch: develop specs: ontologies_linked_data (0.0.1) activesupport @@ -73,8 +73,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master + revision: 55e7dbf858eb571c767bc67868f9af61663859cb + branch: develop specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -114,7 +114,7 @@ GEM base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) - bigdecimal (3.1.6) + bigdecimal (3.1.7) builder (3.2.4) capistrano (3.18.1) airbrussh (>= 1.0.0) @@ -130,6 +130,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.3) + connection_pool (2.4.1) crack (0.4.5) rexml cube-ruby (0.0.3) @@ -138,13 +139,29 @@ GEM docile (1.4.0) domain_name (0.6.20240107) ed25519 (1.3.0) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) + faraday (1.10.3) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.2) - faraday-retry (2.2.0) - faraday (~> 2.0) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.4) + multipart-post (~> 2) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) ffi (1.16.3) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) @@ -168,14 +185,14 @@ GEM google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) + google-protobuf (3.25.3-x86_64-linux) googleapis-common-protos (1.5.0) google-protobuf (~> 3.18) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.13.0) + googleapis-common-protos-types (1.14.0) google-protobuf (~> 3.18) googleauth (1.11.0) faraday (>= 1.0, < 3.a) @@ -184,23 +201,23 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0-aarch64-linux) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) grpc (1.62.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) grpc (1.62.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) + grpc (1.62.0-x86_64-linux) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.1.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) - httpclient (2.8.3) - i18n (1.14.1) + i18n (1.14.4) concurrent-ruby (~> 1.0) json (2.7.1) json-schema (2.8.1) @@ -224,13 +241,14 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) mini_mime (1.1.5) - minitest (5.22.2) + minitest (5.22.3) minitest-hooks (1.5.1) minitest (> 5.3) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) + multipart-post (2.4.0) mutex_m (0.2.0) net-http-persistent (2.9.4) net-imap (0.4.10) @@ -363,10 +381,9 @@ GEM thread_safe (0.3.6) tilt (2.3.0) timeout (0.4.1) - trailblazer-option (0.1.2) tzinfo (1.2.11) thread_safe (~> 0.1) - uber (0.1.0) + unicode-display_width (2.5.0) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) @@ -375,13 +392,15 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.23.0) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - aarch64-linux arm64-darwin-22 - arm64-darwin-23 x86_64-darwin-18 - x86_64-darwin-23 + x86_64-linux DEPENDENCIES activesupport (~> 5) @@ -394,6 +413,7 @@ DEPENDENCIES crack (= 0.4.5) cube-ruby ed25519 (>= 1.2, < 2.0) + faraday (~> 1.9) ffi goo! haml (~> 5.2.2) @@ -433,6 +453,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock BUNDLED WITH 2.4.22 diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 3b21b9e7..751176f6 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -2,7 +2,7 @@ class TestAnnotatorController < TestCase - def self.before_suite + def before_suite @@redis = Redis.new(:host => Annotator.settings.annotator_redis_host, :port => Annotator.settings.annotator_redis_port) db_size = @@redis.dbsize if db_size > MAX_TEST_REDIS_SIZE @@ -20,7 +20,7 @@ def self.before_suite annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) - mapping_test_set + self.class.mapping_test_set end def test_annotate diff --git a/test/controllers/test_batch_controller.rb b/test/controllers/test_batch_controller.rb index 55d9cee9..a4da9766 100644 --- a/test/controllers/test_batch_controller.rb +++ b/test/controllers/test_batch_controller.rb @@ -1,7 +1,7 @@ require_relative '../test_case' class TestBatchController < TestCase - def self.before_suite + def before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies end diff --git a/test/controllers/test_classes_controller.rb b/test/controllers/test_classes_controller.rb index a918ece0..b88ec369 100644 --- a/test/controllers/test_classes_controller.rb +++ b/test/controllers/test_classes_controller.rb @@ -2,7 +2,7 @@ class TestClassesController < TestCase - def self.before_suite + def before_suite options = {ont_count: 1, submission_count: 3, submissions_to_process: [1, 2], diff --git a/test/controllers/test_instances_controller.rb b/test/controllers/test_instances_controller.rb index 4e9b8a5c..1735455f 100644 --- a/test/controllers/test_instances_controller.rb +++ b/test/controllers/test_instances_controller.rb @@ -2,7 +2,7 @@ class TestInstancesController < TestCase - def self.before_suite + def before_suite LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, acronym: 'XCT-TEST-INST', diff --git a/test/controllers/test_mappings_controller.rb b/test/controllers/test_mappings_controller.rb index 44bfa032..d8c6f619 100644 --- a/test/controllers/test_mappings_controller.rb +++ b/test/controllers/test_mappings_controller.rb @@ -2,8 +2,8 @@ class TestMappingsController < TestCase - def self.before_suite - + def before_suite + self.backend_4s_delete ["BRO-TEST-MAP-0","CNO-TEST-MAP-0","FAKE-TEST-MAP-0"].each do |acr| LinkedData::Models::OntologySubmission.where(ontology: [acronym: acr]).to_a.each do |s| s.delete diff --git a/test/controllers/test_metrics_controller.rb b/test/controllers/test_metrics_controller.rb index 1b8890a6..94e55af8 100644 --- a/test/controllers/test_metrics_controller.rb +++ b/test/controllers/test_metrics_controller.rb @@ -2,7 +2,7 @@ class TestMetricsController < TestCase - def self.before_suite + def before_suite if OntologySubmission.all.count > 100 puts "this test is going to wipe out all submission and ontologies. probably this is not a test env." return diff --git a/test/controllers/test_notes_controller.rb b/test/controllers/test_notes_controller.rb index d9223669..0c7d2bcb 100644 --- a/test/controllers/test_notes_controller.rb +++ b/test/controllers/test_notes_controller.rb @@ -3,9 +3,9 @@ class TestNotesController < TestCase - def self.before_suite - self.new("before_suite").delete_ontologies_and_submissions - @@ontology, @@cls = self.new("before_suite")._ontology_and_class + def before_suite + self.delete_ontologies_and_submissions + @@ontology, @@cls = self._ontology_and_class @@note_user = "test_note_user" @@user = LinkedData::Models::User.new( diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4f61256b..1f99912d 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -2,11 +2,11 @@ require_relative '../test_case' class TestOntologiesController < TestCase - def self.before_suite - _set_vars - _delete - _create_user - _create_onts + def before_suite + self.class._set_vars + self.class._delete + self.class._create_user + self.class._create_onts end def teardown diff --git a/test/controllers/test_ontology_analytics_controller.rb b/test/controllers/test_ontology_analytics_controller.rb index b8e36dce..721efea6 100644 --- a/test/controllers/test_ontology_analytics_controller.rb +++ b/test/controllers/test_ontology_analytics_controller.rb @@ -196,7 +196,7 @@ class TestOntologyAnalyticsController < TestCase } } - def self.before_suite + def before_suite @@redis = Redis.new(:host => Annotator.settings.annotator_redis_host, :port => Annotator.settings.annotator_redis_port) db_size = @@redis.dbsize if db_size > MAX_TEST_REDIS_SIZE @@ -212,9 +212,9 @@ def self.before_suite "SNOMEDCT" => "SNOMEDCT Ontology", "TST" => "TST Ontology" } - _delete - _create_user - _create_onts + self.class._delete + self.class._create_user + self.class._create_onts end def teardown diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 40532cd0..4660e10b 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -2,10 +2,11 @@ class TestOntologySubmissionsController < TestCase - def self.before_suite - _set_vars - _create_user - _create_onts + def before_suite + self.backend_4s_delete + self.class._set_vars + self.class._create_user + self.class._create_onts end def self._set_vars diff --git a/test/controllers/test_properties_controller.rb b/test/controllers/test_properties_controller.rb index 8248403c..1bd30d5b 100644 --- a/test/controllers/test_properties_controller.rb +++ b/test/controllers/test_properties_controller.rb @@ -2,7 +2,7 @@ class TestPropertiesController < TestCase - def self.before_suite + def before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, acronym: "BROSEARCHTEST", @@ -25,7 +25,7 @@ def self.before_suite @@acronyms = @@ontologies.map { |ont| ont.bring_remaining; ont.acronym } end - def self.after_suite + def after_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions end diff --git a/test/controllers/test_properties_search_controller.rb b/test/controllers/test_properties_search_controller.rb index f93a90a1..4302eb97 100644 --- a/test/controllers/test_properties_search_controller.rb +++ b/test/controllers/test_properties_search_controller.rb @@ -2,7 +2,9 @@ class TestPropertiesSearchController < TestCase - def self.before_suite + def before_suite + self.backend_4s_delete + count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, acronym: "BROSEARCHTEST", @@ -24,7 +26,7 @@ def self.before_suite @@ontologies = bro.concat(mccl) end - def self.after_suite + def after_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions LinkedData::Models::Ontology.indexClear(:property) LinkedData::Models::Ontology.indexCommit(nil, :property) diff --git a/test/controllers/test_provisional_classes_controller.rb b/test/controllers/test_provisional_classes_controller.rb index 7225d772..e54c7805 100644 --- a/test/controllers/test_provisional_classes_controller.rb +++ b/test/controllers/test_provisional_classes_controller.rb @@ -1,9 +1,9 @@ require_relative '../test_case' class TestProvisionalClassesController < TestCase - def self.before_suite - self.new("before_suite").delete_ontologies_and_submissions - @@ontology, classes = self.new("before_suite")._ontology_and_classes + def before_suite + self.delete_ontologies_and_submissions + @@ontology, classes = self._ontology_and_classes @@cls = classes[0] @@cls1 = classes[1] @@ -32,7 +32,7 @@ def self.before_suite end end - def self.after_suite + def after_suite 3.times do |i| @@pcs[i].delete end diff --git a/test/controllers/test_provisional_relations_controller.rb b/test/controllers/test_provisional_relations_controller.rb index 83f2761d..f097dde9 100644 --- a/test/controllers/test_provisional_relations_controller.rb +++ b/test/controllers/test_provisional_relations_controller.rb @@ -1,9 +1,9 @@ require_relative '../test_case' class TestProvisionalRelationsController < TestCase - def self.before_suite - self.new("before_suite").delete_ontologies_and_submissions - @@ontology, classes = self.new("before_suite")._ontology_and_classes + def before_suite + self.delete_ontologies_and_submissions + @@ontology, classes = self._ontology_and_classes @@cls1 = classes[0] @@cls2 = classes[1] @@ -39,7 +39,7 @@ def self.before_suite @@test_rel.save end - def self.after_suite + def after_suite @@test_pc.delete @@test_user.delete end diff --git a/test/controllers/test_recommender_controller.rb b/test/controllers/test_recommender_controller.rb index 29caf28c..96b6e106 100644 --- a/test/controllers/test_recommender_controller.rb +++ b/test/controllers/test_recommender_controller.rb @@ -2,7 +2,7 @@ class TestRecommenderController < TestCase - def self.before_suite + def before_suite @@redis = Redis.new(:host => Annotator.settings.annotator_redis_host, :port => Annotator.settings.annotator_redis_port) db_size = @@redis.dbsize if db_size > MAX_TEST_REDIS_SIZE diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 7b14a63d..074cda8f 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -2,7 +2,7 @@ class TestRecommenderController < TestCase - def self.before_suite + def before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies @@text = < Date: Wed, 24 Apr 2024 14:19:04 -0700 Subject: [PATCH 03/46] Gemfile.lock update after merging pull request #150 --- Gemfile.lock | 53 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 523439b4..1fa26ad1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 87e991b0e720c580a71e2c2c10f9ab44183db322 + revision: 563a47fa4329ffb8a774731be210f44a054726c0 branch: develop specs: goo (0.0.2) @@ -8,6 +8,7 @@ GIT pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client @@ -15,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: fe0783c542479ed0a12da5663ef28e1673e80eba + revision: a427ff3719f88f887e04ac695a002ef24ce3afb8 branch: develop specs: ncbo_annotator (0.0.1) @@ -26,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 2f8b1c1cff9a00da06ff68c1449f049e3f59c5ce + revision: 2af844a10a31ae808e85821bc00e8d30e0504d20 branch: develop specs: ncbo_cron (0.0.1) @@ -42,7 +43,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: d047de0a9650df321db47300eba8ff8c195e0854 + revision: 20b22ff8f9ba676d8c0bb4876de1d22e040427ab branch: develop specs: ncbo_ontology_recommender (0.0.1) @@ -53,7 +54,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: a466d530877cb48cdfb811d0b61ecaabd2b835eb + revision: ab7b2f0eb6c4256ff1a01ace55b5a966c040e1ca branch: develop specs: ontologies_linked_data (0.0.1) @@ -73,7 +74,7 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 55e7dbf858eb571c767bc67868f9af61663859cb + revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc branch: develop specs: sparql-client (1.0.1) @@ -105,7 +106,7 @@ GEM multi_json (~> 1.0) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.5.1) + airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) backports (3.25.0) @@ -140,7 +141,7 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -165,6 +166,7 @@ GEM google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) + google-protobuf (3.25.3) google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) @@ -173,7 +175,7 @@ GEM google-protobuf (~> 3.18) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.13.0) + googleapis-common-protos-types (1.14.0) google-protobuf (~> 3.18) googleauth (1.11.0) faraday (>= 1.0, < 3.a) @@ -182,6 +184,9 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) + grpc (1.62.0) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) grpc (1.62.0-aarch64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) @@ -202,10 +207,10 @@ GEM domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) + json (2.7.2) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.7.1) + json_pure (2.7.2) jwt (2.8.1) base64 kgio (2.11.4) @@ -219,7 +224,7 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) @@ -242,11 +247,11 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol - net-ssh (7.2.1) + net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.7.1) + newrelic_rpm (9.9.0) oj (3.16.1) omni_logger (0.1.4) logger @@ -261,7 +266,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) racc (1.7.3) rack (1.6.13) rack-accept (0.4.5) @@ -285,9 +290,9 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.0) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -295,16 +300,18 @@ GEM redis-store (1.10.0) redis (>= 4, < 6) regexp_parser (2.9.0) + request_store (1.6.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) rexml (3.2.6) - rsolr (2.5.0) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.62.1) + rubocop (1.63.3) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -350,7 +357,8 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.0) + sshkit (1.22.1) + base64 mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) @@ -375,6 +383,7 @@ PLATFORMS aarch64-linux arm64-darwin-22 arm64-darwin-23 + ruby x86_64-darwin-18 x86_64-darwin-21 x86_64-darwin-23 From bcc5d7816f3a70d5ff1a5008b3a01c758a13d5a1 Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 24 Apr 2024 14:32:04 -0700 Subject: [PATCH 04/46] Gemfile.lock update --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1fa26ad1..27f82c6a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 563a47fa4329ffb8a774731be210f44a054726c0 + revision: 5d911f587f22059581ae3e713e5fe9bde9de82e7 branch: develop specs: goo (0.0.2) @@ -16,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: a427ff3719f88f887e04ac695a002ef24ce3afb8 + revision: 36125bdaafd36f394263b9495af96d21a81b1217 branch: develop specs: ncbo_annotator (0.0.1) @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 2af844a10a31ae808e85821bc00e8d30e0504d20 + revision: ec8f5367b090e851ff199e869432efd89f3d14bc branch: develop specs: ncbo_cron (0.0.1) @@ -43,7 +43,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 20b22ff8f9ba676d8c0bb4876de1d22e040427ab + revision: 522af616b2285c451364028eef8c1045857eddbb branch: develop specs: ncbo_ontology_recommender (0.0.1) @@ -54,7 +54,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: ab7b2f0eb6c4256ff1a01ace55b5a966c040e1ca + revision: cf8100e47090be1bfb79e7ea642eb6a528013efa branch: develop specs: ontologies_linked_data (0.0.1) From bcbe45de10a38e6c84a78ba33a9c48f2adebcbd1 Mon Sep 17 00:00:00 2001 From: mdorf Date: Tue, 30 Apr 2024 15:54:12 -0700 Subject: [PATCH 05/46] Gemfile.lock update --- Gemfile.lock | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 27f82c6a..37dfbfcf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 5d911f587f22059581ae3e713e5fe9bde9de82e7 + revision: 33583fd1c1d72b449cd6bb91815e3422b8448983 branch: develop specs: goo (0.0.2) @@ -16,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 36125bdaafd36f394263b9495af96d21a81b1217 + revision: f9a1be204e81d72f408927b8f60c0be208611929 branch: develop specs: ncbo_annotator (0.0.1) @@ -27,11 +27,13 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: ec8f5367b090e851ff199e869432efd89f3d14bc + revision: 26638bfdbec1e6b01f5eb6a8b655bfd70adac8ac branch: develop specs: ncbo_cron (0.0.1) dante + faraday (~> 2) + faraday-follow_redirects (~> 0) goo google-analytics-data mlanett-redis-lock @@ -43,7 +45,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 522af616b2285c451364028eef8c1045857eddbb + revision: 4770596c52861d743314218532461876a694fa08 branch: develop specs: ncbo_ontology_recommender (0.0.1) @@ -54,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: cf8100e47090be1bfb79e7ea642eb6a528013efa + revision: da1642ed8f9f04db2e427e35d88f235a55138d03 branch: develop specs: ontologies_linked_data (0.0.1) @@ -140,6 +142,8 @@ GEM base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) @@ -258,7 +262,7 @@ GEM os (1.1.4) parallel (1.24.0) parseconfig (1.1.2) - parser (3.3.0.5) + parser (3.3.1.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -311,7 +315,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.63.3) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -322,8 +326,8 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -357,7 +361,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.1) + sshkit (1.22.2) base64 mutex_m net-scp (>= 1.1.2) From 854bf26a2c25ba4f2b36ab97ad9e1a1beb283084 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 3 May 2024 09:58:10 -0700 Subject: [PATCH 06/46] Gemfile update --- Gemfile.lock | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index fc1b44ac..f8f718ad 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: f01386fe55ffba39cbf5e0bf95c8e2dc18767711 + revision: ef2d816df2d263c905bd034efd449a964fa4890f branch: master specs: goo (0.0.2) @@ -8,6 +8,7 @@ GIT pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client @@ -26,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: b5764cc314758dbfdbb855b2b63c1c53b1325f6f + revision: f239b34230131cd7cfb9eec5b34068ba990b698b branch: master specs: ncbo_cron (0.0.1) @@ -53,7 +54,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4dc3b1123e871b53ba9eb51983ff69c5d2c75c18 + revision: 8157c657f665a7148064b2daf3e36b38b1aaa504 branch: master specs: ontologies_linked_data (0.0.1) @@ -73,7 +74,7 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc branch: master specs: sparql-client (1.0.1) @@ -105,7 +106,7 @@ GEM multi_json (~> 1.0) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.5.1) + airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) backports (3.25.0) @@ -140,7 +141,7 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -182,16 +183,16 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0-aarch64-linux) + grpc (1.63.0-aarch64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-arm64-darwin) + grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-darwin) + grpc (1.63.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -202,10 +203,10 @@ GEM domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) + json (2.7.2) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.7.1) + json_pure (2.7.2) jwt (2.8.1) base64 kgio (2.11.4) @@ -219,7 +220,7 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) @@ -244,16 +245,16 @@ GEM net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.5.0) net-protocol - net-ssh (7.2.1) + net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.8.0) + newrelic_rpm (9.9.0) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) parallel (1.24.0) parseconfig (1.1.2) - parser (3.3.0.5) + parser (3.3.1.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -261,7 +262,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) racc (1.7.3) rack (1.6.13) rack-accept (0.4.5) @@ -285,9 +286,9 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.1) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -295,6 +296,8 @@ GEM redis-store (1.10.0) redis (>= 4, < 6) regexp_parser (2.9.0) + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -304,7 +307,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.62.1) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -315,8 +318,8 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -350,7 +353,8 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.0) + sshkit (1.22.2) + base64 mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) From fe2c14bcceb5680aa99e02ec1023ba9099c83b3b Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 3 May 2024 12:59:54 -0700 Subject: [PATCH 07/46] confirm to minitest style guide changes allows to see more detailed output when unit test fail as in #146 --- test/controllers/test_search_controller.rb | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 70bf246e..f120ee10 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -256,9 +256,9 @@ def test_search_obo_id assert_equal ncit_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) assert_equal 'realization', docs[1]["prefLabel"] assert_equal 'realization', docs[2]["prefLabel"] - assert docs[3]["prefLabel"].upcase.include?('OGMS ') - assert docs[4]["prefLabel"].upcase.include?('OGMS ') - assert docs[5]["prefLabel"].upcase.include?('OGMS ') + assert_includes docs[3]["prefLabel"].upcase, 'OGMS ' + assert_includes docs[4]["prefLabel"].upcase, 'OGMS ' + assert_includes docs[5]["prefLabel"].upcase, 'OGMS ' get "/search?q=CNO:0000002" assert last_response.ok? @@ -267,13 +267,13 @@ def test_search_obo_id assert_equal 7, docs.size assert_equal cno_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[0]["links"]["ontology"]) acr_1 = LinkedData::Utils::Triples.last_iri_fragment(docs[1]["links"]["ontology"]) - assert acr_1 === ncit_acronym || acr_1 === ogms_acronym + assert_includes [ncit_acronym, ogms_acronym], acr_1 acr_2= LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) - assert acr_2 === ncit_acronym || acr_2 === ogms_acronym - assert docs[3]["prefLabel"].upcase.include?('CNO ') - assert docs[4]["prefLabel"].upcase.include?('CNO ') - assert docs[5]["prefLabel"].upcase.include?('CNO ') - assert docs[6]["prefLabel"].upcase.include?('CNO ') + assert_includes [ncit_acronym, ogms_acronym], acr_2 + assert_includes docs[3]["prefLabel"].upcase, 'CNO' + assert_includes docs[4]["prefLabel"].upcase, 'CNO' + assert_includes docs[5]["prefLabel"].upcase, 'CNO' + assert_includes docs[6]["prefLabel"].upcase, 'CNO' # mdorf, 3/2/2024, when the : is followed by a LETTER, as in NCIT:C20480, # then Solr does not split the query on the tokens, @@ -304,17 +304,17 @@ def test_search_obo_id ont = LinkedData::Models::Ontology.find(ncit_acronym).first ont.delete if ont ont = LinkedData::Models::Ontology.find(ncit_acronym).first - assert ont.nil? + assert_nil ont ont = LinkedData::Models::Ontology.find(ogms_acronym).first ont.delete if ont ont = LinkedData::Models::Ontology.find(ogms_acronym).first - assert ont.nil? + assert_nil ont ont = LinkedData::Models::Ontology.find(cno_acronym).first ont.delete if ont ont = LinkedData::Models::Ontology.find(cno_acronym).first - assert ont.nil? + assert_nil ont end end From f439e21d4862890c3cf447e45c3438247dd9c908 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 3 May 2024 20:09:07 -0700 Subject: [PATCH 08/46] Gemfile update --- Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 37dfbfcf..642ef823 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: f9a1be204e81d72f408927b8f60c0be208611929 + revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e branch: develop specs: ncbo_annotator (0.0.1) @@ -45,7 +45,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 4770596c52861d743314218532461876a694fa08 + revision: f92a42f660635522eb8709e618ff2e641aef0d17 branch: develop specs: ncbo_ontology_recommender (0.0.1) @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: da1642ed8f9f04db2e427e35d88f235a55138d03 + revision: a3f6cf0493bb595011f98acf95efda33111c537b branch: develop specs: ontologies_linked_data (0.0.1) @@ -188,19 +188,19 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0) + grpc (1.63.0) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-aarch64-linux) + grpc (1.63.0-aarch64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-arm64-darwin) + grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-darwin) + grpc (1.63.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -304,7 +304,7 @@ GEM redis-store (1.10.0) redis (>= 4, < 6) regexp_parser (2.9.0) - request_store (1.6.0) + request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) From f39d91d33e6202c3e156cef1d64b49a9aee56297 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 3 May 2024 20:44:16 -0700 Subject: [PATCH 09/46] Gemfile update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f8f718ad..3b330015 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -54,7 +54,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 8157c657f665a7148064b2daf3e36b38b1aaa504 + revision: 02986ff6db698cd56fdc062fed21b65d706a9598 branch: master specs: ontologies_linked_data (0.0.1) From 75b05fe7579f7358860f9395a45302e70923995c Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:20:00 +0200 Subject: [PATCH 10/46] add again SOLR config file as changed for the multilingual support --- config/solr/property_search/enumsconfig.xml | 12 + .../mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/property_search/schema.xml | 1179 +++++++++++++++ config/solr/property_search/solrconfig.xml | 1299 +++++++++++++++++ config/solr/solr.xml | 60 + config/solr/term_search/enumsconfig.xml | 12 + .../term_search/mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/term_search/schema.xml | 1224 ++++++++++++++++ config/solr/term_search/solrconfig.xml | 1299 +++++++++++++++++ docker-compose.yml | 14 +- .../configsets/term_search/conf/schema.xml | 1015 ++++++------- test/solr/generate_ncbo_configsets.sh | 35 +- 12 files changed, 6131 insertions(+), 510 deletions(-) create mode 100644 config/solr/property_search/enumsconfig.xml create mode 100644 config/solr/property_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/property_search/schema.xml create mode 100644 config/solr/property_search/solrconfig.xml create mode 100644 config/solr/solr.xml create mode 100644 config/solr/term_search/enumsconfig.xml create mode 100644 config/solr/term_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/term_search/schema.xml create mode 100644 config/solr/term_search/solrconfig.xml diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/property_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/property_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml new file mode 100644 index 00000000..20824ea6 --- /dev/null +++ b/config/solr/property_search/schema.xml @@ -0,0 +1,1179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/property_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/config/solr/solr.xml b/config/solr/solr.xml new file mode 100644 index 00000000..d9d089e4 --- /dev/null +++ b/config/solr/solr.xml @@ -0,0 +1,60 @@ + + + + + + + + ${solr.max.booleanClauses:500000} + ${solr.sharedLib:} + ${solr.allowPaths:} + + + + ${host:} + ${solr.port.advertise:0} + ${hostContext:solr} + + ${genericCoreNodeNames:true} + + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + + + ${socketTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} + + + + + diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/term_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/term_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml new file mode 100644 index 00000000..73c75b31 --- /dev/null +++ b/config/solr/term_search/schema.xml @@ -0,0 +1,1224 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/term_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 182a18c9..55de76eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,10 +95,18 @@ services: - 4store solr-ut: - image: ontoportal/solr-ut:0.1.0 + image: solr:8 + volumes: + - ./test/solr/configsets:/configsets:ro + ports: + - "8983:8983" + command: > + bash -c "precreate-core term_search_core1 /configsets/term_search + && precreate-core prop_search_core1 /configsets/property_search + && solr-foreground" healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 3s + test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] + start_period: 5s interval: 10s timeout: 5s retries: 5 diff --git a/test/solr/configsets/term_search/conf/schema.xml b/test/solr/configsets/term_search/conf/schema.xml index 6b18a2a1..73c75b31 100644 --- a/test/solr/configsets/term_search/conf/schema.xml +++ b/test/solr/configsets/term_search/conf/schema.xml @@ -128,11 +128,20 @@ - - - - - + + + + + + + + + + + + + + @@ -140,9 +149,20 @@ - + + + + + + + + + + + + + - @@ -151,8 +171,8 @@ - - + + @@ -251,8 +271,19 @@ - + + + + + + + + + + + + - - - + + + - - - - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -769,255 +800,255 @@ - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - + \ No newline at end of file diff --git a/test/solr/generate_ncbo_configsets.sh b/test/solr/generate_ncbo_configsets.sh index 893f7f3a..29134dad 100755 --- a/test/solr/generate_ncbo_configsets.sh +++ b/test/solr/generate_ncbo_configsets.sh @@ -2,18 +2,23 @@ # generates solr configsets by merging _default configset with config files in config/solr # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/ -pushd solr/configsets -ld_config='../../../../ontologies_linked_data/config/solr/' -#ld_config='../../../../config/solr/' -ls -l $ld_config -pwd -[ -d property_search ] && rm -Rf property_search -[ -d term_search ] && rm -Rf property_search -[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" -mkdir -p property_search/conf -mkdir -p term_search/conf -cp -a _default/conf/* property_search/conf/ -cp -a _default/conf/* term_search/conf/ -cp -a $ld_config/property_search/* property_search/conf -cp -a $ld_config/term_search/* term_search/conf -popd +#cd solr/configsets +ld_config='config/solr' +configsets='test/solr/configsets' +[ -d ${configsets}/property_search ] && rm -Rf ${configsets}/property_search +[ -d ${configsets}/term_search ] && rm -Rf ${configsets}/term_search +if [[ ! -d ${ld_config}/property_search ]]; then + echo 'cant find ld solr config sets' + exit 1 +fi +if [[ ! -d ${configsets}/_default/conf ]]; then + echo 'cant find default solr configset' + exit 1 +fi +mkdir -p ${configsets}/property_search/conf +mkdir -p ${configsets}/term_search/conf +cp -a ${configsets}/_default/conf/* ${configsets}/property_search/conf/ +cp -a ${configsets}/_default/conf/* ${configsets}/term_search/conf/ +cp -a $ld_config/property_search/* ${configsets}/property_search/conf +cp -a $ld_config/term_search/* ${configsets}/term_search/conf + From 9435c7efb50b69ebfb5496a5a50d5b40c28a670d Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:21:49 +0200 Subject: [PATCH 11/46] add request lang middleware to save as a global variable the request parameter "lang" or "language" --- Gemfile | 14 ++- Gemfile.lock | 145 +++++++++++++++---------- app.rb | 6 + lib/rack/request_lang.rb | 16 +++ views/documentation/documentation.haml | 1 + 5 files changed, 119 insertions(+), 63 deletions(-) create mode 100644 lib/rack/request_lang.rb diff --git a/Gemfile b/Gemfile index ec1f76f3..0e113266 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' +gem 'request_store' # Rack middleware gem 'ffi' @@ -40,13 +41,13 @@ gem 'unicorn-worker-killer' gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' -# NCBO -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' +# NCBO gems (can be from a local dev path or from rubygems/git) +gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' +gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' +gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'pr/sync-agroportal-bioportal' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 @@ -74,4 +75,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io + gem 'webmock', '~> 3.19.1' end diff --git a/Gemfile.lock b/Gemfile.lock index fc1b44ac..79d8b762 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,22 +1,44 @@ GIT - remote: https://github.com/ncbo/goo.git - revision: f01386fe55ffba39cbf5e0bf95c8e2dc18767711 + remote: https://github.com/ncbo/ncbo_ontology_recommender.git + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master + specs: + ncbo_ontology_recommender (0.0.1) + goo + ncbo_annotator + ontologies_linked_data + redis + +GIT + remote: https://github.com/ncbo/sparql-client.git + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc branch: master + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + +GIT + remote: https://github.com/ontoportal-lirmm/goo.git + revision: f2751fe9324e48a0a5c2a8b15580ab879fc53a2b + branch: pr/sync-agroportal-bioportal specs: goo (0.0.2) addressable (~> 2.8) pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client uuid GIT - remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git + revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + branch: development specs: ncbo_annotator (0.0.1) goo @@ -25,14 +47,15 @@ GIT ruby-xxHash GIT - remote: https://github.com/ncbo/ncbo_cron.git - revision: b5764cc314758dbfdbb855b2b63c1c53b1325f6f + remote: https://github.com/ontoportal-lirmm/ncbo_cron.git + revision: 9ec0147203007cc368a5119ffe1a019fa8701c14 branch: master specs: ncbo_cron (0.0.1) dante goo google-analytics-data + google-apis-analytics_v3 mlanett-redis-lock multi_json ncbo_annotator @@ -41,20 +64,9 @@ GIT rufus-scheduler (~> 2.0.24) GIT - remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 - branch: master - specs: - ncbo_ontology_recommender (0.0.1) - goo - ncbo_annotator - ontologies_linked_data - redis - -GIT - remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4dc3b1123e871b53ba9eb51983ff69c5d2c75c18 - branch: master + remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git + revision: f70abe981c6fb3533b7785d58b7234855bcd5cf2 + branch: pr/sync-agroportal-ncbo specs: ontologies_linked_data (0.0.1) activesupport @@ -71,16 +83,6 @@ GIT rsolr rubyzip -GIT - remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 @@ -105,7 +107,7 @@ GEM multi_json (~> 1.0) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.5.1) + airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) backports (3.25.0) @@ -129,9 +131,13 @@ GEM coderay (1.1.3) concurrent-ruby (1.2.3) connection_pool (2.4.1) + crack (1.0.0) + bigdecimal + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.4) + declarative (0.0.20) docile (1.4.0) domain_name (0.6.20240107) ed25519 (1.3.0) @@ -140,7 +146,7 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -159,13 +165,22 @@ GEM google-analytics-data-v1beta (0.12.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) + google-apis-analytics_v3 (0.15.0) + google-apis-core (>= 0.14.0, < 2.a) + google-apis-core (0.14.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 1.9) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + rexml google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) @@ -182,30 +197,31 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0-aarch64-linux) + grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-arm64-darwin) + grpc (1.63.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-darwin) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.1.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) + httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) + json (2.7.2) + json-ld (1.1.1) + rdf (>= 1.0.8) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.7.1) + json_pure (2.7.2) jwt (2.8.1) base64 kgio (2.11.4) @@ -219,7 +235,7 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) @@ -244,16 +260,16 @@ GEM net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.5.0) net-protocol - net-ssh (7.2.1) + net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.8.0) + newrelic_rpm (9.9.0) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) parallel (1.24.0) parseconfig (1.1.2) - parser (3.3.0.5) + parser (3.3.1.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -261,7 +277,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) racc (1.7.3) rack (1.6.13) rack-accept (0.4.5) @@ -285,9 +301,9 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.1) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -295,16 +311,23 @@ GEM redis-store (1.10.0) redis (>= 4, < 6) regexp_parser (2.9.0) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + retriable (3.1.2) rexml (3.2.6) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.62.1) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -315,8 +338,8 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -350,7 +373,8 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.0) + sshkit (1.22.2) + base64 mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) @@ -359,8 +383,10 @@ GEM temple (0.10.3) tilt (2.3.0) timeout (0.4.1) + trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + uber (0.1.0) unicode-display_width (2.5.0) unicorn (6.1.0) kgio (~> 2.6) @@ -370,13 +396,14 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - aarch64-linux arm64-darwin-22 - arm64-darwin-23 x86_64-darwin-18 - x86_64-darwin-21 x86_64-darwin-23 x86_64-linux @@ -393,6 +420,7 @@ DEPENDENCIES ffi goo! haml (~> 5.2.2) + json-ld json-schema (~> 2.0) minitest (~> 4.0) minitest-stub_any_instance @@ -403,6 +431,7 @@ DEPENDENCIES newrelic_rpm oj (~> 3.0) ontologies_linked_data! + parallel parseconfig rack rack-accept (~> 0.4) @@ -418,6 +447,7 @@ DEPENDENCIES redis redis-rack-cache (~> 2.0) redis-store (~> 1.10) + request_store rubocop shotgun! simplecov @@ -428,6 +458,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock (~> 3.19.1) BUNDLED WITH 2.4.22 diff --git a/app.rb b/app.rb index 41ad56ac..e09178bd 100644 --- a/app.rb +++ b/app.rb @@ -27,6 +27,7 @@ require_relative 'lib/rack/cube_reporter' require_relative 'lib/rack/param_translator' require_relative 'lib/rack/slice_detection' +require_relative 'lib/rack/request_lang' # Logging setup require_relative "config/logging" @@ -34,6 +35,8 @@ # Inflector setup require_relative "config/inflections" +require 'request_store' + # Protection settings set :protection, :except => :path_traversal @@ -141,6 +144,9 @@ use Rack::PostBodyToParams use Rack::ParamTranslator +use RequestStore::Middleware +use Rack::RequestLang + use LinkedData::Security::Authorization use LinkedData::Security::AccessDenied diff --git a/lib/rack/request_lang.rb b/lib/rack/request_lang.rb new file mode 100644 index 00000000..b2221041 --- /dev/null +++ b/lib/rack/request_lang.rb @@ -0,0 +1,16 @@ +module Rack + class RequestLang + + def initialize(app = nil, options = {}) + @app = app + end + + def call(env) + r = Rack::Request.new(env) + lang = r.params["lang"] || r.params["language"] + lang = lang.upcase.to_sym if lang + RequestStore.store[:requested_lang] = lang + @app.call(env) + end + end +end \ No newline at end of file diff --git a/views/documentation/documentation.haml b/views/documentation/documentation.haml index 527d781f..916eb1e7 100644 --- a/views/documentation/documentation.haml +++ b/views/documentation/documentation.haml @@ -151,6 +151,7 @@ %li include={prefLabel, synonym, definition, notation, cui, semanticType} // default = (see Common Parameters section) %li page={integer representing the page number} // default = 1 %li pagesize={integer representing the size of the returned page} // default = 50 + %li language={an ISO 639-1 language value, e.g 'fr' or 'en'} // by default search in all languages %h4#nav_search_subtree Subtree Search From f7eaf98320687e2c14c1bf8050de26d58f9c25db Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:22:40 +0200 Subject: [PATCH 12/46] update the search helper to add as a suffix the selected language --- helpers/search_helper.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 499e61ac..3096ab47 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -84,6 +84,9 @@ def get_term_search_query(text, params={}) end end + lang = params["lang"] || params["language"] + lang_suffix = lang && !lang.eql?("all") ? "_#{lang}" : "" + query = "" params["defType"] = "edismax" params["stopwords"] = "true" @@ -100,25 +103,25 @@ def get_term_search_query(text, params={}) if params[EXACT_MATCH_PARAM] == "true" query = "\"#{solr_escape(text)}\"" - params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact^10 synonymExact #{QUERYLESS_FIELDS_STR_NO_IDS}" - params["hl.fl"] = "resource_id prefLabelExact synonymExact #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact#{lang_suffix}^10 synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR}" elsif params[SUGGEST_PARAM] == "true" || text[-1] == '*' text.gsub!(/\*+$/, '') query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" - params["qf"] = "prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["qf"] = " prefLabelExact#{lang_suffix}^100 prefLabelSuggestEdge#{lang_suffix}^50 synonymSuggestEdge#{lang_suffix}^10 prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" params["pf"] = "prefLabelSuggest^50" - params["hl.fl"] = "prefLabelExact prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "prefLabelExact#{lang_suffix} prefLabelSuggestEdge#{lang_suffix} synonymSuggestEdge#{lang_suffix} prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? query = '*' else query = solr_escape(text) end - params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact^90 prefLabel^70 synonymExact^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" params["bq"] = "idAcronymMatch:true^80" - params["hl.fl"] = "resource_id prefLabelExact prefLabel synonymExact synonym #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} prefLabel#{lang_suffix} synonymExact#{lang_suffix} synonym#{lang_suffix} #{QUERYLESS_FIELDS_STR}" params["hl.fl"] = "#{params["hl.fl"]} property" if params[INCLUDE_PROPERTIES_PARAM] == "true" end @@ -347,6 +350,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) doc[:submission] = old_class.submission doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = LinkedData::Models::Class.read_only(doc) + instance.prefLabel = instance.prefLabel.first if instance.prefLabel.is_a?(Array) classes_hash[ont_uri_class_uri] = instance end From b5360d242d26c4f216f72cfe688b462a732e489f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:29:07 +0200 Subject: [PATCH 13/46] refactor test to be faster by doing the indexation only if needed --- test/controllers/test_annotator_controller.rb | 17 ++++++---- test/controllers/test_batch_controller.rb | 3 +- test/controllers/test_classes_controller.rb | 4 ++- test/controllers/test_metrics_controller.rb | 23 +++++++------ .../controllers/test_ontologies_controller.rb | 34 +++++++++++++++++-- .../test_ontology_submissions_controller.rb | 16 ++++++--- .../controllers/test_properties_controller.rb | 2 ++ .../test_properties_search_controller.rb | 2 ++ .../test_recommender_controller.rb | 2 +- .../test_recommender_v1_controller.rb | 4 +-- test/middleware/test_rack_attack.rb | 12 +++---- test/test_case.rb | 25 ++++++++++++++ 12 files changed, 108 insertions(+), 36 deletions(-) diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 3b21b9e7..55d8fb66 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -16,7 +16,12 @@ def self.before_suite end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true, + process_options: { + process_rdf: true, + extract_metadata: false, + index_search: true + }) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) @@ -260,16 +265,16 @@ def test_default_properties_output assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].first.downcase <=> b["annotatedClass"]["prefLabel"].first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] - assert_equal "Aggregate Human Data", annotations.first["annotatedClass"]["prefLabel"] + assert_equal "Aggregate Human Data", Array(annotations.first["annotatedClass"]["prefLabel"]).first params = {text: text, include: "prefLabel,definition"} get "/annotator", params assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| Array(a["annotatedClass"]["prefLabel"]).first.downcase <=> Array(b["annotatedClass"]["prefLabel"]).first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] assert_equal ["A resource that provides data from clinical care that comprises combined data from multiple individual human subjects."], annotations.first["annotatedClass"]["definition"] end @@ -348,7 +353,7 @@ def self.mapping_test_set classes = [] class_id = terms_a[i] ont_acr = onts_a[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) @@ -356,7 +361,7 @@ def self.mapping_test_set classes << c class_id = terms_b[i] ont_acr = onts_b[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) diff --git a/test/controllers/test_batch_controller.rb b/test/controllers/test_batch_controller.rb index 55d9cee9..d1e9d144 100644 --- a/test/controllers/test_batch_controller.rb +++ b/test/controllers/test_batch_controller.rb @@ -2,8 +2,7 @@ class TestBatchController < TestCase def self.before_suite - LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) end def test_class_batch_one_ontology diff --git a/test/controllers/test_classes_controller.rb b/test/controllers/test_classes_controller.rb index a918ece0..42ccf29b 100644 --- a/test/controllers/test_classes_controller.rb +++ b/test/controllers/test_classes_controller.rb @@ -7,7 +7,9 @@ def self.before_suite submission_count: 3, submissions_to_process: [1, 2], process_submission: true, - random_submission_count: false} + random_submission_count: false, + process_options: {process_rdf: true, extract_metadata: false} + } return LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end diff --git a/test/controllers/test_metrics_controller.rb b/test/controllers/test_metrics_controller.rb index 1b8890a6..b27fa198 100644 --- a/test/controllers/test_metrics_controller.rb +++ b/test/controllers/test_metrics_controller.rb @@ -18,11 +18,12 @@ def self.before_suite "individuals"=>124, "properties"=>63, "maxDepth"=> 7 } - @@options = {ont_count: 2, - submission_count: 3, - submissions_to_process: [1, 2], - process_submission: true, - random_submission_count: false} + @@options = { ont_count: 2, + submission_count: 3, + submissions_to_process: [1, 2], + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, run_metrics: true, index_properties: true }, + random_submission_count: false } LinkedData::SampleData::Ontology.create_ontologies_and_submissions(@@options) end @@ -78,18 +79,18 @@ def test_metrics_missing get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(0, ontologies.length, msg='Failure to detect 0 ontologies with missing metrics.') + assert_equal(0, ontologies.length, msg = 'Failure to detect 0 ontologies with missing metrics.') # create ontologies with latest submissions that have no metrics delete_ontologies_and_submissions - options = {ont_count: 2, - submission_count: 1, - process_submission: false, - random_submission_count: false} + options = { ont_count: 2, + submission_count: 1, + process_submission: false, + random_submission_count: false } create_ontologies_and_submissions(options) get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(2, ontologies.length, msg='Failure to detect 2 ontologies with missing metrics.') + assert_equal(2, ontologies.length, msg = 'Failure to detect 2 ontologies with missing metrics.') # recreate the before_suite data (this test might not be the last one to run in the suite) delete_ontologies_and_submissions create_ontologies_and_submissions(@@options) diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4f61256b..a5f242bd 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -188,7 +188,9 @@ def test_download_ontology end def test_download_ontology_csv - num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) + num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, + process_submission: true, + process_options:{process_rdf: true, extract_metadata: true, index_search: true}) ont = onts.first acronym = created_ont_acronyms.first @@ -220,13 +222,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test1@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test2@example.org", password: "12345" }) blocked_user.save @@ -296,6 +298,32 @@ def test_on_demand_ontology_pull end end + def test_detach_a_view + view = Ontology.find(@@view_acronym).include(:viewOf).first + ont = view.viewOf + refute_nil view + refute_nil ont + + remove_view_of = {viewOf: ''} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(remove_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}" + onto = MultiJson.load(last_response.body) + assert_nil onto["viewOf"] + + + add_view_of = {viewOf: @@acronym} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(add_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}?include=all" + onto = MultiJson.load(last_response.body) + assert_equal onto["viewOf"], ont.id.to_s + end + private def start_server diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 40532cd0..ef356fcf 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test3@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" @@ -36,6 +39,12 @@ def self._create_onts ont.save end + def setup + delete_ontologies_and_submissions + ont = Ontology.new(acronym: @@acronym, name: @@name, administeredBy: [@@user]) + ont.save + end + def test_submissions_for_given_ontology num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 1) ontology = created_ont_acronyms.first @@ -156,13 +165,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test4@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test5@example.org", password: "12345" }) blocked_user.save @@ -235,5 +244,4 @@ def test_ontology_submissions_access_controller del.delete if del end end - end diff --git a/test/controllers/test_properties_controller.rb b/test/controllers/test_properties_controller.rb index 8248403c..cbf249b8 100644 --- a/test/controllers/test_properties_controller.rb +++ b/test/controllers/test_properties_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_properties_search_controller.rb b/test/controllers/test_properties_search_controller.rb index f93a90a1..2589a293 100644 --- a/test/controllers/test_properties_search_controller.rb +++ b/test/controllers/test_properties_search_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesSearchController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_recommender_controller.rb b/test/controllers/test_recommender_controller.rb index 29caf28c..58d6d942 100644 --- a/test/controllers/test_recommender_controller.rb +++ b/test/controllers/test_recommender_controller.rb @@ -14,7 +14,7 @@ def self.before_suite @@redis.del(mappings) end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 7b14a63d..3ac4862d 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -1,10 +1,10 @@ require_relative '../test_case' -class TestRecommenderController < TestCase +class TestRecommenderV1Controller < TestCase def self.before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) @@text = <= max_retries + server_port = Random.rand(55000..65535) + end + server_port + end + private + def port_in_use?(port) + server = TCPServer.new(port) + server.close + false + rescue Errno::EADDRINUSE + true + end + end From 84f079ef8464b9c0e9916d344cbb30ad136b3f57 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:33:46 +0200 Subject: [PATCH 14/46] add multilingual search unit tests --- test/controllers/test_search_controller.rb | 100 ++++++++++++++++----- test/data/ontology_files/BRO_v3.2.owl | 3 + 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 70bf246e..851616f7 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -85,7 +85,7 @@ def test_search_ontology_filter assert last_response.ok? results = MultiJson.load(last_response.body) doc = results["collection"][0] - assert_equal "cell line", doc["prefLabel"] + assert_equal "cell line", doc["prefLabel"].first assert doc["links"]["ontology"].include? acronym results["collection"].each do |doc| acr = doc["links"]["ontology"].split('/')[-1] @@ -103,7 +103,8 @@ def test_search_other_filters get "search?q=data&require_definitions=true" assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 26, results["collection"].length + assert results["collection"].all? {|doc| !doc["definition"].nil? && doc.values.flatten.join(" ").include?("data") } + #assert_equal 26, results["collection"].length get "search?q=data&require_definitions=false" assert last_response.ok? @@ -115,10 +116,14 @@ def test_search_other_filters get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + + assert results["collection"].all? { |x| !x["obsolete"] } + count = results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=false" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + assert_equal count, results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=true" results = MultiJson.load(last_response.body) assert_equal 29, results["collection"].length @@ -134,8 +139,14 @@ def test_search_other_filters # testing cui and semantic_types flags get "search?q=Funding%20Resource&ontologies=#{acronym}&include=prefLabel,synonym,definition,notation,cui,semanticType" results = MultiJson.load(last_response.body) - assert_equal 35, results["collection"].length - assert_equal "Funding Resource", results["collection"][0]["prefLabel"] + #assert_equal 35, results["collection"].length + assert results["collection"].all? do |r| + ["prefLabel", "synonym", "definition", "notation", "cui", "semanticType"].map {|x| r[x]} + .flatten + .join(' ') + .include?("Funding Resource") + end + assert_equal "Funding Resource", results["collection"][0]["prefLabel"].first assert_equal "T028", results["collection"][0]["semanticType"][0] assert_equal "X123456", results["collection"][0]["cui"][0] @@ -190,7 +201,7 @@ def test_search_provisional_class assert_equal 10, results["collection"].length provisional = results["collection"].select {|res| assert_equal ontology_type, res["ontologyType"]; res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_root.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_root.label, provisional[0]["prefLabel"].first # subtree root with provisional class test get "search?ontology=#{acronym}&subtree_root_id=#{CGI::escape(@@cls_uri.to_s)}&also_search_provisional=true" @@ -199,7 +210,7 @@ def test_search_provisional_class provisional = results["collection"].select {|res| res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_child.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_child.label, provisional[0]["prefLabel"].first end def test_search_obo_id @@ -254,11 +265,11 @@ def test_search_obo_id assert_equal ogms_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[0]["links"]["ontology"]) assert_equal cno_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[1]["links"]["ontology"]) assert_equal ncit_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) - assert_equal 'realization', docs[1]["prefLabel"] - assert_equal 'realization', docs[2]["prefLabel"] - assert docs[3]["prefLabel"].upcase.include?('OGMS ') - assert docs[4]["prefLabel"].upcase.include?('OGMS ') - assert docs[5]["prefLabel"].upcase.include?('OGMS ') + assert_equal 'realization', docs[1]["prefLabel"].first + assert_equal 'realization', docs[2]["prefLabel"].first + assert docs[3]["prefLabel"].first.upcase.include?('OGMS ') + assert docs[4]["prefLabel"].first.upcase.include?('OGMS ') + assert docs[5]["prefLabel"].first.upcase.include?('OGMS ') get "/search?q=CNO:0000002" assert last_response.ok? @@ -270,10 +281,10 @@ def test_search_obo_id assert acr_1 === ncit_acronym || acr_1 === ogms_acronym acr_2= LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) assert acr_2 === ncit_acronym || acr_2 === ogms_acronym - assert docs[3]["prefLabel"].upcase.include?('CNO ') - assert docs[4]["prefLabel"].upcase.include?('CNO ') - assert docs[5]["prefLabel"].upcase.include?('CNO ') - assert docs[6]["prefLabel"].upcase.include?('CNO ') + assert docs[3]["prefLabel"].first.upcase.include?('CNO ') + assert docs[4]["prefLabel"].first.upcase.include?('CNO ') + assert docs[5]["prefLabel"].first.upcase.include?('CNO ') + assert docs[6]["prefLabel"].first.upcase.include?('CNO ') # mdorf, 3/2/2024, when the : is followed by a LETTER, as in NCIT:C20480, # then Solr does not split the query on the tokens, @@ -284,22 +295,22 @@ def test_search_obo_id results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"] + assert_equal 'Cellular Process', docs[0]["prefLabel"].first get "/search?q=NCIT:C20480" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"] + assert_equal 'Cellular Process', docs[0]["prefLabel"].first get "/search?q=Leukocyte Apoptotic Process&ontologies=#{ncit_acronym}" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] - assert_equal 'Leukocyte Apoptotic Process', docs[0]["prefLabel"] - assert_equal 'Leukocyte Apoptotic Test Class', docs[1]["prefLabel"] - assert_equal 'Lymphocyte Apoptotic Process', docs[2]["prefLabel"] + assert_equal 'Leukocyte Apoptotic Process', docs[0]["prefLabel"].first + assert_equal 'Leukocyte Apoptotic Test Class', docs[1]["prefLabel"].first + assert_equal 'Lymphocyte Apoptotic Process', docs[2]["prefLabel"].first ensure ont = LinkedData::Models::Ontology.find(ncit_acronym).first ont.delete if ont @@ -357,4 +368,49 @@ def test_search_short_id end end + def test_multilingual_search + get "/search?q=Activity&ontologies=BROSEARCHTEST-0" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + + doc = res["collection"].select{|doc| doc["@id"].to_s.eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + refute_nil doc + + res = LinkedData::Models::Class.search("prefLabel_none:Activity", {:fq => "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}) + refute_equal 0, res["response"]["numFound"] + refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activity&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + end + + end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index d64075cc..b2aeccf5 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -631,6 +631,9 @@ Activity + Activity + ActivityEnglish + Activité Activity of interest that may be related to a BRO:Resource. activities From f8db0d81036479dcdc7356082bb79d5aebb7eca8 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Wed, 8 May 2024 10:56:18 -0700 Subject: [PATCH 15/46] use gems from ncbo repo --- Gemfile | 15 +++++----- Gemfile.lock | 84 ++++++++++++++++++++++++++++------------------------ 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/Gemfile b/Gemfile index d25c7660..894d1651 100644 --- a/Gemfile +++ b/Gemfile @@ -12,7 +12,6 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' -gem 'request_store' # Rack middleware gem 'ffi' @@ -41,13 +40,13 @@ gem 'unicorn-worker-killer' gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' -# NCBO gems (can be from a local dev path or from rubygems/git) -gem 'ncbo_annotator', git: 'https://github.com/ncbo/ncbo_annotator.git', branch: 'master' -gem 'ncbo_cron', git: 'https://github.com/ncbo/ncbo_cron.git', branch: 'master' -gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' -gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'pr/sync-agroportal-bioportal' -gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' +# NCBO +gem 'goo', github: 'ncbo/goo', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'feature/multilingual-support-add-search-and-languages-serialization' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 488c23ca..f65daacd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,23 @@ +GIT + remote: https://github.com/ncbo/goo.git + revision: a55ef42e92bc02129992c77890278831f8ca5387 + branch: develop + specs: + goo (0.0.2) + addressable (~> 2.8) + pry + rdf (= 1.0.8) + redis + request_store + rest-client + rsolr + sparql-client + uuid + GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e + branch: develop specs: ncbo_annotator (0.0.1) goo @@ -11,11 +27,13 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: f239b34230131cd7cfb9eec5b34068ba990b698b - branch: master + revision: 98dc1a9bb8aba4ef41a1687251ca1245edd5de57 + branch: develop specs: ncbo_cron (0.0.1) dante + faraday (~> 2) + faraday-follow_redirects (~> 0) goo google-analytics-data mlanett-redis-lock @@ -27,8 +45,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 - branch: master + revision: f92a42f660635522eb8709e618ff2e641aef0d17 + branch: develop specs: ncbo_ontology_recommender (0.0.1) goo @@ -37,35 +55,9 @@ GIT redis GIT - remote: https://github.com/ncbo/sparql-client.git - revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc - branch: master - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - -GIT - remote: https://github.com/ontoportal-lirmm/goo.git - revision: f2751fe9324e48a0a5c2a8b15580ab879fc53a2b - branch: pr/sync-agroportal-bioportal - specs: - goo (0.0.2) - addressable (~> 2.8) - pry - rdf (= 1.0.8) - redis - request_store - rest-client - rsolr - sparql-client - uuid - -GIT - remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: fc48343ad7a3f1867821c173be900cf16506b4b7 - branch: pr/sync-agroportal-ncbo + remote: https://github.com/ncbo/ontologies_linked_data.git + revision: ea7b31b70ea9cbc9a08821a89bd8d28761dd4b32 + branch: feature/multilingual-support-add-search-and-languages-serialization specs: ontologies_linked_data (0.0.1) activesupport @@ -82,6 +74,16 @@ GIT rsolr rubyzip +GIT + remote: https://github.com/ncbo/sparql-client.git + revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc + branch: develop + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 @@ -143,6 +145,8 @@ GEM base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) @@ -169,6 +173,7 @@ GEM google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) + google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) @@ -185,6 +190,9 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) + grpc (1.63.0-aarch64-linux) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) @@ -223,7 +231,7 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0305) + mime-types-data (3.2024.0507) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -232,7 +240,7 @@ GEM multi_json (1.15.0) mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.10) + net-imap (0.4.11) date net-protocol net-pop (0.1.2) @@ -380,6 +388,7 @@ GEM hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS + aarch64-linux arm64-darwin-22 arm64-darwin-23 x86_64-darwin-18 @@ -424,7 +433,6 @@ DEPENDENCIES redis redis-rack-cache (~> 2.0) redis-store (~> 1.10) - request_store rubocop shotgun! simplecov From 3dc9f9dcc074956e5a37b511aa63b435042caf6f Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Thu, 9 May 2024 13:23:53 -0700 Subject: [PATCH 16/46] Gemfile update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f65daacd..6bfc719c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: ea7b31b70ea9cbc9a08821a89bd8d28761dd4b32 + revision: 0227d37c6431f52a7fd85bdb597cc3d32e975098 branch: feature/multilingual-support-add-search-and-languages-serialization specs: ontologies_linked_data (0.0.1) @@ -315,7 +315,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.63.4) + rubocop (1.63.5) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) From 9ca250049c6615d8f94c3fb406e8d3753dea814a Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 20 May 2024 20:26:12 +0200 Subject: [PATCH 17/46] make the api use submission naturalLanguage to choose the default language to display --- Gemfile | 2 +- Gemfile.lock | 52 ++++++++++++--------- helpers/application_helper.rb | 19 ++++++++ test/controllers/test_classes_controller.rb | 26 ++++++++++- test/data/ontology_files/BRO_v3.2.owl | 2 +- 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/Gemfile b/Gemfile index d25c7660..f2ecfca5 100644 --- a/Gemfile +++ b/Gemfile @@ -46,7 +46,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ncbo/ncbo_annotator.git', branch: gem 'ncbo_cron', git: 'https://github.com/ncbo/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' -gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'pr/sync-agroportal-bioportal' +gem 'goo', github: 'ncbo/goo', branch: 'develop' gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 488c23ca..95b07be8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,19 @@ +GIT + remote: https://github.com/ncbo/goo.git + revision: 70007faf43d0d96292809f89f15a27a757ccaa25 + branch: develop + specs: + goo (0.0.2) + addressable (~> 2.8) + pry + rdf (= 1.0.8) + redis + request_store + rest-client + rsolr + sparql-client + uuid + GIT remote: https://github.com/ncbo/ncbo_annotator.git revision: 63c986880aa88c9384043e6611a682434a14aba7 @@ -11,11 +27,13 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: f239b34230131cd7cfb9eec5b34068ba990b698b + revision: 67810ddd2e14876d789c63c427b08af6919f661e branch: master specs: ncbo_cron (0.0.1) dante + faraday (~> 2) + faraday-follow_redirects (~> 0) goo google-analytics-data mlanett-redis-lock @@ -46,25 +64,9 @@ GIT net-http-persistent (= 2.9.4) rdf (>= 1.0) -GIT - remote: https://github.com/ontoportal-lirmm/goo.git - revision: f2751fe9324e48a0a5c2a8b15580ab879fc53a2b - branch: pr/sync-agroportal-bioportal - specs: - goo (0.0.2) - addressable (~> 2.8) - pry - rdf (= 1.0.8) - redis - request_store - rest-client - rsolr - sparql-client - uuid - GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: fc48343ad7a3f1867821c173be900cf16506b4b7 + revision: 7c5184fc7d3b25b5c9dd54c25f5ea2b1c4951318 branch: pr/sync-agroportal-ncbo specs: ontologies_linked_data (0.0.1) @@ -143,6 +145,8 @@ GEM base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) @@ -223,7 +227,7 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0305) + mime-types-data (3.2024.0507) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -232,7 +236,7 @@ GEM multi_json (1.15.0) mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.10) + net-imap (0.4.11) date net-protocol net-pop (0.1.2) @@ -295,7 +299,7 @@ GEM redis-store (>= 1.6, < 2) redis-store (1.10.0) redis (>= 4, < 6) - regexp_parser (2.9.0) + regexp_parser (2.9.2) request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) @@ -303,11 +307,12 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.6) + rexml (3.2.8) + strscan (>= 3.0.9) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.63.4) + rubocop (1.63.5) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -359,6 +364,7 @@ GEM net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) + strscan (3.1.0) systemu (2.6.5) temple (0.10.3) tilt (2.3.0) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 251f90f0..916c24c9 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -409,6 +409,9 @@ def get_ontology_and_submission error 404, "Ontology #{@params["acronym"]} does not have any submissions" end end + + save_submission_language(submission) + return ont, submission end @@ -438,6 +441,22 @@ def naive_expiring_cache_read(key) return object[:object] end + + def save_submission_language(submission, language_property = :naturalLanguage) + request_lang = RequestStore.store[:requested_lang] + + return if submission.nil? || !request_lang.blank? + + submission.bring(language_property) if submission.bring?(language_property) + collection_natural_language = submission.send(language_property) rescue nil + return [] if collection_natural_language.blank? + + collection_natural_language = collection_natural_language.values.flatten if collection_natural_language.is_a?(Hash) + submissions_language = collection_natural_language.map { |natural_language| natural_language.to_s.split('/').last[0..1] }.compact.first + + RequestStore.store[:requested_lang] = submissions_language if submissions_language + end + end end end diff --git a/test/controllers/test_classes_controller.rb b/test/controllers/test_classes_controller.rb index 42ccf29b..e43d9e0a 100644 --- a/test/controllers/test_classes_controller.rb +++ b/test/controllers/test_classes_controller.rb @@ -8,7 +8,8 @@ def self.before_suite submissions_to_process: [1, 2], process_submission: true, random_submission_count: false, - process_options: {process_rdf: true, extract_metadata: false} + process_options: {process_rdf: true, extract_metadata: false}, + file_path: "./test/data/ontology_files/BRO_v3.2.owl", } return LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end @@ -531,4 +532,27 @@ def test_class_page_with_metric_count assert page_response["collection"].length == 0 end + def test_default_multilingual + ont = Ontology.find("TEST-ONT-0").include(:acronym).first + sub = ont.latest_submission + sub.bring_remaining + + get "/ontologies/#{ont.acronym}/classes/#{CGI.escape('http://bioontology.org/ontologies/Activity.owl#Biospecimen_Management')}" + assert last_response.ok? + page_response = MultiJson.load(last_response.body) + + # does not contain a value in english show the generated one + assert_equal 'Biospecimen_Management', page_response["prefLabel"] + + + sub.naturalLanguage = ['fr'] + sub.save + + get "/ontologies/#{ont.acronym}/classes/#{CGI.escape('http://bioontology.org/ontologies/Activity.owl#Biospecimen_Management')}" + assert last_response.ok? + page_response = MultiJson.load(last_response.body) + + # show french value as specified in submission naturalLanguage + assert_equal 'Biospecimen Management', page_response["prefLabel"] + end end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index b2aeccf5..7417f79c 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -645,7 +645,7 @@ Activity related to the creation, use, or maintenance of a biorepository (http://en.wikipedia.org/wiki/Biorepository) - Biospecimen Management + Biospecimen Management From 0ee0a68a5bd8ac461e1c06f03e17b656495a9ee6 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 21 May 2024 14:01:02 -0700 Subject: [PATCH 18/46] Gemfile update --- Gemfile | 10 +++++----- Gemfile.lock | 38 +++++++++++++++++++------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Gemfile b/Gemfile index cc8001b2..21509c84 100644 --- a/Gemfile +++ b/Gemfile @@ -42,12 +42,12 @@ gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'ncbo_annotator', git: 'https://github.com/ncbo/ncbo_annotator.git', branch: 'master' -gem 'ncbo_cron', git: 'https://github.com/ncbo/ncbo_cron.git', branch: 'master' -gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'feature/multilingual-support-add-search-and-languages-serialization' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index d0cbf578..91060fcc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e + branch: develop specs: ncbo_annotator (0.0.1) goo @@ -27,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 67810ddd2e14876d789c63c427b08af6919f661e - branch: master + revision: 98dc1a9bb8aba4ef41a1687251ca1245edd5de57 + branch: develop specs: ncbo_cron (0.0.1) dante @@ -45,8 +45,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 - branch: master + revision: f92a42f660635522eb8709e618ff2e641aef0d17 + branch: develop specs: ncbo_ontology_recommender (0.0.1) goo @@ -55,19 +55,9 @@ GIT redis GIT - remote: https://github.com/ncbo/sparql-client.git - revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc - branch: master - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - -GIT - remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 67f9e10c03880431590b15ea5b27d2f1fe1354c4 - branch: pr/sync-agroportal-ncbo + remote: https://github.com/ncbo/ontologies_linked_data.git + revision: 112e39b0b75901c3e32ccd83e2e5f9e83ab73d42 + branch: feature/multilingual-support-add-search-and-languages-serialization specs: ontologies_linked_data (0.0.1) activesupport @@ -84,6 +74,16 @@ GIT rsolr rubyzip +GIT + remote: https://github.com/ncbo/sparql-client.git + revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc + branch: develop + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 From 321b37b808fcf11766ea3a8e992dcb9019514046 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 21 May 2024 15:30:25 -0700 Subject: [PATCH 19/46] Gemfile update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 91060fcc..022a4fd8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 112e39b0b75901c3e32ccd83e2e5f9e83ab73d42 + revision: c2beff66fb8ccf77a30606b8397799b641b8c6f7 branch: feature/multilingual-support-add-search-and-languages-serialization specs: ontologies_linked_data (0.0.1) From 4f5d0b06d2060c3ff0be85e8805dd9791e880a4c Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 21 May 2024 15:47:53 -0700 Subject: [PATCH 20/46] Gemfile update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 022a4fd8..998f701b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,8 +56,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: c2beff66fb8ccf77a30606b8397799b641b8c6f7 - branch: feature/multilingual-support-add-search-and-languages-serialization + revision: b9a6a9de53fd4e66b684129ca901a08a6e08438b + branch: develop specs: ontologies_linked_data (0.0.1) activesupport From 2e9853589e497d9a8f6546690f543f256cc73203 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 21 May 2024 16:38:28 -0700 Subject: [PATCH 21/46] Gemfile update --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 21509c84..5e5c0c8b 100644 --- a/Gemfile +++ b/Gemfile @@ -47,7 +47,7 @@ gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'feature/multilingual-support-add-search-and-languages-serialization' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 998f701b..a1503b67 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: b9a6a9de53fd4e66b684129ca901a08a6e08438b + revision: 048087406b7e7ea6530ed3e02254544f3ca74c0f branch: develop specs: ontologies_linked_data (0.0.1) From 2da8763f0db9c90e9f385d496862c92569e239ba Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 21 May 2024 16:39:26 -0700 Subject: [PATCH 22/46] use 127.0.0.1 for better compat --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 55de76eb..20934d3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,7 +115,7 @@ services: image: ontoportal/mgrep:0.0.2 platform: linux/amd64 healthcheck: - test: ["CMD", "nc", "-z", "-v", "localhost", "55556"] + test: ["CMD", "nc", "-z", "-v", "127.0.0.1", "55556"] start_period: 3s interval: 10s timeout: 5s From c53f7598d1a7a530863340dc1b993e459adb5de2 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Wed, 22 May 2024 11:05:44 -0700 Subject: [PATCH 23/46] Gemfile update --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a1503b67..10597308 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 98dc1a9bb8aba4ef41a1687251ca1245edd5de57 + revision: 89b68775fca3ab7d6451171bc3fee7679a82a1d5 branch: develop specs: ncbo_cron (0.0.1) @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 048087406b7e7ea6530ed3e02254544f3ca74c0f + revision: 013a322440b4cfd899908d15ff605799acf8e6ef branch: develop specs: ontologies_linked_data (0.0.1) @@ -298,7 +298,7 @@ GEM redcarpet (3.6.0) redis (5.2.0) redis-client (>= 0.22.0) - redis-client (0.22.1) + redis-client (0.22.2) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) From 05e4bba33ae4021370e0a314fc34e1fccba4e860 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Thu, 23 May 2024 10:24:26 -0700 Subject: [PATCH 24/46] remove duplicates, fix error message remove extra "i" from the error message remove reduntant code --- helpers/application_helper.rb | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 916c24c9..04999f65 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -396,18 +396,9 @@ def get_ontology_and_submission else submission = ont.latest_submission(status: [:RDF]) end - if submission.nil? - error 404, "Ontology #{@params["ontology"]} submission not found." - end + error 404, "Ontology #{@params["ontology"]} submission not found." if submission.nil? if !submission.ready?(status: [:RDF]) - error(404, - "Ontology #{@params["ontology"]} submission i"+ - "#{submission.submissionId} has not been parsed.") - end - if submission.nil? - if submission.nil? - error 404, "Ontology #{@params["acronym"]} does not have any submissions" - end + error 404, "Ontology #{@params["ontology"]} submission #{submission.submissionId} has not been parsed." end save_submission_language(submission) From 75b98e0f94d272554f9223e8cf9935b687c0056e Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 11 Jun 2024 23:45:30 -0700 Subject: [PATCH 25/46] reset branch specifier to master --- Gemfile | 12 +++++----- Gemfile.lock | 67 +++++++++++++++++++++++++++------------------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/Gemfile b/Gemfile index 5e5c0c8b..4f64b0e5 100644 --- a/Gemfile +++ b/Gemfile @@ -42,12 +42,12 @@ gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'master' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 10597308..4373e47f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 70007faf43d0d96292809f89f15a27a757ccaa25 - branch: develop + revision: 2cec53c2d84179171def81d7dc0a18e462e78fc2 + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -16,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e - branch: develop + revision: 63c986880aa88c9384043e6611a682434a14aba7 + branch: master specs: ncbo_annotator (0.0.1) goo @@ -27,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 89b68775fca3ab7d6451171bc3fee7679a82a1d5 - branch: develop + revision: f6c4ca0e014a1744d5a6460b9bf72784cb8428c8 + branch: master specs: ncbo_cron (0.0.1) dante @@ -45,8 +45,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: f92a42f660635522eb8709e618ff2e641aef0d17 - branch: develop + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master specs: ncbo_ontology_recommender (0.0.1) goo @@ -56,8 +56,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 013a322440b4cfd899908d15ff605799acf8e6ef - branch: develop + revision: 70fc2c8b377b9deabc838ecd56d3bc75ec764cc2 + branch: master specs: ontologies_linked_data (0.0.1) activesupport @@ -76,8 +76,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc - branch: develop + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc + branch: master specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -118,7 +118,7 @@ GEM bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) bigdecimal (1.4.2) - builder (3.2.4) + builder (3.3.0) capistrano (3.18.1) airbrussh (>= 1.0.0) i18n @@ -132,7 +132,7 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.3) connection_pool (2.4.1) crack (1.0.0) bigdecimal @@ -152,7 +152,10 @@ GEM faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) - ffi (1.16.3) + ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) @@ -179,12 +182,12 @@ GEM google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) - googleapis-common-protos (1.5.0) - google-protobuf (~> 3.18) + googleapis-common-protos (1.6.0) + google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.14.0) - google-protobuf (~> 3.18) + googleapis-common-protos-types (1.15.0) + google-protobuf (>= 3.18, < 5.a) googleauth (1.11.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) @@ -192,16 +195,16 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.63.0-aarch64-linux) + grpc (1.64.0-aarch64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-arm64-darwin) + grpc (1.64.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-x86_64-darwin) + grpc (1.64.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-x86_64-linux) + grpc (1.64.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -209,7 +212,7 @@ GEM tilt hashdiff (1.1.0) http-accept (1.7.0) - http-cookie (1.0.5) + http-cookie (1.0.6) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) @@ -233,7 +236,7 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0507) + mime-types-data (3.2024.0604) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -242,7 +245,7 @@ GEM multi_json (1.15.0) mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.11) + net-imap (0.4.12) date net-protocol net-pop (0.1.2) @@ -257,14 +260,14 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.9.0) + newrelic_rpm (9.10.2) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) - parallel (1.24.0) + parallel (1.25.1) parseconfig (1.1.2) - parser (3.3.1.0) + parser (3.3.2.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -313,12 +316,12 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.8) - strscan (>= 3.0.9) + rexml (3.3.0) + strscan rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.63.5) + rubocop (1.64.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) From 888eb0b2772ebeb7fa8130d21a727a66974a0cc3 Mon Sep 17 00:00:00 2001 From: mdorf Date: Mon, 17 Jun 2024 20:37:49 -0700 Subject: [PATCH 26/46] fixed ontologies_linked_data#211 - Multilingual Ontologies: search returns array instead of a string --- Gemfile.lock | 40 ++++++++++++++++---------------- controllers/search_controller.rb | 1 + helpers/search_helper.rb | 16 +++++++++++++ 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 10597308..deade615 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -118,7 +118,7 @@ GEM bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) bigdecimal (1.4.2) - builder (3.2.4) + builder (3.3.0) capistrano (3.18.1) airbrussh (>= 1.0.0) i18n @@ -132,7 +132,7 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.3) connection_pool (2.4.1) crack (1.0.0) bigdecimal @@ -152,7 +152,7 @@ GEM faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) - ffi (1.16.3) + ffi (1.17.0) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) @@ -179,12 +179,12 @@ GEM google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) - googleapis-common-protos (1.5.0) - google-protobuf (~> 3.18) + googleapis-common-protos (1.6.0) + google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.14.0) - google-protobuf (~> 3.18) + googleapis-common-protos-types (1.15.0) + google-protobuf (>= 3.18, < 5.a) googleauth (1.11.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) @@ -192,16 +192,16 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.63.0-aarch64-linux) + grpc (1.64.0-aarch64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-arm64-darwin) + grpc (1.64.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-x86_64-darwin) + grpc (1.64.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-x86_64-linux) + grpc (1.64.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -209,7 +209,7 @@ GEM tilt hashdiff (1.1.0) http-accept (1.7.0) - http-cookie (1.0.5) + http-cookie (1.0.6) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) @@ -233,7 +233,7 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0507) + mime-types-data (3.2024.0604) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -242,7 +242,7 @@ GEM multi_json (1.15.0) mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.11) + net-imap (0.4.12) date net-protocol net-pop (0.1.2) @@ -257,14 +257,14 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.9.0) + newrelic_rpm (9.10.2) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) - parallel (1.24.0) + parallel (1.25.1) parseconfig (1.1.2) - parser (3.3.1.0) + parser (3.3.3.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -313,12 +313,12 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.8) - strscan (>= 3.0.9) + rexml (3.3.0) + strscan rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.63.5) + rubocop (1.64.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 3bc1c13f..492ab2f9 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -45,6 +45,7 @@ def process_search(params=nil) doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) + transform_preflabel(params, instance) docs.push(instance) end diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 3096ab47..d799e500 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -19,6 +19,7 @@ module SearchHelper VALUESET_ROOTS_ONLY_PARAM = "valueset_roots_only" VALUESET_EXCLUDE_ROOTS_PARAM = "valueset_exclude_roots" ONTOLOGY_TYPES_PARAM = "ontology_types" + LANGUAGES_PARAM = "lang" ALSO_SEARCH_VIEWS = "also_search_views" # NCBO-961 MATCH_HTML_PRE = "" @@ -28,6 +29,7 @@ module SearchHelper MATCH_TYPE_PROPERTY = "property" MATCH_TYPE_LABEL = "label" MATCH_TYPE_LABELGENERATED = "labelGenerated" + NO_LANGUAGE_SUFFIX = "none" MATCH_TYPE_MAP = { "resource_id" => "id", @@ -357,6 +359,20 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) classes_hash end + def transform_preflabel(params, class_instance) + site_label = Goo.main_languages[0] + + unless RequestStore.store[:requested_lang] + if class_instance.respond_to?("#{MATCH_TYPE_PREFLABEL}_#{site_label}") && class_instance["#{MATCH_TYPE_PREFLABEL}_#{site_label}"] + class_instance[MATCH_TYPE_PREFLABEL] = class_instance["#{MATCH_TYPE_PREFLABEL}_#{site_label}"][0] + elsif class_instance.respond_to?("#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}") && class_instance["#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}"] + class_instance[MATCH_TYPE_PREFLABEL] = class_instance["#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}"][0] + else + class_instance[MATCH_TYPE_PREFLABEL] = class_instance[MATCH_TYPE_PREFLABEL][0] + end + end + end + def validate_params_solr_population(allowed_includes_params) leftover = includes_param - allowed_includes_params invalid = leftover.length > 0 From bcd8938d79810a90a4545a2960802e2ad17bb347 Mon Sep 17 00:00:00 2001 From: mdorf Date: Mon, 17 Jun 2024 20:38:21 -0700 Subject: [PATCH 27/46] Gemfile.lock update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index deade615..6ded14ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -242,7 +242,7 @@ GEM multi_json (1.15.0) mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.12) + net-imap (0.4.13) date net-protocol net-pop (0.1.2) @@ -272,7 +272,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.5) + public_suffix (5.1.1) racc (1.8.0) rack (1.6.13) rack-accept (0.4.5) From 130ee636cb3446746e73815e119a93036174cfc8 Mon Sep 17 00:00:00 2001 From: mdorf Date: Tue, 18 Jun 2024 15:10:55 -0700 Subject: [PATCH 28/46] fixed failing unit tests related to multi-lingual functionality --- test/controllers/test_search_controller.rb | 63 +++++++++++++++------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 4c216578..54165b74 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -85,7 +85,10 @@ def test_search_ontology_filter assert last_response.ok? results = MultiJson.load(last_response.body) doc = results["collection"][0] - assert_equal "cell line", doc["prefLabel"].first + + pref_label = doc["prefLabel"].kind_of?(Array) ? doc["prefLabel"].first : doc["prefLabel"] + assert_equal "cell line", pref_label + assert doc["links"]["ontology"].include? acronym results["collection"].each do |doc| acr = doc["links"]["ontology"].split('/')[-1] @@ -146,7 +149,9 @@ def test_search_other_filters .join(' ') .include?("Funding Resource") end - assert_equal "Funding Resource", results["collection"][0]["prefLabel"].first + + label0 = results["collection"][0]["prefLabel"].kind_of?(Array) ? results["collection"][0]["prefLabel"].first : results["collection"][0]["prefLabel"] + assert_equal "Funding Resource", label0 assert_equal "T028", results["collection"][0]["semanticType"][0] assert_equal "X123456", results["collection"][0]["cui"][0] @@ -201,7 +206,8 @@ def test_search_provisional_class assert_equal 10, results["collection"].length provisional = results["collection"].select {|res| assert_equal ontology_type, res["ontologyType"]; res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_root.label, provisional[0]["prefLabel"].first + prov_label = provisional[0]["prefLabel"].kind_of?(Array) ? provisional[0]["prefLabel"].first : provisional[0]["prefLabel"] + assert_equal @@test_pc_root.label, prov_label # subtree root with provisional class test get "search?ontology=#{acronym}&subtree_root_id=#{CGI::escape(@@cls_uri.to_s)}&also_search_provisional=true" @@ -210,7 +216,9 @@ def test_search_provisional_class provisional = results["collection"].select {|res| res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_child.label, provisional[0]["prefLabel"].first + + prov_label = provisional[0]["prefLabel"].kind_of?(Array) ? provisional[0]["prefLabel"].first : provisional[0]["prefLabel"] + assert_equal @@test_pc_child.label, prov_label end def test_search_obo_id @@ -266,12 +274,17 @@ def test_search_obo_id assert_equal cno_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[1]["links"]["ontology"]) assert_equal ncit_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) - assert_equal 'realization', docs[1]["prefLabel"].first - assert_equal 'realization', docs[2]["prefLabel"].first - assert docs[3]["prefLabel"].first.upcase.include?('OGMS ') - assert docs[4]["prefLabel"].first.upcase.include?('OGMS ') - assert docs[5]["prefLabel"].first.upcase.include?('OGMS ') + label1 = docs[1]["prefLabel"].kind_of?(Array) ? docs[1]["prefLabel"].first : docs[1]["prefLabel"] + label2 = docs[2]["prefLabel"].kind_of?(Array) ? docs[2]["prefLabel"].first : docs[2]["prefLabel"] + label3 = docs[3]["prefLabel"].kind_of?(Array) ? docs[3]["prefLabel"].first : docs[3]["prefLabel"] + label4 = docs[4]["prefLabel"].kind_of?(Array) ? docs[4]["prefLabel"].first : docs[4]["prefLabel"] + label5 = docs[5]["prefLabel"].kind_of?(Array) ? docs[5]["prefLabel"].first : docs[5]["prefLabel"] + assert_equal 'realization', label1 + assert_equal 'realization', label2 + assert label3.upcase.include?('OGMS ') + assert label4.upcase.include?('OGMS ') + assert label5.upcase.include?('OGMS ') get "/search?q=CNO:0000002" assert last_response.ok? @@ -284,11 +297,16 @@ def test_search_obo_id acr_2= LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) assert acr_2 === ncit_acronym || acr_2 === ogms_acronym - assert docs[3]["prefLabel"].first.upcase.include?('CNO ') - assert docs[4]["prefLabel"].first.upcase.include?('CNO ') - assert docs[5]["prefLabel"].first.upcase.include?('CNO ') - assert docs[6]["prefLabel"].first.upcase.include?('CNO ') + label3 = docs[3]["prefLabel"].kind_of?(Array) ? docs[3]["prefLabel"].first : docs[3]["prefLabel"] + label4 = docs[4]["prefLabel"].kind_of?(Array) ? docs[4]["prefLabel"].first : docs[4]["prefLabel"] + label5 = docs[5]["prefLabel"].kind_of?(Array) ? docs[5]["prefLabel"].first : docs[5]["prefLabel"] + label6 = docs[6]["prefLabel"].kind_of?(Array) ? docs[6]["prefLabel"].first : docs[6]["prefLabel"] + + assert label3.upcase.include?('CNO ') + assert label4.upcase.include?('CNO ') + assert label5.upcase.include?('CNO ') + assert label6.upcase.include?('CNO ') # mdorf, 3/2/2024, when the : is followed by a LETTER, as in NCIT:C20480, # then Solr does not split the query on the tokens, @@ -299,22 +317,31 @@ def test_search_obo_id results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"].first + + label0 = docs[0]["prefLabel"].kind_of?(Array) ? docs[0]["prefLabel"].first : docs[0]["prefLabel"] + assert_equal 'Cellular Process', label0 get "/search?q=NCIT:C20480" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"].first + + label0 = docs[0]["prefLabel"].kind_of?(Array) ? docs[0]["prefLabel"].first : docs[0]["prefLabel"] + assert_equal 'Cellular Process', label0 get "/search?q=Leukocyte Apoptotic Process&ontologies=#{ncit_acronym}" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] - assert_equal 'Leukocyte Apoptotic Process', docs[0]["prefLabel"].first - assert_equal 'Leukocyte Apoptotic Test Class', docs[1]["prefLabel"].first - assert_equal 'Lymphocyte Apoptotic Process', docs[2]["prefLabel"].first + + label0 = docs[0]["prefLabel"].kind_of?(Array) ? docs[0]["prefLabel"].first : docs[0]["prefLabel"] + label1 = docs[1]["prefLabel"].kind_of?(Array) ? docs[1]["prefLabel"].first : docs[1]["prefLabel"] + label2 = docs[2]["prefLabel"].kind_of?(Array) ? docs[2]["prefLabel"].first : docs[2]["prefLabel"] + + assert_equal 'Leukocyte Apoptotic Process', label0 + assert_equal 'Leukocyte Apoptotic Test Class', label1 + assert_equal 'Lymphocyte Apoptotic Process', label2 ensure ont = LinkedData::Models::Ontology.find(ncit_acronym).first ont.delete if ont From a29960b9cd2b7dd0bb5f6578530b1181050b99c5 Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 19 Jun 2024 12:48:08 -0700 Subject: [PATCH 29/46] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6ded14ac..e73f0d7d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -217,7 +217,7 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.7.2) - jwt (2.8.1) + jwt (2.8.2) base64 kgio (2.11.4) language_server-protocol (3.17.0.3) From a4e9013cb74e8a3c85f0d51d9784acdc06fb3555 Mon Sep 17 00:00:00 2001 From: mdorf Date: Thu, 27 Jun 2024 14:18:29 -0700 Subject: [PATCH 30/46] fixes to ncbo/ontologies_linked_data#211 and ontoportal/ontoportal-project#56 --- controllers/search_controller.rb | 2 +- helpers/search_helper.rb | 42 +++++++++++++------ test/controllers/test_search_controller.rb | 49 +++++++++++++++++++--- test/data/ontology_files/BRO_v3.2.owl | 9 ++-- 4 files changed, 81 insertions(+), 21 deletions(-) diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 492ab2f9..280944a6 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -45,7 +45,7 @@ def process_search(params=nil) doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) - transform_preflabel(params, instance) + filter_language_attributes(params, instance) docs.push(instance) end diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index d799e500..4e2dee3c 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -27,6 +27,7 @@ module SearchHelper MATCH_TYPE_PREFLABEL = "prefLabel" MATCH_TYPE_SYNONYM = "synonym" MATCH_TYPE_PROPERTY = "property" + MATCH_TYPE_DEFINITION = "definition" MATCH_TYPE_LABEL = "label" MATCH_TYPE_LABELGENERATED = "labelGenerated" NO_LANGUAGE_SUFFIX = "none" @@ -112,7 +113,7 @@ def get_term_search_query(text, params={}) query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" params["qf"] = " prefLabelExact#{lang_suffix}^100 prefLabelSuggestEdge#{lang_suffix}^50 synonymSuggestEdge#{lang_suffix}^10 prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" - params["pf"] = "prefLabelSuggest^50" + params["pf"] = "prefLabelSuggest#{lang_suffix}^50" params["hl.fl"] = "prefLabelExact#{lang_suffix} prefLabelSuggestEdge#{lang_suffix} synonymSuggestEdge#{lang_suffix} prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? @@ -120,7 +121,7 @@ def get_term_search_query(text, params={}) else query = solr_escape(text) end - params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym#{lang_suffix}^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" params["bq"] = "idAcronymMatch:true^80" params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} prefLabel#{lang_suffix} synonymExact#{lang_suffix} synonym#{lang_suffix} #{QUERYLESS_FIELDS_STR}" @@ -184,6 +185,10 @@ def get_term_search_query(text, params={}) params["fq"] = filter_query params["q"] = query + + # binding.pry + + query end @@ -359,20 +364,33 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) classes_hash end - def transform_preflabel(params, class_instance) - site_label = Goo.main_languages[0] - - unless RequestStore.store[:requested_lang] - if class_instance.respond_to?("#{MATCH_TYPE_PREFLABEL}_#{site_label}") && class_instance["#{MATCH_TYPE_PREFLABEL}_#{site_label}"] - class_instance[MATCH_TYPE_PREFLABEL] = class_instance["#{MATCH_TYPE_PREFLABEL}_#{site_label}"][0] - elsif class_instance.respond_to?("#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}") && class_instance["#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}"] - class_instance[MATCH_TYPE_PREFLABEL] = class_instance["#{MATCH_TYPE_PREFLABEL}_#{NO_LANGUAGE_SUFFIX}"][0] - else - class_instance[MATCH_TYPE_PREFLABEL] = class_instance[MATCH_TYPE_PREFLABEL][0] + def filter_language_attribute(params, class_instance, attr, is_single) + if class_instance.respond_to?(attr) + lang_param = (params["lang"] || params["language"])&.downcase + lang_suffix = lang_param && !lang_param.eql?("all") ? "_#{lang_param}" : "" + + if !lang_suffix.empty? && class_instance.respond_to?("#{attr}#{lang_suffix}") + class_instance[attr] = is_single ? class_instance["#{attr}#{lang_suffix}"][0] : class_instance["#{attr}#{lang_suffix}"] + elsif !lang_param.eql?("all") + site_label = Goo.main_languages[0] + + if class_instance.respond_to?("#{attr}_#{site_label}") && class_instance["#{attr}_#{site_label}"] + class_instance[attr] = is_single ? class_instance["#{attr}_#{site_label}"][0] : class_instance["#{attr}_#{site_label}"] + elsif class_instance.respond_to?("#{attr}_#{NO_LANGUAGE_SUFFIX}") && class_instance["#{attr}_#{NO_LANGUAGE_SUFFIX}"] + class_instance[attr] = is_single ? class_instance["#{attr}_#{NO_LANGUAGE_SUFFIX}"][0] : class_instance["#{attr}_#{NO_LANGUAGE_SUFFIX}"] + elsif is_single + class_instance[attr] = class_instance[attr][0] + end end end end + def filter_language_attributes(params, class_instance) + filter_language_attribute(params, class_instance, MATCH_TYPE_PREFLABEL, true) + filter_language_attribute(params, class_instance, MATCH_TYPE_SYNONYM, false) + filter_language_attribute(params, class_instance, MATCH_TYPE_DEFINITION, false) + end + def validate_params_solr_population(allowed_includes_params) leftover = includes_param - allowed_includes_params invalid = leftover.length > 0 diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 54165b74..99134eed 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -399,6 +399,50 @@ def test_search_short_id end end + def test_language_attribute_filter + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" + results = MultiJson.load(last_response.body) + assert last_response.ok? + assert_equal 1, results["collection"].size + doc = results["collection"][0] + pref_label = doc["prefLabel"].kind_of?(Array) ? doc["prefLabel"].first : doc["prefLabel"] + assert_equal "Activité", pref_label + assert_equal 1, doc["definition"].size + assert 1, doc["definition"][0].include?("d’intérêt pouvant") + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" + results = MultiJson.load(last_response.body) + assert last_response.ok? + assert_equal 1, results["collection"].size + doc = results["collection"][0] + pref_label = doc["prefLabel"].kind_of?(Array) ? doc["prefLabel"].first : doc["prefLabel"] + assert_equal "ActivityEnglish", pref_label + assert_equal 1, doc["definition"].size + assert 1, doc["definition"][0].include?("Activity of interest that may be related to") + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0" + results = MultiJson.load(last_response.body) + assert last_response.ok? + assert_equal 1, results["collection"].size + doc = results["collection"][0] + pref_label = doc["prefLabel"].kind_of?(Array) ? doc["prefLabel"].first : doc["prefLabel"] + assert_equal "ActivityEnglish", pref_label + assert_equal 1, doc["definition"].size + assert 1, doc["definition"][0].include?("Activity of interest that may be related to") + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=all" + results = MultiJson.load(last_response.body) + assert last_response.ok? + assert_equal 1, results["collection"].size + doc = results["collection"][0] + assert doc["prefLabel"].kind_of?(Array) + assert_equal 3, doc["prefLabel"].size + assert doc["synonym"].kind_of?(Array) + assert_equal 1, doc["synonym"].size + assert doc["definition"].kind_of?(Array) + assert_equal 2, doc["definition"].size + end + def test_multilingual_search get "/search?q=Activity&ontologies=BROSEARCHTEST-0" res = MultiJson.load(last_response.body) @@ -416,14 +460,11 @@ def test_multilingual_search refute_equal 0, res["totalCount"] refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first - - get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" res = MultiJson.load(last_response.body) refute_equal 0, res["totalCount"] refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first - get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" res = MultiJson.load(last_response.body) assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first @@ -439,8 +480,6 @@ def test_multilingual_search get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" res = MultiJson.load(last_response.body) refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first - - end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index 7417f79c..33e91d79 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -634,7 +634,8 @@ Activity ActivityEnglish Activité - Activity of interest that may be related to a BRO:Resource. + Activity of interest that may be related to a BRO:Resource. + Activité d’intérêt pouvant être liée à un BRO:Resource. activities @@ -644,8 +645,10 @@ - Activity related to the creation, use, or maintenance of a biorepository (http://en.wikipedia.org/wiki/Biorepository) - Biospecimen Management + Activity related to the creation, use, or maintenance of a biorepository (http://en.wikipedia.org/wiki/Biorepository) + Activité liée à la création, à l'utilisation ou à la maintenance d'un bioréférentiel (http://en.wikipedia.org/wiki/Biorepository) + Biospecimen Management + Gestion des échantillons biologiques From 52e11753efb372a5e4767627d427808aaecc61bf Mon Sep 17 00:00:00 2001 From: mdorf Date: Thu, 27 Jun 2024 15:22:47 -0700 Subject: [PATCH 31/46] Gemfile.lock update --- Gemfile.lock | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e73f0d7d..9b78743d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,8 +106,8 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) @@ -119,7 +119,7 @@ GEM bcrypt_pbkdf (1.1.1-x86_64-darwin) bigdecimal (1.4.2) builder (3.3.0) - capistrano (3.18.1) + capistrano (3.19.0) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -240,9 +240,8 @@ GEM mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - mutex_m (0.2.0) net-http-persistent (2.9.4) - net-imap (0.4.13) + net-imap (0.4.14) date net-protocol net-pop (0.1.2) @@ -257,7 +256,7 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.10.2) + newrelic_rpm (9.11.0) oj (3.16.1) omni_logger (0.1.4) logger @@ -313,7 +312,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.3.0) + rexml (3.3.1) strscan rsolr (2.6.0) builder (>= 2.1.2) @@ -364,16 +363,15 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.2) + sshkit (1.23.0) base64 - mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) strscan (3.1.0) systemu (2.6.5) temple (0.10.3) - tilt (2.3.0) + tilt (2.4.0) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) From 9ad0a8b6cad480ec4674eddf7d371217b26322ae Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 2 Jul 2024 17:24:10 -0700 Subject: [PATCH 32/46] update documentation: by default, search in English --- views/documentation/documentation.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/documentation/documentation.haml b/views/documentation/documentation.haml index 916eb1e7..570cf23b 100644 --- a/views/documentation/documentation.haml +++ b/views/documentation/documentation.haml @@ -151,7 +151,7 @@ %li include={prefLabel, synonym, definition, notation, cui, semanticType} // default = (see Common Parameters section) %li page={integer representing the page number} // default = 1 %li pagesize={integer representing the size of the returned page} // default = 50 - %li language={an ISO 639-1 language value, e.g 'fr' or 'en'} // by default search in all languages + %li language={an ISO 639-1 language value, e.g 'fr' or 'en'} // by default search in English %h4#nav_search_subtree Subtree Search From 1b323bcc4f0eb0cefdc2224a3335013d3c9d99f1 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 22 Jul 2024 21:18:17 +0200 Subject: [PATCH 33/46] fix annotator attributes languague filter --- helpers/search_helper.rb | 2 +- test/controllers/test_annotator_controller.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 4e2dee3c..80eec4cc 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -357,7 +357,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) doc[:submission] = old_class.submission doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = LinkedData::Models::Class.read_only(doc) - instance.prefLabel = instance.prefLabel.first if instance.prefLabel.is_a?(Array) + filter_language_attributes(@params, instance) classes_hash[ont_uri_class_uri] = instance end diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 55d8fb66..53d7bb8a 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -267,14 +267,14 @@ def test_default_properties_output assert_equal 9, annotations.length annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].first.downcase <=> b["annotatedClass"]["prefLabel"].first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] - assert_equal "Aggregate Human Data", Array(annotations.first["annotatedClass"]["prefLabel"]).first + assert_equal "Aggregate Human Data", annotations.first["annotatedClass"]["prefLabel"] params = {text: text, include: "prefLabel,definition"} get "/annotator", params assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| Array(a["annotatedClass"]["prefLabel"]).first.downcase <=> Array(b["annotatedClass"]["prefLabel"]).first.downcase } + annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] assert_equal ["A resource that provides data from clinical care that comprises combined data from multiple individual human subjects."], annotations.first["annotatedClass"]["definition"] end From 27bc7cb360b49981fb10b83c1910b886733f0f77 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Mon, 23 Sep 2024 23:21:38 -0700 Subject: [PATCH 34/46] =?UTF-8?q?Update=20Ruby:=20v2.7=20=E2=86=92=20v3.0?= =?UTF-8?q?=20(#159)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ruby v3.0 update * add rexml gem to Gemlock. unicorn fails to start under ruby 3 without rexml gem in the Gemfile; needs to be further investigated --- .github/workflows/ruby-unit-tests.yml | 4 +- .ruby-version | 2 +- Dockerfile | 7 +- Gemfile | 2 + Gemfile.lock | 129 +++++++++++++------------- docker-compose.yml | 4 +- 6 files changed, 72 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index d764159b..2aac1468 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -14,13 +14,13 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build docker-compose - run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step + run: docker compose --profile 4store build #profile flag is set in order to build all containers in this step - name: Run unit tests # unit tests are run inside a container # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' + docker compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 with: diff --git a/.ruby-version b/.ruby-version index 6a81b4c8..818bd47a 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.7.8 +3.0.6 diff --git a/Dockerfile b/Dockerfile index a7adf16c..b6130644 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION +ARG RUBY_VERSION=3.0 ARG DISTRO_NAME=bullseye FROM ruby:$RUBY_VERSION-$DISTRO_NAME @@ -14,10 +14,7 @@ COPY Gemfile* /srv/ontoportal/ontologies_api/ WORKDIR /srv/ontoportal/ontologies_api -# set rubygem and bundler to the last version supported by ruby 2.7 -# remove version after ruby v3 upgrade -RUN gem update --system '3.4.22' -RUN gem install bundler -v 2.4.22 +RUN gem update --system RUN gem install bundler ENV BUNDLE_PATH=/srv/ontoportal/bundle RUN bundle install diff --git a/Gemfile b/Gemfile index 5e5c0c8b..e050a768 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,7 @@ gem 'oj', '~> 3.0' gem 'parseconfig' gem 'rack' gem 'rake', '~> 10.0' +gem 'rexml' # Investigate why unicorn fails to start under ruby 3 without adding rexml gem to the Gemfile gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' @@ -77,4 +78,5 @@ group :test do gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io gem 'webmock', '~> 3.19.1' + gem 'webrick' end diff --git a/Gemfile.lock b/Gemfile.lock index 17ce1495..985a1127 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e + revision: 2fb892f92f66b876b9793944124f72821f3b3eb8 branch: develop specs: ncbo_annotator (0.0.1) @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 1fbfc8f7f3ca62de35912ec09f23cac72fea1550 + revision: 3bb491139267efc37d3b752888824ca3dd976e85 branch: develop specs: ncbo_cron (0.0.1) @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 013a322440b4cfd899908d15ff605799acf8e6ef + revision: c30887ee907acd39500cae6c12f45edb06fe0802 branch: develop specs: ontologies_linked_data (0.0.1) @@ -108,7 +108,7 @@ GEM multi_json (~> 1.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - airbrussh (1.5.2) + airbrussh (1.5.3) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) backports (3.25.0) @@ -116,7 +116,6 @@ GEM bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) - bcrypt_pbkdf (1.1.1-x86_64-darwin) bigdecimal (1.4.2) builder (3.3.0) capistrano (3.19.1) @@ -124,7 +123,7 @@ GEM i18n rake (>= 10.0.0) sshkit (>= 1.9.0) - capistrano-bundler (2.1.0) + capistrano-bundler (2.1.1) capistrano (~> 3.1) capistrano-locally (0.3.0) capistrano (~> 3.0) @@ -132,7 +131,7 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) crack (1.0.0) bigdecimal @@ -140,53 +139,52 @@ GEM cube-ruby (0.0.3) dante (0.2.0) date (3.3.4) - docile (1.4.0) + docile (1.4.1) domain_name (0.6.20240107) ed25519 (1.3.0) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) + faraday (2.12.0) + faraday-net_http (>= 2.0, < 3.4) + json + logger faraday-follow_redirects (0.3.0) faraday (>= 1, < 3) - faraday-net_http (3.0.2) + faraday-net_http (3.3.0) + net-http faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.17.0-aarch64-linux-gnu) ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86_64-darwin) ffi (1.17.0-x86_64-linux-gnu) - gapic-common (0.21.1) + gapic-common (0.22.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) - google-protobuf (~> 3.18) - googleapis-common-protos (>= 1.4.0, < 2.a) - googleapis-common-protos-types (>= 1.11.0, < 2.a) - googleauth (~> 1.9) - grpc (~> 1.59) + google-protobuf (>= 3.25, < 5.a) + googleapis-common-protos (~> 1.6) + googleapis-common-protos-types (~> 1.15) + googleauth (~> 1.11) + grpc (~> 1.65) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.6.0) + google-analytics-data (0.6.1) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.12.0) + google-analytics-data-v1beta (0.13.1) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) - google-cloud-core (1.7.0) + google-cloud-core (1.7.1) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.1.1) + google-cloud-env (2.2.0) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3-aarch64-linux) - google-protobuf (3.25.3-arm64-darwin) - google-protobuf (3.25.3-x86_64-darwin) - google-protobuf (3.25.3-x86_64-linux) + google-protobuf (3.25.5-aarch64-linux) + google-protobuf (3.25.5-arm64-darwin) + google-protobuf (3.25.5-x86_64-linux) googleapis-common-protos (1.6.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.15.0) + googleapis-common-protos-types (1.16.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.11.0) faraday (>= 1.0, < 3.a) @@ -195,24 +193,21 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.64.0-aarch64-linux) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.64.0-arm64-darwin) - google-protobuf (~> 3.25) + grpc (1.66.0-aarch64-linux) + google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.64.0-x86_64-darwin) - google-protobuf (~> 3.25) + grpc (1.66.0-arm64-darwin) + google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.64.0-x86_64-linux) - google-protobuf (~> 3.25) + grpc (1.66.0-x86_64-linux) + google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt - hashdiff (1.1.0) + hashdiff (1.1.1) http-accept (1.7.0) - http-cookie (1.0.6) + http-cookie (1.0.7) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) @@ -220,12 +215,12 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.7.2) - jwt (2.8.2) + jwt (2.9.0) base64 kgio (2.11.4) language_server-protocol (3.17.0.3) libxml-ruby (5.0.3) - logger (1.6.0) + logger (1.6.1) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -236,15 +231,17 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0702) + mime-types-data (3.2024.0903) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) + net-http (0.4.1) + uri net-http-persistent (2.9.4) - net-imap (0.4.14) + net-imap (0.4.16) date net-protocol net-pop (0.1.2) @@ -259,14 +256,15 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.11.0) + newrelic_rpm (9.13.0) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) - parallel (1.25.1) + ostruct (0.6.0) + parallel (1.26.3) parseconfig (1.1.2) - parser (3.3.3.0) + parser (3.3.5.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -274,8 +272,8 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.1.1) - racc (1.8.0) + public_suffix (6.0.1) + racc (1.8.1) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -298,14 +296,14 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.2.0) + redis (5.3.0) redis-client (>= 0.22.0) redis-client (0.22.2) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.10.0) + redis-store (1.11.0) redis (>= 4, < 6) regexp_parser (2.9.2) request_store (1.7.0) @@ -315,27 +313,24 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.3.1) - strscan + rexml (3.3.7) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.64.1) + rubocop (1.66.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.4, < 3.0) + rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.3) + rubocop-ast (1.32.3) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) - ruby2_keywords (0.0.5) rubyzip (2.3.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) @@ -351,7 +346,7 @@ GEM simplecov-cobertura (2.1.0) rexml simplecov (~> 0.19) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) sinatra (1.4.8) rack (~> 1.5) @@ -366,37 +361,37 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.23.0) + sshkit (1.23.1) base64 net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) - strscan (3.1.0) + ostruct systemu (2.6.5) temple (0.10.3) tilt (2.4.0) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) + unicode-display_width (2.6.0) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) unicorn-worker-killer (0.4.5) get_process_mem (~> 0) unicorn (>= 4, < 7) + uri (0.13.1) uuid (2.3.9) macaddr (~> 1.0) webmock (3.19.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + webrick (1.8.1) PLATFORMS aarch64-linux arm64-darwin-22 - arm64-darwin-23 - x86_64-darwin-23 x86_64-linux DEPENDENCIES @@ -437,6 +432,7 @@ DEPENDENCIES redis redis-rack-cache (~> 2.0) redis-store (~> 1.10) + rexml rubocop shotgun! simplecov @@ -448,6 +444,7 @@ DEPENDENCIES unicorn unicorn-worker-killer webmock (~> 3.19.1) + webrick BUNDLED WITH - 2.4.22 + 2.5.19 diff --git a/docker-compose.yml b/docker-compose.yml index 20934d3d..11811fe7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,9 @@ x-app: &app build: context: . args: - RUBY_VERSION: '2.7' + RUBY_VERSION: '3.0' # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_api:0.0.4 + image: ontologies_api:0.0.5 environment: &env BUNDLE_PATH: /srv/ontoportal/bundle # default bundle config resolves to /usr/local/bundle/config inside of the container From 81ca0337e6b2bf5e9f83a05edd3d75d604f88e5e Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Wed, 25 Sep 2024 14:04:16 -0700 Subject: [PATCH 35/46] reset branch specifier to master --- Gemfile | 12 ++++++------ Gemfile.lock | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Gemfile b/Gemfile index e050a768..a3bcd66b 100644 --- a/Gemfile +++ b/Gemfile @@ -43,12 +43,12 @@ gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'master' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 985a1127..9c5d6f8d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 70007faf43d0d96292809f89f15a27a757ccaa25 - branch: develop + revision: 2cec53c2d84179171def81d7dc0a18e462e78fc2 + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -16,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 2fb892f92f66b876b9793944124f72821f3b3eb8 - branch: develop + revision: 63c986880aa88c9384043e6611a682434a14aba7 + branch: master specs: ncbo_annotator (0.0.1) goo @@ -27,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 3bb491139267efc37d3b752888824ca3dd976e85 - branch: develop + revision: 15ccdaa2835be9f35f9c3afdcc4ed5bf5d130999 + branch: master specs: ncbo_cron (0.0.1) dante @@ -45,8 +45,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: f92a42f660635522eb8709e618ff2e641aef0d17 - branch: develop + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master specs: ncbo_ontology_recommender (0.0.1) goo @@ -56,8 +56,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: c30887ee907acd39500cae6c12f45edb06fe0802 - branch: develop + revision: a9da0982eef16ea8272002d14d6b8851dc07fb0a + branch: master specs: ontologies_linked_data (0.0.1) activesupport @@ -76,8 +76,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc - branch: develop + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc + branch: master specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -215,7 +215,7 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.7.2) - jwt (2.9.0) + jwt (2.9.1) base64 kgio (2.11.4) language_server-protocol (3.17.0.3) @@ -387,7 +387,7 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.8.1) + webrick (1.8.2) PLATFORMS aarch64-linux From 16d50db896305fd5ce1f2adc2c0ef2ebced75055 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Wed, 25 Sep 2024 21:36:21 -0700 Subject: [PATCH 36/46] replace delete_if with select to avoid modifying the original array, which prevents issues with frozen arrays, fixes #161 "FrozenError - can't modify frozen Array" --- helpers/slices_helper.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/helpers/slices_helper.rb b/helpers/slices_helper.rb index d17db53e..6f527f29 100644 --- a/helpers/slices_helper.rb +++ b/helpers/slices_helper.rb @@ -10,10 +10,8 @@ def filter_for_slice(obj) slice = current_slice() - if obj.is_a?(Enumerable) - if obj.first.is_a?(LinkedData::Models::Ontology) - obj.delete_if {|o| !slice.ontology_id_set.include?(o.id.to_s)} - end + if obj.is_a?(Enumerable) && obj.first.is_a?(LinkedData::Models::Ontology) + obj = obj.select { |o| slice.ontology_id_set.include?(o.id.to_s) } end obj end @@ -35,4 +33,4 @@ def current_slice_acronyms end end -helpers Sinatra::Helpers::SlicesHelper \ No newline at end of file +helpers Sinatra::Helpers::SlicesHelper From f98beadeecea22463590f460145356f232543736 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Thu, 26 Sep 2024 20:32:53 -0700 Subject: [PATCH 37/46] replace delete_if with select to avoid modifying the original array, which prevents issues with frozen arrays, fixes #161 "FrozenError - can't modify frozen Array" (part 2) --- helpers/application_helper.rb | 8 ++++---- helpers/users_helper.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 04999f65..ad813e31 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -125,10 +125,10 @@ def reply(*response) check_access(obj) if LinkedData.settings.enable_security # Slice or set check - filter_for_slice(obj) if LinkedData.settings.enable_slices + obj = filter_for_slice(obj) if LinkedData.settings.enable_slices # Check for custom ontologies set by user - filter_for_user_onts(obj) + obj = filter_for_user_onts(obj) LinkedData::Serializer.build_response(@env, status: status, ld_object: obj) end @@ -211,8 +211,8 @@ def restricted_ontologies(params=nil) found_onts = onts.length > 0 - filter_for_slice(onts) - filter_for_user_onts(onts) + onts = filter_for_slice(onts) + onts = filter_for_user_onts(onts) end onts = filter_access(onts) diff --git a/helpers/users_helper.rb b/helpers/users_helper.rb index 5d4266c1..eeb6f800 100644 --- a/helpers/users_helper.rb +++ b/helpers/users_helper.rb @@ -12,7 +12,7 @@ def filter_for_user_onts(obj) user = env["REMOTE_USER"] if obj.first.is_a?(LinkedData::Models::Ontology) - obj.delete_if {|o| !user.custom_ontology_id_set.include?(o.id.to_s)} + obj = obj.select {|o| user.custom_ontology_id_set.include?(o.id.to_s)} end obj @@ -21,4 +21,4 @@ def filter_for_user_onts(obj) end end -helpers Sinatra::Helpers::UsersHelper \ No newline at end of file +helpers Sinatra::Helpers::UsersHelper From af63f6a6d89186fedca97c40829536a8df4fd721 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 27 Sep 2024 19:11:42 -0700 Subject: [PATCH 38/46] deprecate recommender_v1 endpoint --- controllers/recommender_v1_controller.rb | 11 +----- .../test_recommender_v1_controller.rb | 38 +++---------------- 2 files changed, 7 insertions(+), 42 deletions(-) diff --git a/controllers/recommender_v1_controller.rb b/controllers/recommender_v1_controller.rb index 5b0123a9..29991299 100644 --- a/controllers/recommender_v1_controller.rb +++ b/controllers/recommender_v1_controller.rb @@ -1,17 +1,10 @@ class RecommenderController < ApplicationController namespace "/recommender_v1" do - # execute an annotator query + # Mark this route as deprecated get do - text = params["text"] - raise error 400, "A text to be analyzed by the recommender must be supplied using the argument text=" if text.nil? || text.strip.empty? - acronyms = restricted_ontologies_to_acronyms(params) - display_classes = params['display_classes'].eql?('true') # default will be false - recommender = Recommender::Models::NcboRecommender.new - recommendations = recommender.recommend(text, acronyms, display_classes) - reply 200, recommendations + reply 410, { message: "This API endpoint has been deprecated and is no longer available. Please use /recommender or refer to the API documentation for updated routes." } end end end - diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 3ac4862d..fe67ffa7 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -1,40 +1,12 @@ require_relative '../test_case' - +# recommender_v1 is deprecated as of 2024-10-27 +# TODO: remove completely after 2025-10-27 class TestRecommenderV1Controller < TestCase - - def self.before_suite - LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) - @@text = < @@text + :text => 'recommender v1 is deprecated' } get "/recommender_v1", params - assert last_response.ok? - recommendations = MultiJson.load(last_response.body) - assert_instance_of(Array, recommendations) - assert_equal(3, recommendations.length, msg='Failed to return 3 recommendations') - rec = recommendations.first - assert_instance_of(Hash, rec) - ont_acronyms = @@ontologies.map {|o| o.bring(:acronym); o.acronym } - assert ont_acronyms.include? rec['ontology']['acronym'] - assert rec['annotatedClasses'].length == 0 # no classes requested - assert rec['numTermsMatched'] > 0 - assert rec['numTermsTotal'] > 0 - assert rec['numTermsTotal'] >= rec['numTermsMatched'] - assert recommendations[0]['score'].to_i >= recommendations[1]['score'].to_i - assert recommendations[1]['score'].to_i >= recommendations[2]['score'].to_i + assert_equal 410, last_response.status end - end From c6619cbe0ee0d2e90310685cba9122412e25a405 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 1 Oct 2024 11:21:31 -0700 Subject: [PATCH 39/46] Gemfile.lock update --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 132094bb..3f35f32b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: aaa3941b48edbc317ec42ba4ecb44c0defeccef6 + revision: 74a012eebb9433d031eb00df5abbe488cb8b4512 branch: develop specs: goo (0.0.2) @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: a22fae3150d8e84e3b6fd19223a5ce2111e8ce94 + revision: 8572cd25722cab2fbb73813d0fe011b5647d5067 branch: develop specs: ontologies_linked_data (0.0.1) @@ -265,7 +265,7 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.13.0) + newrelic_rpm (9.14.0) oj (3.16.6) bigdecimal (>= 3.0) ostruct (>= 0.2) @@ -324,7 +324,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.3.7) + rexml (3.3.8) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) From ea55c9e469676fea504e0f1739c31805ad6095cd Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 1 Oct 2024 11:49:18 -0700 Subject: [PATCH 40/46] set default ruby version for image builds in Dockerfile --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 358ff4f1..8c912cc7 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,8 +35,8 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - build-args: | - RUBY_VERSION=2.7 + # build-args: | + # RUBY_VERSION=2.7 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 09ca679401cfac1f0ec8a374fc1bcd4bb9e88abc Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 1 Oct 2024 11:49:18 -0700 Subject: [PATCH 41/46] set default ruby version for image builds in Dockerfile --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 358ff4f1..8c912cc7 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,8 +35,8 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - build-args: | - RUBY_VERSION=2.7 + # build-args: | + # RUBY_VERSION=2.7 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From d40541717e1ca172d0593d02d63ea82c501f6dde Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 2 Oct 2024 12:57:02 -0700 Subject: [PATCH 42/46] added a unit test for ncbo/ontologies_linked_data/pull#216 --- Gemfile.lock | 17 +++++++++-------- test/controllers/test_users_controller.rb | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3f35f32b..c12a226d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: cf66e7d076a94e6d669ffeb09c6bcd988226ece3 + revision: 9d1b6ba54019cac2eeeb5c173e452b4eebbeaa3a branch: develop specs: ncbo_cron (0.0.1) @@ -56,7 +56,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 8572cd25722cab2fbb73813d0fe011b5647d5067 + revision: 812dd78f02b77c9c6d579b0febf1c2a42d513022 branch: develop specs: ontologies_linked_data (0.0.1) @@ -154,10 +154,9 @@ GEM net-http faraday-retry (2.2.1) faraday (~> 2.0) - ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0) ffi (1.17.0-arm64-darwin) ffi (1.17.0-x86_64-darwin) - ffi (1.17.0-x86_64-linux-gnu) gapic-common (0.22.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) @@ -236,9 +235,10 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.5.2) + mime-types (3.6.0) + logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.0903) + mime-types-data (3.2024.1001) mini_mime (1.1.5) minitest (5.25.1) minitest-hooks (1.5.2) @@ -263,7 +263,7 @@ GEM net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.5.0) net-protocol - net-ssh (7.2.3) + net-ssh (7.3.0) netrc (0.11.0) newrelic_rpm (9.14.0) oj (3.16.6) @@ -404,6 +404,7 @@ GEM PLATFORMS aarch64-linux arm64-darwin-22 + arm64-darwin-23 x86_64-darwin-23 x86_64-linux @@ -462,4 +463,4 @@ DEPENDENCIES webrick BUNDLED WITH - 2.5.19 + 2.5.11 diff --git a/test/controllers/test_users_controller.rb b/test/controllers/test_users_controller.rb index 4a33a79a..1aea137a 100644 --- a/test/controllers/test_users_controller.rb +++ b/test/controllers/test_users_controller.rb @@ -48,6 +48,21 @@ def test_single_user assert_equal "fred", MultiJson.load(last_response.body)["username"] end + def test_hide_sensitive_data + user = @@users[0] + reset_token = token(36) + user.resetToken = reset_token + user.resetTokenExpireTime = Time.now.to_i - 2.hours.to_i + user.save + + username = user.username + get "/users/#{username}?display=resetToken,resetTokenExpireTime" + assert last_response.ok? + + refute_includes MultiJson.load(last_response.body), 'resetToken', "resetToken should NOT be included in the response" + refute_includes MultiJson.load(last_response.body), 'resetTokenExpireTime', "resetTokenExpireTime should NOT be included in the response" + end + def test_create_new_user user = {email: "#{@@username}@example.org", password: "pass_the_word"} put "/users/#{@@username}", MultiJson.dump(user), "CONTENT_TYPE" => "application/json" @@ -171,6 +186,13 @@ def test_authentication private + def token(len) + chars = ("a".."z").to_a + ("A".."Z").to_a + ("1".."9").to_a + token = "" + 1.upto(len) { |i| token << chars[rand(chars.size-1)] } + token + end + def _delete_user(username) LinkedData::Models::User.find(@@username).first&.delete end From 720c900d86a695ac36338ca02f691e4d70a3adc5 Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 2 Oct 2024 14:18:39 -0700 Subject: [PATCH 43/46] fixed reset_password endpoint after committing PR ncbo/ontologies_linked_data/pull#216 --- controllers/users_controller.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index fdad6a74..a25c1b5d 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -46,14 +46,24 @@ class UsersController < ApplicationController params["display"] = User.attributes.join(",") # used to serialize everything via the serializer user = LinkedData::Models::User.where(email: email, username: username).include(User.goo_attrs_to_load(includes_param)).first error 404, "User not found" unless user + + user.bring(:resetToken) + user.bring(:resetTokenExpireTime) + user.bring(:passwordHash) + if token.eql?(user.resetToken) error 401, "Invalid password reset token" if user.resetTokenExpireTime.nil? error 401, "The password reset token expired" if user.resetTokenExpireTime < Time.now.to_i user.resetToken = nil user.resetTokenExpireTime = nil - user.save(override_security: true) if user.valid? - user.show_apikey = true - reply user + + if user.valid? + user.save(override_security: true) + user.show_apikey = true + reply user + else + error 422, "Error resetting password" + end else error 401, "Password reset not authorized with this token" end From a84633b0f9432ba316a4c5b572978b328ce05912 Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 2 Oct 2024 14:18:39 -0700 Subject: [PATCH 44/46] fixed reset_password endpoint after committing PR https://github.com/ncbo/ontologies_linked_data/pull/216 --- controllers/users_controller.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/controllers/users_controller.rb b/controllers/users_controller.rb index fdad6a74..a25c1b5d 100644 --- a/controllers/users_controller.rb +++ b/controllers/users_controller.rb @@ -46,14 +46,24 @@ class UsersController < ApplicationController params["display"] = User.attributes.join(",") # used to serialize everything via the serializer user = LinkedData::Models::User.where(email: email, username: username).include(User.goo_attrs_to_load(includes_param)).first error 404, "User not found" unless user + + user.bring(:resetToken) + user.bring(:resetTokenExpireTime) + user.bring(:passwordHash) + if token.eql?(user.resetToken) error 401, "Invalid password reset token" if user.resetTokenExpireTime.nil? error 401, "The password reset token expired" if user.resetTokenExpireTime < Time.now.to_i user.resetToken = nil user.resetTokenExpireTime = nil - user.save(override_security: true) if user.valid? - user.show_apikey = true - reply user + + if user.valid? + user.save(override_security: true) + user.show_apikey = true + reply user + else + error 422, "Error resetting password" + end else error 401, "Password reset not authorized with this token" end From ebf586cb12b2c9c4b8b7ca5d8fa4362ec6887bcd Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Mon, 7 Oct 2024 19:51:17 -0700 Subject: [PATCH 45/46] gemfile update --- Gemfile.lock | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 827a8843..c15eb4af 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -117,6 +117,7 @@ GEM base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) + bcrypt_pbkdf (1.1.1-arm64-darwin) bigdecimal (3.1.8) builder (3.3.0) capistrano (3.19.1) @@ -153,6 +154,8 @@ GEM faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-linux-gnu) gapic-common (0.22.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) @@ -176,6 +179,8 @@ GEM faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) google-protobuf (3.25.5-aarch64-linux) + google-protobuf (3.25.5-arm64-darwin) + google-protobuf (3.25.5-x86_64-linux) googleapis-common-protos (1.6.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) @@ -192,6 +197,12 @@ GEM grpc (1.66.0-aarch64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) + grpc (1.66.0-arm64-darwin) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) + grpc (1.66.0-x86_64-linux) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt @@ -387,6 +398,8 @@ GEM PLATFORMS aarch64-linux + arm64-darwin-22 + x86_64-linux DEPENDENCIES activesupport (~> 5) @@ -443,4 +456,4 @@ DEPENDENCIES webrick BUNDLED WITH - 2.5.7 + 2.5.20 From 581eb6905987e2c3f027baba0cde484748b12a9f Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Mon, 7 Oct 2024 20:55:09 -0700 Subject: [PATCH 46/46] reset branch specifier to master --- Gemfile | 12 ++++++------ Gemfile.lock | 26 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Gemfile b/Gemfile index c836033a..806fbf75 100644 --- a/Gemfile +++ b/Gemfile @@ -43,12 +43,12 @@ gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'master' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index c15eb4af..c21d2c66 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 74a012eebb9433d031eb00df5abbe488cb8b4512 - branch: develop + revision: f943ac33e87c55c88131c32d826ed5d39c89302c + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -16,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: b2731c98bfef0958fecb440c8648b7755a670895 - branch: develop + revision: d46d667ec1b2343fbcdab587f1a70e763b894133 + branch: master specs: ncbo_annotator (0.0.1) goo @@ -27,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: b279f9caa9a6a169bad7350890d7015d56e9e9b2 - branch: develop + revision: 32a721543f518167385256fa4f0a08c8c6d987b7 + branch: master specs: ncbo_cron (0.0.1) dante @@ -45,8 +45,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 084cfc74a78b1c6334f3b74e228ddbf80d36e30a - branch: develop + revision: 9dbd4f179e42c52095129d353a5ac584e9bd47f3 + branch: master specs: ncbo_ontology_recommender (0.0.1) goo @@ -56,8 +56,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 812dd78f02b77c9c6d579b0febf1c2a42d513022 - branch: develop + revision: 9ab4be437ed84f5a480e7f0d8799824fcea310ae + branch: master specs: ontologies_linked_data (0.0.1) activesupport @@ -76,8 +76,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc - branch: develop + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc + branch: master specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -456,4 +456,4 @@ DEPENDENCIES webrick BUNDLED WITH - 2.5.20 + 2.5.7