From 014eb3dc1b6270642ff189130c82f608f241f1cf Mon Sep 17 00:00:00 2001 From: Bilel Kihal <61744974+Bilelkihal@users.noreply.github.com> Date: Thu, 1 Aug 2024 20:25:08 +0200 Subject: [PATCH 1/2] Feature: update agents search endpoint to add option to have a custom qf paramter (#90) * fix agents search sensibility * improve agents search endpoint to search only exact string or substring match * make the agent search endpoint query filter configurable --------- Co-authored-by: Syphax --- controllers/search_controller.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index cf2d76c6..ce34d51d 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -63,7 +63,7 @@ class SearchController < ApplicationController page_size: page_size, sort: sort }) - + total_found = page_data.aggregate ontology_rank = LinkedData::Models::Ontology.rank docs = {} @@ -153,11 +153,17 @@ class SearchController < ApplicationController fq = "agentType_t:#{type}" if type - qf = [ - "acronymSuggestEdge^25 nameSuggestEdge^15 emailSuggestEdge^15 identifiersSuggestEdge^10 ", # start of the word first - "identifiers_texts^20 acronym_text^15 name_text^10 email_text^10 ", # full word match - "acronymSuggestNgram^2 nameSuggestNgram^1.5 email_text^1" # substring match last - ].join(' ') + if params[:qf] + qf = params[:qf] + else + qf = [ + "acronymSuggestEdge^25 nameSuggestEdge^15 emailSuggestEdge^15 identifiersSuggestEdge^10 ", # start of the word first + "identifiers_texts^20 acronym_text^15 name_text^10 email_text^10 ", # full word match + "acronymSuggestNgram^2 nameSuggestNgram^1.5 email_text^1" # substring match last + ].join(' ') + end + + if params[:sort] sort = "#{params[:sort]} asc, score desc" From fc11608eec406707314c620900022489e52ed46b Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Thu, 1 Aug 2024 23:38:04 +0200 Subject: [PATCH 2/2] Fix: hide duplicated agents endpoint ('/Agents') (#91) * remove duplicated agents endpoint ('/Agents') * put again the Agents endpoint * hide Agents endpoint in the home endpoint * fix properties tests --------- Co-authored-by: Bilel KIHAL --- Gemfile.lock | 47 +++--- controllers/agents_controller.rb | 219 ++++++++++++++------------- controllers/home_controller.rb | 4 + controllers/properties_controller.rb | 12 +- docker-compose.yml | 3 +- 5 files changed, 144 insertions(+), 141 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ccb0b5ee..ce2a0b4f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: b2a635fb1e8206e6e3010be4dbe033b47eb58481 + revision: a95245b8c964431505ca6315907440996c59a00d branch: development specs: goo (0.0.2) @@ -40,7 +40,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: 6bb53a13f514a60513afe25e37c5c69475140452 + revision: fabd04ef4fa37989d526fc6a7aa1e98830008dae branch: master specs: ncbo_cron (0.0.1) @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: fd78d689dac4a7393e20a36ac930c6c9d191a619 + revision: ca79d5a84a3b6e961118b7e1062f082e7f7b99fc branch: development specs: ontologies_linked_data (0.0.1) @@ -117,7 +117,7 @@ GEM bcrypt_pbkdf (1.1.1-x86_64-darwin) bigdecimal (1.4.2) builder (3.3.0) - capistrano (3.19.0) + capistrano (3.19.1) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -139,7 +139,7 @@ GEM dante (0.2.0) date (3.3.4) declarative (0.0.20) - docile (1.4.0) + docile (1.4.1) domain_name (0.6.20240107) ed25519 (1.3.0) faraday (1.10.3) @@ -160,12 +160,12 @@ GEM faraday-httpclient (1.0.1) faraday-multipart (1.0.4) multipart-post (~> 2) - faraday-net_http (1.0.1) + faraday-net_http (1.0.2) faraday-net_http_persistent (1.2.0) faraday-patron (1.0.0) faraday-rack (1.0.0) faraday-retry (1.0.3) - ffi (1.16.3) + ffi (1.17.0) gapic-common (0.21.1) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) @@ -184,23 +184,21 @@ GEM google-cloud-errors (~> 1.0) google-apis-analytics_v3 (0.16.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.15.0) + google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) - httpclient (>= 2.8.1, < 3.a) + httpclient (>= 2.8.3, < 3.a) mini_mime (~> 1.0) + mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - rexml google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3) - google-protobuf (3.25.3-x86_64-darwin) - google-protobuf (3.25.3-x86_64-linux) + google-protobuf (3.25.4) googleapis-common-protos (1.6.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) @@ -214,14 +212,11 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.64.0) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.64.0-x86_64-darwin) - google-protobuf (~> 3.25) + grpc (1.65.2-x86_64-darwin) + google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.64.0-x86_64-linux) - google-protobuf (~> 3.25) + grpc (1.65.2-x86_64-linux) + google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) @@ -256,7 +251,7 @@ GEM method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0604) + mime-types-data (3.2024.0702) mini_mime (1.1.5) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -264,6 +259,7 @@ GEM redis multi_json (1.15.0) multipart-post (2.4.1) + mutex_m (0.2.0) net-http-persistent (4.0.2) connection_pool (~> 2.2) net-imap (0.4.14) @@ -281,7 +277,7 @@ GEM net-protocol net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.11.0) + newrelic_rpm (9.12.0) oj (3.16.1) omni_logger (0.1.4) logger @@ -349,7 +345,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.3.1) + rexml (3.3.4) strscan rsolr (2.6.0) builder (>= 2.1.2) @@ -394,7 +390,7 @@ GEM strscan (3.1.0) systemu (2.6.5) temple (0.10.3) - tilt (2.3.0) + tilt (2.4.0) timeout (0.4.1) trailblazer-option (0.1.2) tzinfo (2.0.6) @@ -414,7 +410,6 @@ GEM hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - ruby x86_64-darwin-23 x86_64-linux @@ -473,4 +468,4 @@ DEPENDENCIES webmock (~> 3.19.1) BUNDLED WITH - 2.4.22 + 2.3.23 diff --git a/controllers/agents_controller.rb b/controllers/agents_controller.rb index 6b69fbc5..0b47c0c2 100644 --- a/controllers/agents_controller.rb +++ b/controllers/agents_controller.rb @@ -22,147 +22,150 @@ class AgentsController < ApplicationController reply agents end - namespace "/agents" do - get do - check_last_modified_collection(LinkedData::Models::Agent) - query = LinkedData::Models::Agent.where - query = apply_filters(LinkedData::Models::Agent, query) - query = query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)) - if page? - page, size = page_params - agents = query.page(page, size).all - else - agents = query.to_a - end + %w[agents Agents].each do |namespace| + namespace "/#{namespace}" do + get do + check_last_modified_collection(LinkedData::Models::Agent) + query = LinkedData::Models::Agent.where + query = apply_filters(LinkedData::Models::Agent, query) + query = query.include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)) + if page? + page, size = page_params + agents = query.page(page, size).all + else + agents = query.to_a + end + + if includes_param.include?(:all) || includes_param.include?(:usages) + LinkedData::Models::Agent.load_agents_usages(agents) + end - if includes_param.include?(:all) || includes_param.include?(:usages) - LinkedData::Models::Agent.load_agents_usages(agents) + reply agents end - reply agents - end + # Display a single agent + get '/:id' do + check_last_modified_collection(LinkedData::Models::Agent) + id = params["id"] + agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first + error 404, "Agent #{id} not found" if agent.nil? + reply 200, agent + end - # Display a single agent - get '/:id' do - check_last_modified_collection(LinkedData::Models::Agent) - id = params["id"] - agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first - error 404, "Agent #{id} not found" if agent.nil? - reply 200, agent - end + # Create a agent with the given acronym + post do + reply 201, create_new_agent + end - # Create a agent with the given acronym - post do - reply 201, create_new_agent - end + # Create a agent with the given acronym + put '/:acronym' do + reply 201, create_new_agent + end - # Create a agent with the given acronym - put '/:acronym' do - reply 201, create_new_agent - end + # Update an existing submission of a agent + patch '/:id' do + acronym = params["id"] + agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.attributes).first - # Update an existing submission of a agent - patch '/:id' do - acronym = params["id"] - agent = LinkedData::Models::Agent.find(acronym).include(LinkedData::Models::Agent.attributes).first + if agent.nil? + error 400, "Agent does not exist, please create using HTTP PUT before modifying" + else + agent = update_agent(agent, params) - if agent.nil? - error 400, "Agent does not exist, please create using HTTP PUT before modifying" - else - agent = update_agent(agent, params) + error 400, agent.errors unless agent.errors.empty? + end + halt 204 + end - error 400, agent.errors unless agent.errors.empty? + # Delete a agent + delete '/:id' do + agent = LinkedData::Models::Agent.find(params["id"]).first + agent.delete + halt 204 end - halt 204 - end - # Delete a agent - delete '/:id' do - agent = LinkedData::Models::Agent.find(params["id"]).first - agent.delete - halt 204 - end + private - private + def update_identifiers(identifiers) + Array(identifiers).map do |i| + next nil if i.empty? - def update_identifiers(identifiers) - Array(identifiers).map do |i| - next nil if i.empty? + id = i["id"] || LinkedData::Models::AgentIdentifier.generate_identifier(i['notation'], i['schemaAgency']) + identifier = LinkedData::Models::AgentIdentifier.find(RDF::URI.new(id)).first - id = i["id"] || LinkedData::Models::AgentIdentifier.generate_identifier(i['notation'], i['schemaAgency']) - identifier = LinkedData::Models::AgentIdentifier.find(RDF::URI.new(id)).first + if identifier + identifier.bring_remaining + else + identifier = LinkedData::Models::AgentIdentifier.new + end - if identifier - identifier.bring_remaining - else - identifier = LinkedData::Models::AgentIdentifier.new - end + i.delete "id" - i.delete "id" + next identifier if i.keys.size.zero? - next identifier if i.keys.size.zero? + populate_from_params(identifier, i) - populate_from_params(identifier, i) + if identifier.valid? + identifier.save + else + error 400, identifier.errors + end + identifier + end.compact + end - if identifier.valid? - identifier.save - else - error 400, identifier.errors - end - identifier - end.compact - end + def update_affiliations(affiliations) + Array(affiliations).map do |aff| + affiliation = aff["id"] ? LinkedData::Models::Agent.find(RDF::URI.new(aff["id"])).first : nil - def update_affiliations(affiliations) - Array(affiliations).map do |aff| - affiliation = aff["id"] ? LinkedData::Models::Agent.find(RDF::URI.new(aff["id"])).first : nil + if affiliation + affiliation.bring_remaining + affiliation.identifiers.each{|i| i.bring_remaining} + end - if affiliation - affiliation.bring_remaining - affiliation.identifiers.each{|i| i.bring_remaining} - end + next affiliation if aff.keys.size.eql?(1) && aff["id"] - next affiliation if aff.keys.size.eql?(1) && aff["id"] + if affiliation + affiliation = update_agent(affiliation, aff) + else + affiliation = create_new_agent(aff["id"], aff) + end - if affiliation - affiliation = update_agent(affiliation, aff) - else - affiliation = create_new_agent(aff["id"], aff) + error 400, affiliation.errors unless affiliation.errors.empty? + + affiliation end + end - error 400, affiliation.errors unless affiliation.errors.empty? + def create_new_agent (id = @params['id'], params = @params) + agent = nil + agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if id - affiliation + if agent.nil? + agent = update_agent(LinkedData::Models::Agent.new, params) + error 400, agent.errors unless agent.errors.empty? + + return agent + else + error 400, "Agent exists, please use HTTP PATCH to update" + end end - end - def create_new_agent (id = @params['id'], params = @params) - agent = nil - agent = LinkedData::Models::Agent.find(id).include(LinkedData::Models::Agent.goo_attrs_to_load(includes_param)).first if id + def update_agent(agent, params) + return agent unless agent - if agent.nil? - agent = update_agent(LinkedData::Models::Agent.new, params) - error 400, agent.errors unless agent.errors.empty? + identifiers = params.delete "identifiers" + affiliations = params.delete "affiliations" + params.delete "id" + populate_from_params(agent, params) + agent.identifiers = update_identifiers(identifiers) + agent.affiliations = update_affiliations(affiliations) + agent.save if agent.valid? return agent - else - error 400, "Agent exists, please use HTTP PATCH to update" end - end - def update_agent(agent, params) - return agent unless agent - - identifiers = params.delete "identifiers" - affiliations = params.delete "affiliations" - params.delete "id" - populate_from_params(agent, params) - agent.identifiers = update_identifiers(identifiers) - agent.affiliations = update_affiliations(affiliations) - - agent.save if agent.valid? - return agent end - end + end diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index c2a67fb4..a44fd22e 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -13,11 +13,15 @@ class HomeController < ApplicationController expires 3600, :public last_modified @@root_last_modified ||= Time.now.httpdate routes = routes_list + #TODO: delete when ccv will be on production routes.delete("/ccv") if LinkedData.settings.enable_resource_index == false routes.delete("/resource_index") end + + routes.delete('/Agents') + routes_hash = {} context = {} routes.each do |route| diff --git a/controllers/properties_controller.rb b/controllers/properties_controller.rb index f98e9016..d32180d5 100644 --- a/controllers/properties_controller.rb +++ b/controllers/properties_controller.rb @@ -24,7 +24,7 @@ class PropertiesController < ApplicationController get '/:property' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? reply 200, p end @@ -51,7 +51,7 @@ class PropertiesController < ApplicationController get '/:property/tree' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? root_tree = p.tree @@ -79,7 +79,7 @@ class PropertiesController < ApplicationController get '/:property/ancestors' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? ancestors = p.ancestors p.class.in(submission).models(ancestors).include(:label, :definition).all @@ -91,7 +91,7 @@ class PropertiesController < ApplicationController get '/:property/descendants' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? descendants = p.descendants p.class.in(submission).models(descendants).include(:label, :definition).all @@ -103,7 +103,7 @@ class PropertiesController < ApplicationController get '/:property/parents' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? p.bring(:parents) @@ -120,7 +120,7 @@ class PropertiesController < ApplicationController get '/:property/children' do prop = params[:property] ont, submission = get_ontology_and_submission - p = ont.property(prop, submission) + p = ont.property(prop, submission, display_all_attributes: false) error 404, "Property #{prop} not found in ontology #{ont.id.to_s}" if p.nil? p.bring(:children) diff --git a/docker-compose.yml b/docker-compose.yml index a75136d7..07b0cda1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: environment: <<: *env BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle + profiles: - 4store depends_on: @@ -48,7 +49,7 @@ services: - "9393:9393" volumes: # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development - - app_api:/srv/ontoportal/ontologies_api + - .:/srv/ontoportal/ontologies_api - repository:/srv/ontoportal/data/repository ncbo_cron: