diff --git a/Gemfile.lock b/Gemfile.lock index 46b2a605..33b4b6f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2 + revision: 1484ad56e39810208ee08a14d1319b6bc112f647 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: e9b708c40b2b22b935fb48d18ed19de8148fca35 + revision: f2168c8ad9ec447338da914d10a352df558494b5 branch: development specs: ontologies_linked_data (0.0.1) @@ -103,11 +103,11 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.1) + addressable (2.8.4) public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.23.0) + backports (3.24.1) bcrypt (3.1.18) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.3.0) + googleauth (1.5.2) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.16.0) + newrelic_rpm (9.2.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -249,7 +249,7 @@ GEM rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.0.0) + rack-mini-profiler (3.1.0) rack (>= 1.2.0) rack-protection (1.5.5) rack @@ -341,6 +341,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index cf55659d..8749786d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -1,9 +1,13 @@ class OntologySubmissionsController < ApplicationController get "/submissions" do check_last_modified_collection(LinkedData::Models::OntologySubmission) - #using appplication_helper method - options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")} - reply retrieve_latest_submissions(options).values + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_latest_submissions(options) + subs = subs.values unless page? + reply subs end ## diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 6c44f25f..84c26497 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -355,40 +355,18 @@ def replace_url_prefix(id) end def retrieve_latest_submissions(options = {}) - status = (options[:status] || "RDF").to_s.upcase - include_ready = status.eql?("READY") ? true : false - status = "RDF" if status.eql?("READY") - any = true if status.eql?("ANY") - include_views = options[:also_include_views] || false - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << :submissionStatus unless includes.include?(:submissionStatus) - if any - submissions_query = OntologySubmission.where - else - submissions_query = OntologySubmission.where(submissionStatus: [ code: status]) - end + submissions = retrieve_submissions(options) - submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views - submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] - end - submissions = submissions_query.include(includes).to_a - - # Figure out latest parsed submissions using all submissions - latest_submissions = {} + latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| # To retrieve all metadata, but slow when a lot of ontologies - if includes_param.first == :all - sub.bring_remaining + sub.bring_remaining if includes_param.first == :all + unless page? + next if include_ready?(options) && !sub.ready? + next if sub.ontology.nil? + latest_submissions[sub.ontology.acronym] ||= sub + latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end - next if include_ready && !sub.ready? - next if sub.ontology.nil? - latest_submissions[sub.ontology.acronym] ||= sub - latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end latest_submissions end diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index e7ec091a..251af299 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -13,6 +13,10 @@ def settings_params(klass) [attributes, page, size, order_by, bring_unmapped] end + def page? + !params[:page].nil? + end + def is_set?(param) !param.nil? && param != "" end @@ -25,6 +29,53 @@ def filter build_filter end + def apply_filters(query) + + filters = { + naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng], + hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS], + ontology_hasDomain_acronym: params[:hasDomain]&.split(',') , #%w[Crop Vue_francais], + ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], + ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), + isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + } + inverse_filters = { + status: params[:status], #"retired", + submissionStatus: params[:submissionStatus] #"RDF", + } + + filters.each do |key , values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + + inverse_filters.each do |key ,value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) orders = (params["order"] || default_order.to_s).split(',') diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb new file mode 100644 index 00000000..38108050 --- /dev/null +++ b/helpers/submission_helper.rb @@ -0,0 +1,51 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module SubmissionHelper + + def retrieve_submissions(options) + status = (options[:status] || "RDF").to_s.upcase + status = "RDF" if status.eql?("READY") + any = status.eql?("ANY") + include_views = options[:also_include_views] || false + includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) + includes << :submissionStatus unless includes.include?(:submissionStatus) + + + submissions_query = LinkedData::Models::OntologySubmission + if any + submissions_query = submissions_query.where + else + submissions_query = submissions_query.where({submissionStatus: [ code: status]}) + end + + submissions_query = apply_filters(submissions_query) + submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions_query = submissions_query.filter(filter) if filter? + + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + if includes_param.first == :all + includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]} + end + + submissions = submissions_query.include(includes) + if page? + submissions.page(page, size).all + else + submissions.to_a + end + end + + def include_ready?(options) + options[:status] && options[:status].to_s.upcase.eql?("READY") + end + + end + end +end + +helpers Sinatra::Helpers::SubmissionHelper \ No newline at end of file diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7500dce4..92b67585 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -192,4 +192,19 @@ def test_download_acl_only end end + def test_submissions_pagination + num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 2, submission_count: 2) + + get "/submissions" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal 2, submissions.length + + + get "/submissions?page=1&pagesize=1" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + end end