Merge pull request #34 from ontoportal-lirmm/feature/paginate-and-fil…

…ter-ontologies-endpoint

Feature: Implement pagination and filters to submissions endpoint

Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2
+  revision: 1484ad56e39810208ee08a14d1319b6bc112f647
   branch: development
   specs:
     goo (0.0.2)
@@ -53,7 +53,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: e9b708c40b2b22b935fb48d18ed19de8148fca35
+  revision: f2168c8ad9ec447338da914d10a352df558494b5
   branch: development
   specs:
     ontologies_linked_data (0.0.1)
@@ -103,11 +103,11 @@ GEM
     activesupport (3.2.22.5)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
-    addressable (2.8.1)
+    addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     airbrussh (1.4.1)
       sshkit (>= 1.6.1, != 1.7.0)
-    backports (3.23.0)
+    backports (3.24.1)
     bcrypt (3.1.18)
     bcrypt_pbkdf (1.1.0)
     bigdecimal (1.4.2)
@@ -171,7 +171,7 @@ GEM
       retriable (>= 2.0, < 4.a)
       rexml
       webrick
-    googleauth (1.3.0)
+    googleauth (1.5.2)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
@@ -228,7 +228,7 @@ GEM
       net-protocol
     net-ssh (7.0.1)
     netrc (0.11.0)
-    newrelic_rpm (8.16.0)
+    newrelic_rpm (9.2.0)
     oj (2.18.5)
     omni_logger (0.1.4)
       logger
@@ -249,7 +249,7 @@ GEM
       rack (>= 0.4)
     rack-cors (1.0.6)
       rack (>= 1.6.0)
-    rack-mini-profiler (3.0.0)
+    rack-mini-profiler (3.1.0)
       rack (>= 1.2.0)
     rack-protection (1.5.5)
       rack
@@ -341,6 +341,7 @@ GEM
     webrick (1.8.1)
 
 PLATFORMS
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES

controllers/ontology_submissions_controller.rb

@@ -1,9 +1,13 @@
 class OntologySubmissionsController < ApplicationController
   get "/submissions" do
     check_last_modified_collection(LinkedData::Models::OntologySubmission)
-    #using appplication_helper method
-    options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")}
-    reply retrieve_latest_submissions(options).values
+    options = {
+               also_include_views: params["also_include_views"],
+               status: (params["include_status"] || "ANY")
+    }
+    subs = retrieve_latest_submissions(options)
+    subs = subs.values unless page?
+    reply subs
   end
 
   ##

helpers/application_helper.rb

@@ -355,40 +355,18 @@ def replace_url_prefix(id)
       end
 
       def retrieve_latest_submissions(options = {})
-        status = (options[:status] || "RDF").to_s.upcase
-        include_ready = status.eql?("READY") ? true : false
-        status = "RDF" if status.eql?("READY")
-        any = true if status.eql?("ANY")
-        include_views = options[:also_include_views] || false
-        includes = OntologySubmission.goo_attrs_to_load(includes_param)
-
-        includes << :submissionStatus unless includes.include?(:submissionStatus)
-        if any
-          submissions_query = OntologySubmission.where
-        else
-          submissions_query = OntologySubmission.where(submissionStatus: [ code: status])
-        end
+        submissions = retrieve_submissions(options)
 
-        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
-        submissions_query = submissions_query.filter(filter) if filter?
-        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
-        if includes_param.first == :all
-          includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl,
-                                       :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]
-        end
-        submissions = submissions_query.include(includes).to_a
-
-        # Figure out latest parsed submissions using all submissions
-        latest_submissions = {}
+        latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination
         submissions.each do |sub|
           # To retrieve all metadata, but slow when a lot of ontologies
-          if includes_param.first == :all
-            sub.bring_remaining
+          sub.bring_remaining   if includes_param.first == :all
+          unless page?
+            next if include_ready?(options) && !sub.ready?
+            next if sub.ontology.nil?
+            latest_submissions[sub.ontology.acronym] ||= sub
+            latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
           end
-          next if include_ready && !sub.ready?
-          next if sub.ontology.nil?
-          latest_submissions[sub.ontology.acronym] ||= sub
-          latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
         end
         latest_submissions
       end

helpers/request_params_helper.rb

@@ -13,6 +13,10 @@ def settings_params(klass)
         [attributes, page, size, order_by, bring_unmapped]
       end
 
+      def page?
+        !params[:page].nil?
+      end
+
       def is_set?(param)
         !param.nil? && param != ""
       end
@@ -25,6 +29,53 @@ def filter
         build_filter
       end
 
+      def apply_filters(query)
+
+        filters = {
+          naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng],
+          hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS],
+          ontology_hasDomain_acronym:  params[:hasDomain]&.split(',') , #%w[Crop Vue_francais],
+          ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP],
+          ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize),
+          isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"],
+          viewingRestriction: params[:viewingRestriction]&.split(','), #["private"]
+        }
+        inverse_filters = {
+          status: params[:status], #"retired",
+          submissionStatus: params[:submissionStatus] #"RDF",
+        }
+
+        filters.each do |key , values|
+          attr = extract_attr(key)
+          next if Array(values).empty?
+
+          filter = Goo::Filter.new(attr).regex(values.first)
+          values.drop(1).each do |v|
+            filter = filter.or(Goo::Filter.new(attr).regex(v))
+          end
+          query = query.filter(filter)
+        end
+
+        inverse_filters.each do |key ,value|
+          attr = extract_attr(key)
+          next unless value
+
+          filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$")
+          query = query.filter(filter)
+        end
+        query
+      end
+
+      def extract_attr(key)
+        attr, sub_attr, sub_sub_attr = key.to_s.split('_')
+
+        return attr.to_sym unless sub_attr
+
+        return {attr.to_sym => [sub_attr.to_sym]} unless  sub_sub_attr
+
+        {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]}
+      end
+
       def get_order_by_from(params, default_order = :asc)
         if is_set?(params['sortby'])
           orders = (params["order"] || default_order.to_s).split(',')

helpers/submission_helper.rb

@@ -0,0 +1,51 @@
+require 'sinatra/base'
+
+module Sinatra
+  module Helpers
+    module SubmissionHelper
+
+      def retrieve_submissions(options)
+        status = (options[:status] || "RDF").to_s.upcase
+        status = "RDF" if status.eql?("READY")
+        any = status.eql?("ANY")
+        include_views = options[:also_include_views] || false
+        includes, page, size, order_by, _  =  settings_params(LinkedData::Models::OntologySubmission)
+        includes << :submissionStatus unless includes.include?(:submissionStatus)
+
+
+        submissions_query = LinkedData::Models::OntologySubmission
+        if any
+          submissions_query = submissions_query.where
+        else
+          submissions_query = submissions_query.where({submissionStatus: [ code: status]})
+        end
+
+        submissions_query = apply_filters(submissions_query)
+        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
+        submissions_query = submissions_query.filter(filter) if filter?
+
+        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
+        if includes_param.first == :all
+          includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl,
+                                                                             :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]
+        elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)}
+          includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]}
+        end
+
+        submissions = submissions_query.include(includes)
+        if page?
+          submissions.page(page, size).all
+        else
+          submissions.to_a
+        end
+      end
+
+      def include_ready?(options)
+        options[:status] && options[:status].to_s.upcase.eql?("READY")
+      end
+
+    end
+  end
+end
+
+helpers Sinatra::Helpers::SubmissionHelper

test/controllers/test_ontology_submissions_controller.rb

@@ -192,4 +192,19 @@ def test_download_acl_only
     end
   end
 
+  def test_submissions_pagination
+    num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 2, submission_count: 2)
+
+    get "/submissions"
+    assert last_response.ok?
+    submissions = MultiJson.load(last_response.body)
+
+    assert_equal 2, submissions.length
+
+
+    get "/submissions?page=1&pagesize=1"
+    assert last_response.ok?
+    submissions = MultiJson.load(last_response.body)
+    assert_equal 1, submissions["collection"].length
+  end
 end