From 13c91455426ea47f2b4378badbd58fd795a7e6f2 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 00:47:57 +0200 Subject: [PATCH 1/3] add get submission all including all properties test --- .../test_ontology_submissions_controller.rb | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index ee8576bd..437b9180 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -210,4 +210,114 @@ def test_submissions_pagination submissions = MultiJson.load(last_response.body) assert_equal 1, submissions["collection"].length end + + + def test_submissions_default_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + + submission_default_attributes = LinkedData::Models::OntologySubmission.hypermedia_settings[:serialize_default].map(&:to_s) + + get("/submissions?display_links=false&display_context=false&include_status=ANY") + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| submission_default_attributes.eql?(submission_keys(sub)) }) + end + + def test_submissions_all_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + def submission_all_attributes + attrs = OntologySubmission.goo_attrs_to_load([:all]) + embed_attrs = attrs.select { |x| x.is_a?(Hash) }.first + + attrs.delete_if { |x| x.is_a?(Hash) }.map(&:to_s) + embed_attrs.keys.map(&:to_s) + end + get("/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=all&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + + assert(submissions.all? { |sub| submission_all_attributes.sort.eql?(submission_keys(sub).sort) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=all&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + + assert(submission_all_attributes.sort.eql?(submission_keys(sub).sort)) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_custom_includes + ontology_count = 5 + num_onts_created, created_ont_acronyms, ontologies = create_ontologies_and_submissions(ont_count: ontology_count, submission_count: 1, submissions_to_process: []) + include = 'ontology,contact,submissionId' + + get("/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal ontology_count, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/submissions?include=#{include}&display_links=false&display_context=false") + + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions.size + assert(submissions.all? { |sub| include.split(',').eql?(submission_keys(sub)) }) + assert(submissions.all? { |sub| sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id])) }) + + get("/ontologies/#{created_ont_acronyms.first}/latest_submission?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + + get("/ontologies/#{created_ont_acronyms.first}/submissions/1?include=#{include}&display_links=false&display_context=false") + assert last_response.ok? + sub = MultiJson.load(last_response.body) + assert(include.split(',').eql?(submission_keys(sub))) + assert(sub["contact"] && (sub["contact"].first.nil? || sub["contact"].first.keys.eql?(%w[name email id]))) + end + + def test_submissions_param_include + skip('only for local development regrouping a set of tests') + test_submissions_default_includes + test_submissions_all_includes + test_submissions_custom_includes + end + + private + def submission_keys(sub) + sub.to_hash.keys - %w[@id @type id] + end end From 62c7b0e54b1e6a517d2be1e4d2deb134625a1ade Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 02:37:41 +0200 Subject: [PATCH 2/3] extract and use submission_include_params where we use submission.bring --- controllers/admin_controller.rb | 4 ++-- controllers/ontologies_controller.rb | 23 ++++--------------- .../ontology_submissions_controller.rb | 2 +- helpers/submission_helper.rb | 12 ++++++++++ 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/controllers/admin_controller.rb b/controllers/admin_controller.rb index 7ae6d800..747def93 100644 --- a/controllers/admin_controller.rb +++ b/controllers/admin_controller.rb @@ -68,7 +68,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil? check_last_modified(latest) - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + latest.bring(*submission_include_params) NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions) halt 204 end @@ -84,7 +84,7 @@ class AdminController < ApplicationController latest = ont.latest_submission(status: :any) end check_last_modified(latest) if latest - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + latest.bring(*submission_include_params) if latest reply(latest || {}) end diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 1a8f2ebf..58518420 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -38,25 +38,12 @@ class OntologiesController < ApplicationController else latest = ont.latest_submission(status: :any) end - check_last_modified(latest) if latest - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) + if latest - if includes_param.first == :all - # Bring what we need to display all attr of the submission - latest.bring_remaining - latest.bring({:contact=>[:name, :email], - :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, - :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) - else - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << {:contact=>[:name, :email]} if includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} - - latest.bring(*includes) - end + check_last_modified(latest) + latest.bring(*submission_include_params) end - #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest + reply(latest || {}) end @@ -66,7 +53,7 @@ class OntologiesController < ApplicationController patch '/:acronym/latest_submission' do ont = Ontology.find(params["acronym"]).first error 422, "You must provide an existing `acronym` to patch" if ont.nil? - + submission = ont.latest_submission(status: :any) submission.bring(*OntologySubmission.attributes) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 96411a8d..d88e2f43 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -60,7 +60,7 @@ class OntologySubmissionsController < ApplicationController ont.bring(:submissions) ont_submission = ont.submission(params["ontology_submission_id"]) error 404, "`submissionId` not found" if ont_submission.nil? - ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + ont_submission.bring(*submission_include_params) reply ont_submission end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 07e4f27a..c23bcdaa 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -3,6 +3,18 @@ module Sinatra module Helpers module SubmissionHelper + def submission_include_params + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + includes = OntologySubmission.goo_attrs_to_load(includes_param) + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} + end + + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + includes + end def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase From 5bf6a05862e2a36ad7fc1d16b96b0961fee900c5 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 15 Sep 2023 02:38:54 +0200 Subject: [PATCH 3/3] use retrieve_submissions helper in the :acronym/submissions endpoint --- .../ontology_submissions_controller.rb | 20 ++++++---------- helpers/application_helper.rb | 2 -- helpers/submission_helper.rb | 23 +++++-------------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index d88e2f43..5fd5218f 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -29,19 +29,13 @@ class OntologySubmissionsController < ApplicationController error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) check_access(ont) - if includes_param.first == :all - # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) - ont.bring(submissions: [:released, :creationDate, :status, :submissionId, - {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus], :metrics =>[:classes, :individuals, :properties]) - - ont.submissions.each do |sub| - sub.bring_remaining - end - else - ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param)) - end - reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_submissions(options) + + reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i } # descending order of submissionId end ## diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 84c26497..cde04a34 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -359,8 +359,6 @@ def retrieve_latest_submissions(options = {}) latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| - # To retrieve all metadata, but slow when a lot of ontologies - sub.bring_remaining if includes_param.first == :all unless page? next if include_ready?(options) && !sub.ready? next if sub.ontology.nil? diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index c23bcdaa..b9a960b9 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -19,6 +19,7 @@ def submission_include_params def retrieve_submissions(options) status = (options[:status] || "RDF").to_s.upcase status = "RDF" if status.eql?("READY") + ontology_acronym = options[:ontology] any = status.eql?("ANY") include_views = options[:also_include_views] || false includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) @@ -32,28 +33,16 @@ def retrieve_submissions(options) submissions_query = submissions_query.where({submissionStatus: [ code: status]}) end + + submissions_query.where(ontology: [acronym: ontology_acronym]) if ontology_acronym + + submissions_query = apply_filters(submissions_query) submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], - :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, - :submissionStatus] - else - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} - end - - if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} - includes << {:contact=>[:name, :email]} - end - end - submissions = submissions_query.include(includes) + submissions = submissions_query.include(submission_include_params) if page? submissions.page(page, size).all else