Feature: add mapping bulk load endpoint "/mappings/load"

.gitignore

@@ -68,3 +68,5 @@ test/test_run.log
 test/data/ontology_files/catalog-v001.xml
 
 create_permissions.log
+
+ontologies_api.iml

Gemfile

@@ -46,7 +46,7 @@ gem 'goo', github: 'ncbo/goo', branch: 'master'
 gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master'
 gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master'
 gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master'
-gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master'
+gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/feature/add-rest-mapping-bulk-loader'
 gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master'
 
 group :development do

Gemfile.lock

@@ -52,9 +52,19 @@ GIT
       redis
 
 GIT
-  remote: https://github.com/ncbo/ontologies_linked_data.git
-  revision: 8196bf34b45c75f8104bb76dfcba1db0f2c048e4
+  remote: https://github.com/ncbo/sparql-client.git
+  revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9
   branch: master
+  specs:
+    sparql-client (1.0.1)
+      json_pure (>= 1.4)
+      net-http-persistent (= 2.9.4)
+      rdf (>= 1.0)
+
+GIT
+  remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
+  revision: df75e275a52071dd4083ef4ef04cc4e23772bec7
+  branch: pr/feature/add-rest-mapping-bulk-loader
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -71,16 +81,6 @@ GIT
       rsolr
       rubyzip
 
-GIT
-  remote: https://github.com/ncbo/sparql-client.git
-  revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9
-  branch: master
-  specs:
-    sparql-client (1.0.1)
-      json_pure (>= 1.4)
-      net-http-persistent (= 2.9.4)
-      rdf (>= 1.0)
-
 GIT
   remote: https://github.com/palexander/rack-post-body-to-params.git
   revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398
@@ -103,8 +103,8 @@ GEM
     activesupport (3.2.22.5)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
     airbrussh (1.4.1)
       sshkit (>= 1.6.1, != 1.7.0)
     backports (3.23.0)
@@ -131,7 +131,7 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.10.1)
+    faraday (1.10.2)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -159,7 +159,7 @@ GEM
       ffi (~> 1.0)
     google-apis-analytics_v3 (0.10.0)
       google-apis-core (>= 0.7, < 2.a)
-    google-apis-core (0.7.0)
+    google-apis-core (0.7.2)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.16.2, < 2.a)
       httpclient (>= 2.8.1, < 3.a)
@@ -188,7 +188,7 @@ GEM
     json-schema (2.8.1)
       addressable (>= 2.4)
     json_pure (2.6.2)
-    jwt (2.4.1)
+    jwt (2.5.0)
     kgio (2.11.4)
     libxml-ruby (3.2.3)
     logger (1.5.1)
@@ -213,7 +213,7 @@ GEM
       net-ssh (>= 2.6.5)
     net-ssh (7.0.1)
     netrc (0.11.0)
-    newrelic_rpm (8.9.0)
+    newrelic_rpm (8.10.1)
     oj (2.18.5)
     omni_logger (0.1.4)
       logger
@@ -224,7 +224,7 @@ GEM
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.7)
+    public_suffix (5.0.0)
     rack (1.6.13)
     rack-accept (0.4.5)
       rack (>= 0.4)
@@ -246,7 +246,7 @@ GEM
     rdf (1.0.8)
       addressable (>= 2.2)
     redcarpet (3.5.1)
-    redis (4.7.1)
+    redis (4.8.0)
     redis-activesupport (5.3.0)
       activesupport (>= 3, < 8)
       redis-store (>= 1.3, < 2)
@@ -301,7 +301,7 @@ GEM
       rack-test
       sinatra (~> 1.4.0)
       tilt (>= 1.3, < 3)
-    sshkit (1.21.2)
+    sshkit (1.21.3)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
     systemu (2.6.5)
@@ -325,6 +325,7 @@ GEM
     webrick (1.7.0)
 
 PLATFORMS
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES

controllers/mappings_controller.rb

@@ -11,7 +11,7 @@ class MappingsController < ApplicationController
     end
 
     mappings = LinkedData::Mappings.mappings_ontology(submission,
-                                                      0,0,
+                                                      0, 0,
                                                       cls.id)
     populate_mapping_classes(mappings.to_a)
     reply mappings
@@ -21,15 +21,15 @@ class MappingsController < ApplicationController
   get '/ontologies/:ontology/mappings' do
     ontology = ontology_from_acronym(@params[:ontology])
     if ontology.nil?
-        error(404, "Ontology not found")
+      error(404, "Ontology not found")
     end
     page, size = page_params
     submission = ontology.latest_submission
     if submission.nil?
-        error(404, "Submission not found for ontology " + ontology.acronym)
+      error(404, "Submission not found for ontology " + ontology.acronym)
     end
     mappings = LinkedData::Mappings.mappings_ontology(submission,
-                                                      page,size,
+                                                      page, size,
                                                       nil)
     populate_mapping_classes(mappings)
     reply mappings
@@ -55,8 +55,8 @@ class MappingsController < ApplicationController
       if sub2.nil?
         error(404, "Submission not found for ontology " + ontologies[1].id.to_s)
       end
-      mappings = LinkedData::Mappings.mappings_ontologies(sub1,sub2,
-                                                          page,size)
+      mappings = LinkedData::Mappings.mappings_ontologies(sub1, sub2,
+                                                          page, size)
       populate_mapping_classes(mappings)
       reply mappings
     end
@@ -70,7 +70,7 @@ class MappingsController < ApplicationController
       else
         mappings = LinkedData::Mappings.recent_rest_mappings(size + 15)
         populate_mapping_classes(mappings)
-        reply mappings[0..size-1]
+        reply mappings[0..size - 1]
       end
     end
 
@@ -147,6 +147,18 @@ class MappingsController < ApplicationController
       reply(201, mapping)
     end
 
+    post '/load' do
+      begin
+        mappings = parse_bulk_load_file
+        loaded_mappings, errors = LinkedData::Mappings.bulk_load_mappings(mappings, current_user, check_exist: true)
+        response = {}
+        response[:created] = loaded_mappings unless loaded_mappings.empty?
+        response[:errors] = errors unless errors.empty?
+        reply(201, response)
+      rescue ::JSON::ParserError => e
+        error(404, "File parsing error: #{e.message}")
+      end
+    end
     # Delete a mapping
     delete '/:mapping' do
       mapping_id = RDF::URI.new(replace_url_prefix(params[:mapping]))
@@ -166,9 +178,9 @@ class MappingsController < ApplicationController
       persistent_counts = {}
       f = Goo::Filter.new(:pair_count) == false
       LinkedData::Models::MappingCount.where.filter(f)
-      .include(:ontologies,:count)
-      .all
-      .each do |m|
+                                      .include(:ontologies, :count)
+                                      .all
+                                      .each do |m|
         persistent_counts[m.ontologies.first] = m.count
       end
       reply persistent_counts
@@ -189,9 +201,9 @@ class MappingsController < ApplicationController
       persistent_counts = {}
       LinkedData::Models::MappingCount.where(pair_count: true)
                                       .and(ontologies: ontology.acronym)
-      .include(:ontologies,:count)
-      .all
-      .each do |m|
+                                      .include(:ontologies, :count)
+                                      .all
+                                      .each do |m|
         other = m.ontologies.first
         if other == ontology.acronym
           other = m.ontologies[1]

helpers/application_helper.rb

@@ -421,6 +421,23 @@ def include_param_contains?(str)
         return class_params_include || params_include
       end
 
+
+      ##
+      # Checks to see if the request has a file attached
+      def request_has_file?
+        @params.any? {|p,v| v.instance_of?(Hash) && v.key?(:tempfile) && v[:tempfile].instance_of?(Tempfile)}
+      end
+
+      ##
+      # Looks for a file that was included as a multipart in a request
+      def file_from_request
+        @params.each do |param, value|
+          if value.instance_of?(Hash) && value.has_key?(:tempfile) && value[:tempfile].instance_of?(Tempfile)
+            return value[:filename], value[:tempfile]
+          end
+        end
+        return nil, nil
+      end
       private
 
       def naive_expiring_cache_write(key, object, timeout = 60)

helpers/mappings_helper.rb

@@ -33,6 +33,18 @@ def populate_mapping_classes(mappings)
 
         mappings
       end
+      ##
+      # Parse the uploaded mappings file
+      def parse_bulk_load_file
+        filename, tmpfile = file_from_request
+        if tmpfile
+          if filename.nil?
+            error 400, "Failure to resolve mappings json filename from upload file."
+          end
+          Array(::JSON.parse(tmpfile.read,{:symbolize_names => true}))
+        end
+
+      end
     end
   end
 end

helpers/ontology_helper.rb

@@ -35,22 +35,6 @@ def create_submission(ont)
         ont_submission
       end
 
-      ##
-      # Checks to see if the request has a file attached
-      def request_has_file?
-        @params.any? {|p,v| v.instance_of?(Hash) && v.key?(:tempfile) && v[:tempfile].instance_of?(Tempfile)}
-      end
-
-      ##
-      # Looks for a file that was included as a multipart in a request
-      def file_from_request
-        @params.each do |param, value|
-          if value.instance_of?(Hash) && value.has_key?(:tempfile) && value[:tempfile].instance_of?(Tempfile)
-            return value[:filename], value[:tempfile]
-          end
-        end
-        return nil, nil
-      end
 
       ##
       # Add a file to the submission if a file exists in the params