fix issue with entry model being wrong

.github/workflows/build-test-lint.yaml

@@ -1,4 +1,5 @@
 name: "Ruby on Rails CI"
+run-name: Build of ${{ github.ref_name }} by @${{ github.actor }}
 on:
   pull_request:
     branches:

.github/workflows/deploy.yaml

@@ -1,4 +1,5 @@
 name: "Deploy"
+run-name: Deploy (${{ github.ref_name }} -> ${{ inputs.environment }}) by @${{ github.actor }}
 on:
   workflow_dispatch:
     inputs:

Gemfile.lock

@@ -41,7 +41,7 @@ GIT
 
 GIT
   remote: https://github.com/samvera/hyrax.git
-  revision: d91636abdddde752ca2bd59f43097ca3067b1d8f
+  revision: 943c760886df5e7daea839b5bd22b6e6a52e1229
   branch: double_combo
   specs:
     hyrax (5.0.0.rc2)

app/forms/digital_instantiation_resource_form.rb

@@ -14,7 +14,6 @@ class DigitalInstantiationResourceForm < Hyrax::Forms::ResourceForm(DigitalInsta
 
   attr_accessor :controller, :current_ability
 
-  self.required_fields -= [:creator, :keyword, :rights_statement]
   self.required_fields += [:title, :location, :holding_organization]
 
   class_attribute :field_groups

app/forms/physical_instantiation_resource_form.rb

@@ -16,7 +16,6 @@ class PhysicalInstantiationResourceForm < Hyrax::Forms::ResourceForm(PhysicalIns
 
   attr_accessor :controller, :current_ability
 
-  self.required_fields -= [:creator, :keyword, :rights_statement]
   self.required_fields += [:format, :location, :media_type, :holding_organization]
 
   self.single_valued_fields = [:title]

app/models/asset_resource.rb

@@ -14,6 +14,7 @@ class AssetResource < Hyrax::Work
   VALIDATION_STATUSES = {
     valid: 'valid',
     missing_children: 'missing child record(s)',
+    invalid_children: 'invalid child record(s)',
     status_not_validated: 'not yet validated',
     empty: 'missing a validation status'
   }.freeze
@@ -146,7 +147,8 @@ def find_admin_data_attribute(attribute)
     end
   end
 
-  def set_validation_status
+  def set_validation_status(child_statuses = [])
+    return [([Asset::VALIDATION_STATUSES[:invalid_children]] + child_statuses).to_sentence] if child_statuses.present?
     current_children_count = SolrDocument.get_members(self).reject { |child| child.is_a?(Contribution) || child.is_a?(ContributionResource) || child.id == self.id }.size
     intended_children_count = self.intended_children_count.to_i
 

app/models/bulkrax/pbcore_xml_entry.rb

@@ -36,29 +36,24 @@ def self.data_for_entry(data, source_id)
 
     def build_metadata
       raise StandardError, 'Record not found' if record.nil?
-
       self.parsed_metadata = {}
       self.parsed_metadata[work_identifier] = self.raw_metadata[source_identifier]
-      self.parsed_metadata['model'] = self.raw_metadata['model']
-      if self.raw_metadata['model'] == 'DigitalInstantiationResource'
-        self.parsed_metadata['pbcore_xml'] = self.raw_metadata['pbcore_xml'] if self.raw_metadata['pbcore_xml'].present?
-        self.parsed_metadata['format'] = self.raw_metadata['format']
-        self.parsed_metadata['skip_file_upload_validation'] = self.raw_metadata['skip_file_upload_validation'] if self.raw_metadata['skip_file_upload_validation'] == true
-      end
+      self.parsed_metadata['model'] = if self.raw_metadata['model']&.match(/Resource/)
+                                        self.raw_metadata['model']
+                                      elsif self.raw_metadata['model'].present?
+                                        "#{self.raw_metadata['model']}Resource"
+
+                                      end
+
 
       self.raw_metadata.each do |key, value|
+        # skip the ones we've already added
+        next if key == 'model' || key == 'pbcore_xml' || key == 'format' || key == 'skip_file_upload_validation'
         add_metadata(key_without_numbers(key), value)
       end
 
-      if self.raw_metadata['model'] == 'AssetResource'
-        self.parsed_metadata["contributors"] = self.raw_metadata["contributors"]
-        self.parsed_metadata['bulkrax_importer_id'] = importer.id
-        self.parsed_metadata['admin_data_gid'] = admin_data_gid
-        self.parsed_metadata['sonyci_id'] = self.raw_metadata['sonyci_id']
-        build_annotations(self.raw_metadata['annotations']) if self.raw_metadata['annotations'].present?
-      end
-
       self.parsed_metadata['label'] = nil if self.parsed_metadata['label'] == "[]"
+      self.parsed_metadata['dimensions'] = nil if self.parsed_metadata['dimensions'] == "[]"
       add_visibility
       add_rights_statement
       add_admin_set_id
@@ -68,44 +63,5 @@ def build_metadata
 
       self.parsed_metadata
     end
-
-    def admin_data
-      return @admin_data if @admin_data.present?
-      asset_resource_id = self.raw_metadata['Asset.id'].strip if self.raw_metadata.keys.include?('Asset.id')
-      asset_resource_id ||= self.raw_metadata['id']
-      begin
-        work = Hyrax.query_service.find_by(id: asset_resource_id) if asset_resource_id
-      rescue Valkyrie::Persistence::ObjectNotFoundError
-        work = nil
-      end
-
-      @admin_data = work.admin_data if work.present?
-      @admin_data ||= AdminData.find_by_gid(self.raw_metadata['admin_data_gid']) if self.raw_metadata['admin_data_gid'].present?
-      @admin_data ||= AdminData.new
-      @admin_data.bulkrax_importer_id = importer.id
-      @admin_data.save
-      @admin_data
-    end
-
-    def admin_data_gid
-      admin_data.gid
-    end
-
-    def build_annotations(annotations)
-      annotations.each do |annotation|
-        if annotation['annotation_type'].nil?
-          raise "annotation_type not registered with the AnnotationTypesService: #{annotation['annotation_type']}."
-        end
-
-        Annotation.find_or_create_by(
-          annotation_type: annotation['annotation_type'],
-          source: annotation['source'],
-          value: annotation['value'],
-          annotation: annotation['annotation'],
-          version: annotation['version'],
-          admin_data_id: admin_data.id
-        )
-      end
-    end
   end
 end

app/models/concerns/bulkrax/has_local_processing.rb

@@ -4,5 +4,77 @@ module Bulkrax::HasLocalProcessing
   # This method is called during build_metadata
   # add any special processing here, for example to reset a metadata property
   # to add a custom property from outside of the import data
-  def add_local; end
+  def add_local
+    case self.parsed_metadata['model']
+    when 'DigitalInstantiationResource',
+        add_digital_metadata
+    when 'PhysicalInstantiationResource'
+
+      add_physical_metadata
+    when 'AssetResource'
+      add_asset_metadata
+    end
+  end
+
+  def add_asset_metadata
+    self.parsed_metadata["contributors"] = self.raw_metadata["contributors"]
+    self.parsed_metadata['bulkrax_importer_id'] = importer.id
+    self.parsed_metadata['admin_data_gid'] = admin_data_gid
+    self.parsed_metadata['sonyci_id'] = self.raw_metadata['sonyci_id']
+    build_annotations(self.raw_metadata['annotations']) if self.raw_metadata['annotations'].present?
+  end
+
+  def add_digital_metadata
+    add_instantiation_metadata
+    self.parsed_metadata['pbcore_xml'] = self.raw_metadata['pbcore_xml'] if self.raw_metadata['pbcore_xml'].present?
+
+    self.parsed_metadata['skip_file_upload_validation'] = self.raw_metadata['skip_file_upload_validation'] if self.raw_metadata['skip_file_upload_validation'] == true
+  end
+
+  def add_physical_metadata
+    add_instantiation_metadata
+  end
+
+  def add_instantiation_metadata
+    self.parsed_metadata['format'] = self.raw_metadata['format']
+  end
+
+  def admin_data
+    return @admin_data if @admin_data.present?
+    asset_resource_id = self.raw_metadata['Asset.id'].strip if self.raw_metadata.keys.include?('Asset.id')
+    asset_resource_id ||= self.raw_metadata['id']
+    begin
+      work = Hyrax.query_service.find_by(id: asset_resource_id) if asset_resource_id
+    rescue Valkyrie::Persistence::ObjectNotFoundError
+      work = nil
+    end
+
+    @admin_data = work.admin_data if work.present?
+    @admin_data ||= AdminData.find_by_gid(self.raw_metadata['admin_data_gid']) if self.raw_metadata['admin_data_gid'].present?
+    @admin_data ||= AdminData.new
+    @admin_data.bulkrax_importer_id = importer.id
+    @admin_data.save
+    @admin_data
+  end
+
+  def admin_data_gid
+    admin_data.gid
+  end
+
+  def build_annotations(annotations)
+    annotations.each do |annotation|
+      if annotation['annotation_type'].nil?
+        raise "annotation_type not registered with the AnnotationTypesService: #{annotation['annotation_type']}."
+      end
+
+      Annotation.find_or_create_by(
+        annotation_type: annotation['annotation_type'],
+        source: annotation['source'],
+        value: annotation['value'],
+        annotation: annotation['annotation'],
+        version: annotation['version'],
+        admin_data_id: admin_data.id
+      )
+    end
+  end
 end

app/models/date_validator.rb

@@ -3,6 +3,7 @@ def validate_each(record, attribute, value)
     value = Array.wrap(value)
     value.each do |val|
       if AMS::NonExactDateService.invalid?(val)
+        next if options[:allow_blank] && val.blank?
         record.errors.add attribute, (options[:message] || "invalid date format: #{val}")
       end
     end

app/models/digital_instantiation_resource.rb

@@ -53,4 +53,16 @@ def instantiation_admin_data=(new_admin_data)
     @instantiation_admin_data = new_admin_data
   end
 
+  def aapb_valid?
+    aapb_invalid_message.blank?
+  end
+
+  def aapb_invalid_message
+    msg = []
+    msg << "#{self.id} title is required" unless title.present?
+    msg << "#{self.id} location is required" unless location.present?
+    msg << "#{self.id} media_type is required" unless media_type.present?
+    msg << "#{self.id} holding_organization is required" unless holding_organization.present?
+    msg.to_sentence if msg.present?
+  end
 end

app/models/essence_track_resource.rb

@@ -8,6 +8,18 @@ class EssenceTrackResource < Hyrax::Work
   include Hyrax::ArResource
   include AMS::WorkBehavior
 
+  VALIDATION_STATUSES = {
+    valid: 'valid',
+    track_missing: 'track id or track type is missing',
+  }.freeze
+
   self.valid_child_concerns = []
 
+  def aapb_valid?
+    track_id.present? && track_type.present?
+  end
+
+  def aapb_invalid_message
+    "#{self.id} track id or track type is missing" unless aapb_valid?
+  end
 end

app/models/physical_instantiation_resource.rb

@@ -33,4 +33,17 @@ def instantiation_admin_data=(new_admin_data)
     self.instantiation_admin_data_gid = new_admin_data.gid
     @instantiation_admin_data = new_admin_data
   end
+
+  def aapb_valid?
+    aapb_invalid_message.blank?
+  end
+
+  def aapb_invalid_message
+    msg = []
+    msg << "#{self.id} format is required" unless format.present?
+    msg << "#{self.id} location is required" unless location.present?
+    msg << "#{self.id} media_type is required" unless media_type.present?
+    msg << "#{self.id} holding_organization is required" unless holding_organization.present?
+    msg.to_sentence if msg.present?
+  end
 end

app/models/solr_document.rb

@@ -560,7 +560,7 @@ def annotations
 
   def instantiation_admin_data_gid
     return unless is_instantiation?
-    @instantiation_admin_data_gid ||= self['admin_data_gid_ssim'].first
+    @instantiation_admin_data_gid ||= Array.wrap(self['admin_data_gid_ssim']).first
   end
 
   def instantiation_admin_data

app/parsers/pbcore_xml_parser.rb

@@ -24,7 +24,7 @@ def records(_opts = {})
           begin
             data = entry_class.read_data(md).xpath("//#{record_element}").first # Take only the first record
             entry_class.data_for_entry(data, source_identifier)
-          rescue Nokogiri::XML::SyntaxError => e
+          rescue => e
             invalid_files << { message: e, filepath: md }
           end
         end.compact # No need to flatten because we take only the first record
@@ -72,7 +72,7 @@ def create_works
   ##
   # This method is useful for updating existing entries with out reimporting the works themselves
   # used in scripts and on the console
-  def recreate_entries
+  def recreate_entries(progress = nil)
     self.record_objects = []
     records.each_with_index do |file, index|
       set_objects(file, index).each do |record|
@@ -82,8 +82,8 @@ def recreate_entries
         new_entry = find_or_create_entry(entry_class, record[work_identifier], 'Bulkrax::Importer', record.compact)
       end
       increment_counters(index)
+      progress.increment if progress
     end
-    importer.record_status
   rescue StandardError => e
     status_info(e)
   end

app/presenters/hyrax/asset_resource_presenter.rb

@@ -34,16 +34,16 @@ def batch_ingest_date
     end
 
     def bulkrax_import
-      raise 'No Bulkrax Import ID associated with this Asset' unless bulkrax_importer_id.present?
-      @bulkrax_import ||= Bulkrax::Importer.find(bulkrax_importer_id.first)
+      return nil unless bulkrax_importer_id.present?
+      @bulkrax_import ||= Bulkrax::Importer.find_by(id: bulkrax_importer_id.first)
     end
 
     def bulkrax_import_url
-      @bulkrax_import_url ||= "/importers/#{bulkrax_import.id}"
+      @bulkrax_import_url ||= "/importers/#{bulkrax_import.id}" if bulkrax_import&.id
     end
 
     def bulkrax_import_label
-      @bulkrax_import_ingest_label ||= bulkrax_import.parser_klass
+      @bulkrax_import_ingest_label ||= bulkrax_import&.parser_klass
     end
 
     def bulkrax_import_date

app/services/ams/backfill_asset_validation_status.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 require 'ruby-progressbar'
 
+# TODO convert to new reprocessor style
 module AMS
-  class BackfillAssetValidationStatus < AMS::WorkReprocessor
+  class BackfillAssetValidationStatus # < AMS::WorkReprocessor
     def initialize
       super(dir_name: 'backfill_asset_validation_status')
       @query = 'has_model_ssim:Asset -intended_children_count_isi:[* TO *]'

app/services/ams/export/search/catalog_search.rb

@@ -5,22 +5,18 @@ class CatalogSearch < Base
         # Include Blacklight modules that provide methods for configurating and
         # performing searches.
 
-        # this is required - advanced_search will crash without it
-        copy_blacklight_config_from(CatalogController)
-        configure_blacklight do |config|
-          # This is necessary to prevent Blacklight's default value of 100 for
-          # config.max_per_page from capping the number of results.
-          config.max_per_page = MAX_LIMIT
-        end
-
         private
           # Overwrite Base#response to use Blacklight::SearchHelper#search_results.
-          def response
+        def response
+          blacklight_config = CatalogController.blacklight_config.dup
+          blacklight_config.default_solr_params = { rows: 2_000_000 }
+          blacklight_config.max_per_page = 2_000_000
             @response ||= Hyrax::SearchService.new(
-                            config: CatalogController.blacklight_config,
+                            config: blacklight_config,
                             user_params: search_params,
                             scope: self,
-                            current_ability: user.ability
+                            current_ability: user.ability,
+                            rows: 2_000_000
                           ).search_results[0]
           end