90830/104654: issue PR#8267 - Only first 10 predb records resynced (d…

…efault size) - Fix: - Query with 0 rows to get total number of records to process - Process them in batches of 100
atmire · Dec 29, 2023 · 91f53c2 · 91f53c2
1 parent 7536537
commit 91f53c2
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 28 deletions.
diff --git a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
@@ -105,42 +105,56 @@ private void performStatusUpdate(Context context) throws SearchServiceException,
         solrQuery.addFilterQuery(dateRangeFilter);
         solrQuery.addField(SearchUtils.RESOURCE_ID_FIELD);
         solrQuery.addField(SearchUtils.RESOURCE_UNIQUE_ID);
+        solrQuery.setRows(0);
         QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);
-
-        if (response != null) {
-            logInfoAndOut(response.getResults().size() + " items found to process");
-
-            for (SolrDocument doc : response.getResults()) {
-                String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
-                String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
-                logDebugAndOut("Processing item with UUID: " + uuid);
-
-                Optional<IndexableObject> indexableObject = Optional.empty();
-                try {
-                    indexableObject = indexObjectServiceFactory
-                            .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
-                } catch (SQLException e) {
-                    log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
-                            "\" from the database, removing related solr document", e);
-                }
-
-                try {
-                    if (indexableObject.isPresent()) {
-                        logDebugAndOut("Item exists in DB, updating solr document");
-                        updateItem(context, indexableObject.get());
-                    } else {
-                        logDebugAndOut("Item doesn't exist in DB, removing solr document");
-                        removeItem(context, uniqueId);
-                    }
-                } catch (SQLException | IOException e) {
-                    log.error(e.getMessage(), e);
+        if (response != null && response.getResults() != null) {
+            long nrOfPreDBResults = response.getResults().getNumFound();
+            if (nrOfPreDBResults > 0) {
+                logInfoAndOut(nrOfPreDBResults + " items found to process");
+                int batchSize = configurationService.getIntProperty("script.solr-database-resync.batch-size", 100);
+                for (int start = 0; start < nrOfPreDBResults; start += batchSize) {
+                    solrQuery.setStart(start);
+                    solrQuery.setRows(batchSize);
+                    performStatusUpdateOnNextBatch(context, solrQuery);
                 }
             }
         }
 
         indexingService.commit();
     }
 
+    private void performStatusUpdateOnNextBatch(Context context, SolrQuery solrQuery)
+            throws SolrServerException, IOException {
+        QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);
+
+        for (SolrDocument doc : response.getResults()) {
+            String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
+            String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
+            logDebugAndOut("Processing item with UUID: " + uuid);
+
+            Optional<IndexableObject> indexableObject = Optional.empty();
+            try {
+                indexableObject = indexObjectServiceFactory
+                        .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
+            } catch (SQLException e) {
+                log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
+                        "\" from the database, removing related solr document", e);
+            }
+
+            try {
+                if (indexableObject.isPresent()) {
+                    logDebugAndOut("Item exists in DB, updating solr document");
+                    updateItem(context, indexableObject.get());
+                } else {
+                    logDebugAndOut("Item doesn't exist in DB, removing solr document");
+                    removeItem(context, uniqueId);
+                }
+            } catch (SQLException | IOException e) {
+                log.error(e.getMessage(), e);
+            }
+        }
+    }
+
     private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {
         Map<String,Object> fieldModifier = new HashMap<>(1);
         fieldModifier.put("remove", STATUS_FIELD_PREDB);

diff --git a/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java b/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java
@@ -65,6 +65,7 @@ public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase {
     public void setUp() throws Exception {
         super.setUp();
         configurationService.setProperty("solr-database-resync.time-until-reindex", 1);
+        configurationService.setProperty("script.solr-database-resync.batch-size", 5);
 
         ServiceManager serviceManager = DSpaceServicesFactory.getInstance().getServiceManager();
         searchService = serviceManager.getServiceByName(null, MockSolrSearchCore.class);