Solr commit changes:

* Add hard commit on the IndexEventConsumer in case of deletion of a document * Remove changes to non solr search cores
atmire · Feb 22, 2024 · 12714dc · 12714dc
1 parent 14007e4
commit 12714dc
Show file tree

Hide file tree

Showing 10 changed files with 25 additions and 28 deletions.
diff --git a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
@@ -138,7 +138,7 @@ private void performStatusUpdate(Context context) throws SearchServiceException,
             }
         }
 
-        indexingService.commit(true);
+        indexingService.commit();
     }
 
     private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {

diff --git a/dspace-api/src/main/java/org/dspace/authority/AuthoritySolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/authority/AuthoritySolrServiceImpl.java
@@ -103,7 +103,7 @@ public void cleanIndex() throws Exception {
     @Override
     public void commit() {
         try {
-            getSolr().commit(false, false);
+            getSolr().commit();
         } catch (SolrServerException e) {
             log.error("Error while committing authority solr server", e);
         } catch (IOException e) {

diff --git a/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java b/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java
@@ -223,7 +223,12 @@ public void end(Context ctx) throws Exception {
         } finally {
             if (!objectsToUpdate.isEmpty() || !uniqueIdsToDelete.isEmpty()) {
 
-                indexer.commit();
+                // Check if our consumer just deleted some items, in that case a hard comment is needed
+                if (uniqueIdsToDelete.isEmpty()) {
+                    indexer.commit();
+                } else {
+                    indexer.commit(true);
+                }
 
                 // "free" the resources
                 objectsToUpdate.clear();

diff --git a/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
@@ -659,7 +659,7 @@ public boolean isUseProxies() {
     public void removeIndex(String query) throws IOException,
         SolrServerException {
         solr.deleteByQuery(query);
-        solr.commit(true, true);
+        solr.commit();
     }
 
     @Override
@@ -730,7 +730,7 @@ public void execute(String query) throws SolrServerException, IOException {
         }
 
         public void commit() throws IOException, SolrServerException {
-            solr.commit(false, false);
+            solr.commit();
         }
 
         /**
@@ -780,7 +780,7 @@ public void process(SolrInputDocument doc) throws IOException, SolrServerExcepti
                 /* query for ip, exclude results previously set as bots. */
                 processor.execute("ip:" + ip + "* AND -isBot:true");
 
-                solr.commit(false, false);
+                solr.commit();
 
             } catch (Exception e) {
                 log.error(e.getMessage(), e);
@@ -808,7 +808,7 @@ public void process(SolrInputDocument doc) throws IOException, SolrServerExcepti
             /* query for ip, exclude results previously set as bots. */
             processor.execute("userAgent:" + agent + " AND -isBot:true");
 
-            solr.commit(false, false);
+            solr.commit();
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }
@@ -1340,7 +1340,7 @@ public void shardSolrIndex() throws IOException, SolrServerException {
                 statisticsYearServer.request(contentStreamUpdateRequest);
             }
 
-            statisticsYearServer.commit(false, false);
+            statisticsYearServer.commit(true, true);
 
 
             //Delete contents of this year from our year query !
@@ -1587,7 +1587,7 @@ public void exportHits() throws Exception {
 
     @Override
     public void commit() throws IOException, SolrServerException {
-        solr.commit(false, false);
+        solr.commit();
     }
 
     protected void addDocumentsToFile(Context context, SolrDocumentList docs, File exportOutput)

diff --git a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
@@ -398,7 +398,7 @@ protected void load(String filename, Context context, boolean verbose) {
 
             // Commit at the end because it takes a while
             try {
-                solr.commit(false, false);
+                solr.commit();
             } catch (SolrServerException sse) {
                 System.err.println("Error committing statistics to solr server!");
                 sse.printStackTrace();

diff --git a/dspace-api/src/main/java/org/dspace/util/SolrImportExport.java b/dspace-api/src/main/java/org/dspace/util/SolrImportExport.java
@@ -347,7 +347,7 @@ private static void reindex(String indexName, String exportDirName, boolean keep
 
             // commit changes
             HttpSolrClient origSolr = new HttpSolrClient.Builder(origSolrUrl).build();
-            origSolr.commit(false, false);
+            origSolr.commit();
 
             // swap back (statistics now going to actual core name in actual data dir)
             swapRequest = new CoreAdminRequest();
@@ -364,7 +364,7 @@ private static void reindex(String indexName, String exportDirName, boolean keep
             importIndex(tempIndexName, exportDir, origSolrUrl, false);
 
             // commit changes
-            origSolr.commit(false, false);
+            origSolr.commit();
 
             // unload now-temp core (temp core name)
             CoreAdminRequest.unloadCore(tempIndexName, false, false, adminSolr);
@@ -465,7 +465,7 @@ public boolean accept(File dir, String name) {
             solr.request(contentStreamUpdateRequest);
         }
 
-        solr.commit(false, false);
+        solr.commit(true, true);
     }
 
     /**
@@ -502,7 +502,7 @@ private static List<String> getMultiValuedFields(HttpSolrClient solr) {
     public static void clearIndex(String solrUrl) throws IOException, SolrServerException {
         HttpSolrClient solr = new HttpSolrClient.Builder(solrUrl).build();
         solr.deleteByQuery("*:*");
-        solr.commit(false, false);
+        solr.commit();
         solr.optimize();
     }
 

diff --git a/dspace-api/src/main/java/org/dspace/util/SolrUpgradePre6xStatistics.java b/dspace-api/src/main/java/org/dspace/util/SolrUpgradePre6xStatistics.java
@@ -160,7 +160,7 @@ public SolrUpgradePre6xStatistics(String indexName, int numRec, int batchSize)
     private void batchUpdateStats() throws SolrServerException, IOException {
         if (docs.size() > 0) {
             server.add(docs);
-            server.commit(false, false);
+            server.commit(true, true);
             docs.clear();
         }
     }

diff --git a/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java b/dspace-api/src/test/java/org/dspace/discovery/DiscoveryIT.java
@@ -820,7 +820,6 @@ private void deposit(WorkspaceItem workspaceItem)
         workspaceItem = context.reloadEntity(workspaceItem);
         XmlWorkflowItem unusedWorkflowItem = workflowService.startWithoutNotify(context, workspaceItem);
         context.commit();
-        indexer.commit();
         context.restoreAuthSystemState();
     }
 
@@ -829,7 +828,6 @@ private void deleteItem(Item item) throws SQLException, AuthorizeException, IOEx
         item = context.reloadEntity(item);
         itemService.delete(context, item);
         context.commit();
-        indexer.commit(true);
         context.restoreAuthSystemState();
     }
 
@@ -839,7 +837,6 @@ private void deleteSubmission(WorkspaceItem anotherWorkspaceItem)
         anotherWorkspaceItem = context.reloadEntity(anotherWorkspaceItem);
         workspaceItemService.deleteAll(context, anotherWorkspaceItem);
         context.commit();
-        indexer.commit(true);
         context.restoreAuthSystemState();
     }
 
@@ -849,7 +846,6 @@ private void deleteWorkflowItem(XmlWorkflowItem workflowItem)
         workflowItem = context.reloadEntity(workflowItem);
         workflowService.deleteWorkflowByWorkflowItem(context, workflowItem, admin);
         context.commit();
-        indexer.commit(true);
         context.restoreAuthSystemState();
     }
 
@@ -863,7 +859,6 @@ private void returnClaimedTask(ClaimedTask taskToUnclaim) throws SQLException, I
         workflowRequirementsService.removeClaimedUser(context, workflowItem, taskToUnclaim.getOwner(),
                 taskToUnclaim.getStepID());
         context.commit();
-        indexer.commit();
         context.setCurrentUser(previousUser);
     }
 
@@ -876,7 +871,6 @@ private void claim(Workflow workflow, PoolTask task, EPerson user)
         WorkflowActionConfig currentActionConfig = step.getActionConfig(task.getActionID());
         workflowService.doState(context, user, null, task.getWorkflowItem().getID(), workflow, currentActionConfig);
         context.commit();
-        indexer.commit();
         context.setCurrentUser(previousUser);
     }
 
@@ -888,7 +882,6 @@ private void executeWorkflowAction(HttpServletRequest httpServletRequest, Workfl
         workflowService.doState(context, task.getOwner(), httpServletRequest, task.getWorkflowItem().getID(), workflow,
                                 workflow.getStep(task.getStepID()).getActionConfig(task.getActionID()));
         context.commit();
-        indexer.commit();
         context.setCurrentUser(previousUser);
     }
 
@@ -906,7 +899,6 @@ private void abort(XmlWorkflowItem workflowItem)
         context.setCurrentUser(admin);
         workflowService.abort(context, workflowItem, admin);
         context.commit();
-        indexer.commit();
         context.setCurrentUser(previousUser);
     }
 }
diff --git a/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java b/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java
@@ -108,7 +108,7 @@ private static synchronized SolrClient initSolrServerForCore(final String coreNa
             //Start with an empty index
             try {
                 server.deleteByQuery("*:*");
-                server.commit(false, false);
+                server.commit();
             } catch (SolrServerException | IOException e) {
                 log.error("Failed to empty Solr index:  {}", e.getMessage(), e);
             }

diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java
@@ -171,7 +171,7 @@ public int index() throws DSpaceSolrIndexerException {
                 }
 
             }
-            solrServerResolver.getServer().commit(false, false);
+            solrServerResolver.getServer().commit();
 
             if (optimize) {
                 println("Optimizing Index");
@@ -340,7 +340,7 @@ private int index(Iterator<Item> iterator) throws DSpaceSolrIndexerException {
                 if (i % batchSize == 0) {
                     System.out.println(i + " items imported so far...");
                     server.add(list);
-                    server.commit(false, false);
+                    server.commit();
                     list.clear();
                 }
             }
@@ -349,7 +349,7 @@ private int index(Iterator<Item> iterator) throws DSpaceSolrIndexerException {
                 if (!list.isEmpty()) {
                     server.add(list);
                 }
-                server.commit(false, false);
+                server.commit(true, true);
                 list.clear();
             }
             return i;
@@ -553,7 +553,7 @@ private void clearIndex() throws DSpaceSolrIndexerException {
         try {
             System.out.println("Clearing index");
             solrServerResolver.getServer().deleteByQuery("*:*");
-            solrServerResolver.getServer().commit(true, true);
+            solrServerResolver.getServer().commit();
             System.out.println("Index cleared");
         } catch (SolrServerException | IOException ex) {
             throw new DSpaceSolrIndexerException(ex.getMessage(), ex);
-Original file line number
+Diff line change
@@ Expand Up @@
                 }
             }
-            indexingService.commit(true);
+            indexingService.commit();
         }
         private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {
@@ Expand Down @@