From 78c724a87fa97ba6ca460b2ceafe4f27888c0d61 Mon Sep 17 00:00:00 2001 From: sheiksalahudeen Date: Tue, 20 Sep 2016 19:53:27 -0400 Subject: [PATCH 1/2] OLE-9004 : Fixed the issue with inaccurate record counts when doing Incremental exports to vufind --- .../dao/export/ExportBibIdFinderCallable.java | 49 ++++++++++++ .../kuali/ole/oleng/dao/export/ExportDao.java | 76 +++++++++++++++---- .../dao/export/ExportDaoCallableImpl.java | 76 ++++++++++--------- .../dao/export/ExportHoldingsDaoCallable.java | 4 +- .../helper/ExportEholdingsMappingHelper.java | 2 +- .../helper/ExportHoldingsMappingHelper.java | 2 +- .../kuali/ole/constants/OleNGConstants.java | 3 + 7 files changed, 157 insertions(+), 55 deletions(-) create mode 100644 ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java new file mode 100644 index 0000000000..d35236e021 --- /dev/null +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java @@ -0,0 +1,49 @@ +package org.kuali.ole.oleng.dao.export; + +import org.apache.commons.lang.StringUtils; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.kuali.ole.DocumentUniqueIDPrefix; +import org.kuali.ole.constants.OleNGConstants; +import org.kuali.ole.oleng.handler.BatchExportHandler; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; + +/** + * Created by sheiks on 16/09/16. + */ +public class ExportBibIdFinderCallable implements Callable { + + private String query; + private int start; + private int chunkSize; + private BatchExportHandler batchExportHandler; + + public ExportBibIdFinderCallable(String query, int start, int chunkSize, BatchExportHandler batchExportHandler) { + this.query = query; + this.start = start; + this.chunkSize = chunkSize; + this.batchExportHandler = batchExportHandler; + + } + + @Override + public Object call() throws Exception { + Set bibIdentifiers = new HashSet<>(); + SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); + if (solrDocumentList.size() > 0) { + for (SolrDocument solrDocument : solrDocumentList) { + if (solrDocument.containsKey(OleNGConstants.BIB_IDENTIFIER)) { + List bibIds = (List) solrDocument.getFieldValue(OleNGConstants.BIB_IDENTIFIER); + for (String bibId : bibIds) { + bibIdentifiers.add(DocumentUniqueIDPrefix.getDocumentId(bibId)); + } + } + } + } + return bibIdentifiers; + } +} diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java index 666f0738ab..6b485807c7 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java @@ -1,10 +1,9 @@ package org.kuali.ole.oleng.dao.export; +import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.kuali.ole.DocumentUniqueIDPrefix; import org.kuali.ole.OLEConstants; import org.kuali.ole.constants.OleNGConstants; import org.kuali.ole.deliver.service.ParameterValueResolver; @@ -53,7 +52,8 @@ private void init() throws Exception { fetchExtentOfOwnershipType(); } - public void export(BatchExportHandler batchExportHandler, String query, BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { + public void export(BatchExportHandler batchExportHandler, String query, BatchProcessTxObject batchProcessTxObject, + OleNGBatchExportResponse oleNGBatchExportResponse) { try { init(); this.chunkSize = batchExportHandler.getBatchChunkSize(); @@ -64,18 +64,31 @@ public void export(BatchExportHandler batchExportHandler, String query, BatchPro int numOfRecordsInFile = 0; SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); totalCount = solrDocumentList.getNumFound(); + + List bibIds = getBibIds(query, totalCount, chunkSize, batchExportHandler, batchProcessTxObject); + + totalCount = bibIds.size(); + + if(fileSize < chunkSize) { + chunkSize = fileSize; + } + batchProcessTxObject.getBatchJobDetails().setTotalRecords(String.valueOf(totalCount)); - oleNGBatchExportResponse.setTotalNumberOfRecords((int) totalCount); + oleNGBatchExportResponse.setTotalNumberOfRecords(((int)totalCount)); batchExportHandler.updateBatchJob(batchProcessTxObject.getBatchJobDetails()); - do { - futures.add(executorService.submit(new ExportDaoCallableImpl(commonFields, getJdbcTemplate(), start, chunkSize, query, fileCount, batchExportHandler, batchProcessTxObject, oleNGBatchExportResponse))); + + List> partition = Lists.partition(bibIds, chunkSize); + + for (Iterator> iterator = partition.iterator(); iterator.hasNext(); ) { + List bibIdLists = iterator.next(); + futures.add(executorService.submit(new ExportDaoCallableImpl(commonFields, getJdbcTemplate(),bibIdLists, + fileCount, batchExportHandler, batchProcessTxObject))); numOfRecordsInFile += chunkSize; - if (numOfRecordsInFile == fileSize) { + if (numOfRecordsInFile >= fileSize) { fileCount++; numOfRecordsInFile = 0; } - start += chunkSize; - } while (start <= totalCount); + } prepareBatchExportResponse(futures, batchExportHandler, batchProcessTxObject, oleNGBatchExportResponse); executorService.shutdown(); } catch (Exception e) { @@ -84,14 +97,41 @@ public void export(BatchExportHandler batchExportHandler, String query, BatchPro } } - private void prepareBatchExportResponse(List futures, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { + private List getBibIds(String query, long totalCount, int chunkSize, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject) { + Set bibIdentifiers = new HashSet<>(); + List futures = new ArrayList<>(); + int startIndex = 0; + ExecutorService executorService = Executors.newFixedThreadPool(getMaximumNumberOfThreadForExportService()); + do { + futures.add(executorService.submit(new ExportBibIdFinderCallable(query,startIndex, chunkSize, batchExportHandler))); + startIndex += chunkSize; + } while (startIndex <= totalCount); + + for (Future future : futures) { + try { + Object response = future.get(); + if (null != response) { + Set bibIds = (Set) response; + bibIdentifiers.addAll(bibIds); + } + } catch (Exception e) { + e.printStackTrace(); + batchExportHandler.addBatchExportFailureResponseToExchange(e, null, batchProcessTxObject.getExchangeObjectForBatchExport()); + } + } + return new ArrayList(bibIdentifiers); + } + + private void prepareBatchExportResponse(List futures, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse originalResponse) { if (CollectionUtils.isNotEmpty(futures)) { for (Future future : futures) { try { - if (null != future.get()) { - OleNGBatchExportResponse exportResponse = (OleNGBatchExportResponse) future.get(); - batchProcessTxObject.getBatchJobDetails().setTotalFailureRecords(String.valueOf(exportResponse.getNoOfFailureRecords())); - batchProcessTxObject.getBatchJobDetails().setTotalRecordsProcessed(String.valueOf(exportResponse.getNoOfSuccessRecords() + exportResponse.getNoOfFailureRecords())); + Object response = future.get(); + if (null != response) { + OleNGBatchExportResponse exportResponse = (OleNGBatchExportResponse) response; + mergeResponses(originalResponse, exportResponse); + batchProcessTxObject.getBatchJobDetails().setTotalFailureRecords(String.valueOf(originalResponse.getNoOfFailureRecords())); + batchProcessTxObject.getBatchJobDetails().setTotalRecordsProcessed(String.valueOf(originalResponse.getNoOfSuccessRecords() + originalResponse.getNoOfFailureRecords())); batchExportHandler.updateBatchJob(batchProcessTxObject.getBatchJobDetails()); } } catch (InterruptedException e) { @@ -103,6 +143,14 @@ private void prepareBatchExportResponse(List futures, BatchExportHandler } } + private void mergeResponses(OleNGBatchExportResponse originalResponse, OleNGBatchExportResponse exportResponse) { + originalResponse.addNoOfSuccessRecords(exportResponse.getNoOfSuccessRecords()); + originalResponse.addNoOfFailureRecords(exportResponse.getNoOfFailureRecords()); + originalResponse.getBatchExportSuccessResponseList().addAll(exportResponse.getBatchExportSuccessResponseList()); + originalResponse.getBatchExportFailureResponseList().addAll(exportResponse.getBatchExportFailureResponseList()); + originalResponse.getDeletedBibIds().addAll(exportResponse.getDeletedBibIds()); + } + private void fetchCallNumberType() throws SQLException { SqlRowSet resultSet = getJdbcTemplate().queryForRowSet("SELECT SHVLG_SCHM_ID,SHVLG_SCHM_CD,SHVLG_SCHM_NM from OLE_CAT_SHVLG_SCHM_T"); diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java index 6df5075bbd..fcfcc05c6d 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java @@ -1,8 +1,7 @@ package org.kuali.ole.oleng.dao.export; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; import org.kuali.ole.DocumentUniqueIDPrefix; import org.kuali.ole.constants.OleNGConstants; import org.kuali.ole.docstore.common.document.*; @@ -45,28 +44,18 @@ public class ExportDaoCallableImpl implements Callable { private PlatformTransactionManager transactionManager; - String bibQuery = "SELECT * FROM OLE_DS_BIB_T WHERE BIB_ID "; - JdbcTemplate jdbcTemplate; - String bibIds; - int fileNumber; - OleNGBatchExportResponse oleNGBatchExportResponse; - BatchExportHandler batchExportHandler; - BatchProcessTxObject batchProcessTxObject; - - public ExportDaoCallableImpl(Map> commomFields, JdbcTemplate jdbcTemplate, int start, int chunkSize, String query, int fileNumber, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { - SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); - if (solrDocumentList.size() > 0) { - Set bibIdentifiers = new HashSet<>(); - for (SolrDocument solrDocument : solrDocumentList) { - if (solrDocument.containsKey(OleNGConstants.BIB_IDENTIFIER)) { - List bibIds = (List) solrDocument.getFieldValue(OleNGConstants.BIB_IDENTIFIER); - for (String bibId : bibIds) { - bibIdentifiers.add(DocumentUniqueIDPrefix.getDocumentId(bibId)); - } - } - } - this.bibIds = StringUtils.join(bibIdentifiers, ','); - } + private String bibQuery = "SELECT * FROM OLE_DS_BIB_T WHERE BIB_ID "; + private JdbcTemplate jdbcTemplate; + private String bibIdsString; + private int fileNumber; + private OleNGBatchExportResponse oleNGBatchExportResponse; + private BatchExportHandler batchExportHandler; + private BatchProcessTxObject batchProcessTxObject; + private List bibIds; + + public ExportDaoCallableImpl(Map> commomFields, JdbcTemplate jdbcTemplate,List bibIds, int fileNumber, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject) { + this.bibIds = new ArrayList<>(bibIds); + this.bibIdsString = StringUtils.join(bibIds, ','); this.jdbcTemplate = jdbcTemplate; this.callNumberType = commomFields.get("callNumberType"); this.receiptStatus = commomFields.get("receiptStatus"); @@ -76,15 +65,15 @@ public ExportDaoCallableImpl(Map> commomFields, Jdbc this.statisticalSearchCodeMap = commomFields.get("statisticalSearchCodeMap"); this.extentOfOwnershipTypeMap = commomFields.get("extentOfOwnershipTypeMap"); this.fileNumber = fileNumber; - this.oleNGBatchExportResponse = oleNGBatchExportResponse; this.batchExportHandler = batchExportHandler; this.batchProcessTxObject = batchProcessTxObject; + this.oleNGBatchExportResponse = new OleNGBatchExportResponse(); } @Override public Object call() throws Exception { final TransactionTemplate template = new TransactionTemplate(getTransactionManager()); - final SqlRowSet bibResultSet = this.jdbcTemplate.queryForRowSet(bibQuery + " IN ( " + bibIds + " )"); + final SqlRowSet bibResultSet = this.jdbcTemplate.queryForRowSet(bibQuery + " IN ( " + bibIdsString + " )"); try { template.execute(new TransactionCallback() { @Override @@ -93,14 +82,17 @@ public Object doInTransaction(TransactionStatus status) { Bib bib = null; List holdingsTreeList = null; List marcRecord = null; + int numberOfSuccesssRecord = 0; + int numberOfFailureRecord = 0; while (bibResultSet.next()) { try { + bibIds.remove(bibResultSet.getString(1)); bib = fetchBibRecord(bibResultSet); holdingsTreeList = fetchHoldingsTreeForBib(Integer.parseInt(bib.getLocalId())); marcRecord = batchExportHandler.getMarcRecordUtil().convertMarcXmlContentToMarcRecord(bib.getContent()); - processDataMappings(bib.getId(), marcRecord.get(0), holdingsTreeList, batchProcessTxObject.getBatchProcessProfile(), oleNGBatchExportResponse); - processDataTransformations(bib.getId(), marcRecord.get(0), batchProcessTxObject.getBatchProcessProfile(), batchExportHandler, oleNGBatchExportResponse); - oleNGBatchExportResponse.addNoOfSuccessRecords(1); + processDataMappings(bib.getId(), marcRecord.get(0), holdingsTreeList, batchProcessTxObject.getBatchProcessProfile()); + processDataTransformations(bib.getId(), marcRecord.get(0), batchProcessTxObject.getBatchProcessProfile(), batchExportHandler); + numberOfSuccesssRecord++; oleNGBatchExportResponse.addSuccessRecord(bib.getLocalId(), bib.getId(), OleNGConstants.SUCCESS); marcRecords.addAll(marcRecord); @@ -110,15 +102,15 @@ public Object doInTransaction(TransactionStatus status) { } } catch (Exception e) { e.printStackTrace(); - oleNGBatchExportResponse.addNoOfFailureRecords(1); + numberOfFailureRecord++; oleNGBatchExportResponse.addFailureRecord(bib.getLocalId(), bib.getId(), e.getMessage()); batchExportHandler.addBatchExportFailureResponseToExchange(e, bib.getId(), batchProcessTxObject.getExchangeObjectForBatchExport()); } } + preparedReportForUnSyncRecords(bibIds, oleNGBatchExportResponse); + oleNGBatchExportResponse.addNoOfSuccessRecords(numberOfSuccesssRecord); + oleNGBatchExportResponse.addNoOfFailureRecords(numberOfFailureRecord + bibIds.size()); batchExportHandler.generateFileForMarcRecords(fileNumber, marcRecords, batchProcessTxObject); - batchProcessTxObject.getBatchJobDetails().setTotalFailureRecords(String.valueOf(oleNGBatchExportResponse.getNoOfFailureRecords())); - batchProcessTxObject.getBatchJobDetails().setTotalRecordsProcessed(String.valueOf(oleNGBatchExportResponse.getNoOfSuccessRecords() + oleNGBatchExportResponse.getNoOfFailureRecords())); - batchExportHandler.updateBatchJob(batchProcessTxObject.getBatchJobDetails()); return oleNGBatchExportResponse; } }); @@ -131,13 +123,23 @@ public Object doInTransaction(TransactionStatus status) { return oleNGBatchExportResponse; } - private void processDataMappings(String bibId, Record marcRecord, List holdingsTreeList, BatchProcessProfile batchProcessProfile, OleNGBatchExportResponse oleNGBatchExportResponse) { + private void preparedReportForUnSyncRecords(List bibIds, OleNGBatchExportResponse oleNGBatchExportResponse) { + if(CollectionUtils.isNotEmpty(bibIds)) { + for (Iterator iterator = bibIds.iterator(); iterator.hasNext(); ) { + String bibId = iterator.next(); + String bibIdWithPrefix = DocumentUniqueIDPrefix.getPrefixedId(DocumentUniqueIDPrefix.PREFIX_WORK_BIB_MARC, bibId); + oleNGBatchExportResponse.addFailureRecord(bibId, bibIdWithPrefix, OleNGConstants.SOLR_RECORD_NOT_SYNC_WITH_DB_RECORD); + } + } + } + + private void processDataMappings(String bibId, Record marcRecord, List holdingsTreeList, BatchProcessProfile batchProcessProfile) { for (HoldingsTree holdingsTree : holdingsTreeList) { try { if (holdingsTree.getHoldings().getHoldingsType().equalsIgnoreCase(OleNGConstants.PRINT)) { - marcRecord.getDataFields().addAll(new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile, oleNGBatchExportResponse)); + marcRecord.getDataFields().addAll(new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile)); } else { - marcRecord.getDataFields().addAll(new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile, oleNGBatchExportResponse)); + marcRecord.getDataFields().addAll(new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile)); } } catch (Exception e) { e.printStackTrace(); @@ -146,7 +148,7 @@ private void processDataMappings(String bibId, Record marcRecord, List dataTransformerList = batchProcessProfile.getBatchProfileDataTransformerList(); for (BatchProfileDataTransformer dataTransformer : dataTransformerList) { try { diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportHoldingsDaoCallable.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportHoldingsDaoCallable.java index 364731ea1c..79f7177757 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportHoldingsDaoCallable.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportHoldingsDaoCallable.java @@ -41,9 +41,9 @@ public Object call() throws Exception { List dataFields = new ArrayList<>(); try { if (holdingsTree.getHoldings().getHoldingsType().equalsIgnoreCase(OleNGConstants.PRINT)) { - dataFields = new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile, oleNGBatchExportResponse); + dataFields = new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile); } else { - dataFields = new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile, oleNGBatchExportResponse); + dataFields = new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile); } } catch (Exception e) { e.printStackTrace(); diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportEholdingsMappingHelper.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportEholdingsMappingHelper.java index 9278b5d8f5..df07d0c3d7 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportEholdingsMappingHelper.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportEholdingsMappingHelper.java @@ -30,7 +30,7 @@ public class ExportEholdingsMappingHelper extends ExportHoldingsMappingHelper { private static final Logger LOG = Logger.getLogger(InstanceMappingHelper.class); private HoldingOlemlRecordProcessor workEHoldingOlemlRecordProcessor = new HoldingOlemlRecordProcessor(); - public List generateDataFieldForEHolding(HoldingsTree holdingsTree, BatchProcessProfile profile, OleNGBatchExportResponse oleNGBatchExportResponse) throws Exception { + public List generateDataFieldForEHolding(HoldingsTree holdingsTree, BatchProcessProfile profile) throws Exception { dataFieldList.clear(); if (holdingsTree != null) { OleHoldings oleHoldings = null; diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportHoldingsMappingHelper.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportHoldingsMappingHelper.java index 4b7434acae..e5cd934714 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportHoldingsMappingHelper.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/helper/ExportHoldingsMappingHelper.java @@ -38,7 +38,7 @@ public class ExportHoldingsMappingHelper { private static final Logger LOG = Logger.getLogger(ExportHoldingsMappingHelper.class); - public List generateDataFieldForHolding(HoldingsTree holdingsTree, BatchProcessProfile profile, OleNGBatchExportResponse oleNGBatchExportResponse) throws Exception { + public List generateDataFieldForHolding(HoldingsTree holdingsTree, BatchProcessProfile profile) throws Exception { dataFieldList.clear(); Item item = null; if (holdingsTree != null) { diff --git a/ole-common/ole-utility/src/main/java/org/kuali/ole/constants/OleNGConstants.java b/ole-common/ole-utility/src/main/java/org/kuali/ole/constants/OleNGConstants.java index c8ce7d8cbf..8b70c221b5 100644 --- a/ole-common/ole-utility/src/main/java/org/kuali/ole/constants/OleNGConstants.java +++ b/ole-common/ole-utility/src/main/java/org/kuali/ole/constants/OleNGConstants.java @@ -423,4 +423,7 @@ public static final class BatchProcess { public static final String DESC_NMSPC = "OLE-DESC"; public static final String APPL_ID_OLE = "OLE"; + + public static final String SOLR_RECORD_NOT_SYNC_WITH_DB_RECORD = "Solr record is not sync with database"; + } From 1155cfa2a19d183dddb4e2b412d57370cba9ad57 Mon Sep 17 00:00:00 2001 From: sheiksalahudeen Date: Wed, 21 Sep 2016 09:59:33 -0400 Subject: [PATCH 2/2] OLE-9004 : Refactored the code. --- .../dao/export/ExportBibIdFinderCallable.java | 33 +- .../ole/oleng/dao/export/ExportCallable.java | 581 +++++++++++++++++ .../kuali/ole/oleng/dao/export/ExportDao.java | 51 +- .../dao/export/ExportDaoCallableImpl.java | 586 +----------------- .../export/IncrementalExportCallableImpl.java | 30 + .../ole/oleng/handler/BatchExportHandler.java | 10 +- .../kuali/ole/oleng/util/BatchExportUtil.java | 16 + .../incubator/SolrRequestReponseHandler.java | 10 +- .../SolrRequestReponseHandlerTest.java | 28 + 9 files changed, 738 insertions(+), 607 deletions(-) create mode 100644 ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportCallable.java create mode 100644 ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/IncrementalExportCallableImpl.java create mode 100644 ole-common/ole-solr-client/src/test/java/org/kuali/incubator/SolrRequestReponseHandlerTest.java diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java index d35236e021..1a2e4643a0 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportBibIdFinderCallable.java @@ -1,14 +1,8 @@ package org.kuali.ole.oleng.dao.export; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.kuali.ole.DocumentUniqueIDPrefix; -import org.kuali.ole.constants.OleNGConstants; import org.kuali.ole.oleng.handler.BatchExportHandler; +import org.kuali.ole.oleng.util.BatchExportUtil; -import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.concurrent.Callable; @@ -21,6 +15,7 @@ public class ExportBibIdFinderCallable implements Callable { private int start; private int chunkSize; private BatchExportHandler batchExportHandler; + private BatchExportUtil batchExportUtil; public ExportBibIdFinderCallable(String query, int start, int chunkSize, BatchExportHandler batchExportHandler) { this.query = query; @@ -32,18 +27,18 @@ public ExportBibIdFinderCallable(String query, int start, int chunkSize, BatchEx @Override public Object call() throws Exception { - Set bibIdentifiers = new HashSet<>(); - SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); - if (solrDocumentList.size() > 0) { - for (SolrDocument solrDocument : solrDocumentList) { - if (solrDocument.containsKey(OleNGConstants.BIB_IDENTIFIER)) { - List bibIds = (List) solrDocument.getFieldValue(OleNGConstants.BIB_IDENTIFIER); - for (String bibId : bibIds) { - bibIdentifiers.add(DocumentUniqueIDPrefix.getDocumentId(bibId)); - } - } - } - } + Set bibIdentifiers = getBatchExportUtil().getBibIdentifiersForQuery(query, start, chunkSize); return bibIdentifiers; } + + public BatchExportUtil getBatchExportUtil() { + if(null == batchExportUtil) { + batchExportUtil = new BatchExportUtil(); + } + return batchExportUtil; + } + + public void setBatchExportUtil(BatchExportUtil batchExportUtil) { + this.batchExportUtil = batchExportUtil; + } } diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportCallable.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportCallable.java new file mode 100644 index 0000000000..887f54f953 --- /dev/null +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportCallable.java @@ -0,0 +1,581 @@ +package org.kuali.ole.oleng.dao.export; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.kuali.ole.DocumentUniqueIDPrefix; +import org.kuali.ole.constants.OleNGConstants; +import org.kuali.ole.docstore.common.document.*; +import org.kuali.ole.docstore.common.document.HoldingsTree; +import org.kuali.ole.docstore.common.document.content.instance.*; +import org.kuali.ole.docstore.common.response.OleNGBatchExportResponse; +import org.kuali.ole.docstore.util.RebuildIndexUtil; +import org.kuali.ole.oleng.batch.process.model.BatchProcessTxObject; +import org.kuali.ole.oleng.batch.profile.model.BatchProcessProfile; +import org.kuali.ole.oleng.batch.profile.model.BatchProfileDataTransformer; +import org.kuali.ole.oleng.handler.BatchExportHandler; +import org.kuali.ole.oleng.helper.ExportEholdingsMappingHelper; +import org.kuali.ole.oleng.helper.ExportHoldingsMappingHelper; +import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader; +import org.marc4j.marc.Record; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.rowset.SqlRowSet; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.Callable; + +/** + * Created by sheiks on 20/09/16. + */ +public abstract class ExportCallable implements Callable { + protected static Map callNumberType = new HashMap<>(); + protected static Map receiptStatus = new HashMap<>(); + protected static Map authenticationType = new HashMap<>(); + protected static Map itemTypeMap = new HashMap<>(); + protected static Map itemStatusMap = new HashMap<>(); + protected static Map statisticalSearchCodeMap = new HashMap<>(); + protected static Map extentOfOwnershipTypeMap = new HashMap<>(); + + protected PlatformTransactionManager transactionManager; + protected String bibQuery = "SELECT * FROM OLE_DS_BIB_T WHERE BIB_ID "; + protected JdbcTemplate jdbcTemplate; + protected String bibIdsString; + protected int fileNumber; + protected OleNGBatchExportResponse oleNGBatchExportResponse; + protected BatchExportHandler batchExportHandler; + protected BatchProcessTxObject batchProcessTxObject; + protected List bibIds; + + public ExportCallable(Map> commomFields, JdbcTemplate jdbcTemplate, int fileNumber, BatchExportHandler batchExportHandler, + BatchProcessTxObject batchProcessTxObject) { + this.jdbcTemplate = jdbcTemplate; + this.callNumberType = commomFields.get("callNumberType"); + this.receiptStatus = commomFields.get("receiptStatus"); + this.authenticationType = commomFields.get("authenticationType"); + this.itemTypeMap = commomFields.get("itemTypeMap"); + this.itemStatusMap = commomFields.get("itemStatusMap"); + this.statisticalSearchCodeMap = commomFields.get("statisticalSearchCodeMap"); + this.extentOfOwnershipTypeMap = commomFields.get("extentOfOwnershipTypeMap"); + this.fileNumber = fileNumber; + this.batchExportHandler = batchExportHandler; + this.batchProcessTxObject = batchProcessTxObject; + this.oleNGBatchExportResponse = new OleNGBatchExportResponse(); + } + + + public Object processRecords() { + final TransactionTemplate template = new TransactionTemplate(getTransactionManager()); + final SqlRowSet bibResultSet = this.jdbcTemplate.queryForRowSet(bibQuery + " IN ( " + bibIdsString + " )"); + try { + template.execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + List marcRecords = new ArrayList<>(); + Bib bib = null; + List holdingsTreeList = null; + List marcRecord = null; + int numberOfSuccesssRecord = 0; + int numberOfFailureRecord = 0; + while (bibResultSet.next()) { + try { + bibIds.remove(bibResultSet.getString(1)); + bib = fetchBibRecord(bibResultSet); + holdingsTreeList = fetchHoldingsTreeForBib(Integer.parseInt(bib.getLocalId())); + marcRecord = batchExportHandler.getMarcRecordUtil().convertMarcXmlContentToMarcRecord(bib.getContent()); + processDataMappings(bib.getId(), marcRecord.get(0), holdingsTreeList, batchProcessTxObject.getBatchProcessProfile()); + processDataTransformations(bib.getId(), marcRecord.get(0), batchProcessTxObject.getBatchProcessProfile(), batchExportHandler); + numberOfSuccesssRecord++; + oleNGBatchExportResponse.addSuccessRecord(bib.getLocalId(), bib.getId(), OleNGConstants.SUCCESS); + marcRecords.addAll(marcRecord); + + if (bib.isStaffOnly()) { + // For Incremental Except Staff Only + oleNGBatchExportResponse.getDeletedBibIds().add(bib.getId()); + } + } catch (Exception e) { + e.printStackTrace(); + numberOfFailureRecord++; + oleNGBatchExportResponse.addFailureRecord(bib.getLocalId(), bib.getId(), e.getMessage()); + batchExportHandler.addBatchExportFailureResponseToExchange(e, bib.getId(), batchProcessTxObject.getExchangeObjectForBatchExport()); + } + } + preparedReportForUnSyncRecords(bibIds, oleNGBatchExportResponse); + oleNGBatchExportResponse.addNoOfSuccessRecords(numberOfSuccesssRecord); + oleNGBatchExportResponse.addNoOfFailureRecords(numberOfFailureRecord + bibIds.size()); + batchExportHandler.generateFileForMarcRecords(fileNumber, marcRecords, batchProcessTxObject); + return oleNGBatchExportResponse; + } + }); + } catch (Exception ex) { + throw ex; + } finally { + this.transactionManager = null; + + } + return oleNGBatchExportResponse; + } + + private void preparedReportForUnSyncRecords(List bibIds, OleNGBatchExportResponse oleNGBatchExportResponse) { + if(CollectionUtils.isNotEmpty(bibIds)) { + for (Iterator iterator = bibIds.iterator(); iterator.hasNext(); ) { + String bibId = iterator.next(); + String bibIdWithPrefix = DocumentUniqueIDPrefix.getPrefixedId(DocumentUniqueIDPrefix.PREFIX_WORK_BIB_MARC, bibId); + oleNGBatchExportResponse.addFailureRecord(bibId, bibIdWithPrefix, OleNGConstants.SOLR_RECORD_NOT_SYNC_WITH_DB_RECORD); + } + } + } + + private void processDataMappings(String bibId, Record marcRecord, List holdingsTreeList, BatchProcessProfile batchProcessProfile) { + for (HoldingsTree holdingsTree : holdingsTreeList) { + try { + if (holdingsTree.getHoldings().getHoldingsType().equalsIgnoreCase(OleNGConstants.PRINT)) { + marcRecord.getDataFields().addAll(new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile)); + } else { + marcRecord.getDataFields().addAll(new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile)); + } + } catch (Exception e) { + e.printStackTrace(); + batchExportHandler.addBatchExportFailureResponseToExchange(e, bibId, batchProcessTxObject.getExchangeObjectForBatchExport()); + } + } + } + + private void processDataTransformations(String bibId, Record marcRecord, BatchProcessProfile batchProcessProfile, BatchExportHandler batchExportHandler) { + List dataTransformerList = batchProcessProfile.getBatchProfileDataTransformerList(); + for (BatchProfileDataTransformer dataTransformer : dataTransformerList) { + try { + if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.DELETE_FIELD)) { + batchExportHandler.getMarcRecordUtil().removeFieldFromRecord(marcRecord, dataTransformer.getDataField()); + } else if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.DELETE_SUBFIELD)) { + batchExportHandler.getMarcRecordUtil().removeSubField(marcRecord, dataTransformer.getDataField(), dataTransformer.getInd1(), dataTransformer.getInd2(), dataTransformer.getSubField()); + } else if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.CUT_PASTE)) { + batchExportHandler.getMarcRecordUtil().renameDataField(marcRecord, dataTransformer.getDataField(), dataTransformer.getDestDataField()); + } + } catch (Exception e) { + e.printStackTrace(); + batchExportHandler.addBatchExportFailureResponseToExchange(e, bibId, batchProcessTxObject.getExchangeObjectForBatchExport()); + } + } + } + + private Bib fetchBibRecord(SqlRowSet bibResultSet) throws SQLException { + Bib bib = new BibMarc(); + bib.setCreatedBy(bibResultSet.getString("CREATED_BY")); + bib.setCreatedOn(bibResultSet.getString("DATE_CREATED")); + if (bibResultSet.getString("STAFF_ONLY") != null) { + bib.setStaffOnly((bibResultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); + } + bib.setContent(bibResultSet.getString("CONTENT")); + bib.setUpdatedBy(bibResultSet.getString("UPDATED_BY")); + bib.setUpdatedOn(bibResultSet.getString("DATE_UPDATED")); + bib.setStatus(bibResultSet.getString("STATUS")); + bib.setStatusUpdatedBy(bibResultSet.getString("STATUS_UPDATED_BY")); + bib.setStatusUpdatedOn(bibResultSet.getString("STATUS_UPDATED_DATE")); + bib.setLastUpdated(bibResultSet.getString("DATE_UPDATED")); + String uuid = bibResultSet.getString("UNIQUE_ID_PREFIX") + "-" + bibResultSet.getString(1); + bib.setId(uuid); + bib.setLocalId(bibResultSet.getString(1)); + return bib; + } + + + private List fetchHoldingsTreeForBib(int bibId) throws Exception { + List holdingsTrees = new ArrayList(); + String holdingsQuery = getHoldingsQuery(bibId); + SqlRowSet resultSet = jdbcTemplate.queryForRowSet(holdingsQuery); + Map map = new HashMap<>(); + Map extentOfOwnershipMap = new HashMap<>(); + Set uriSet = null; + Set noteSet = null; + Set donorSet = null; + Set coverageSet = null; + Set perpetualSet = null; + Set extentOfOwnershipNoteSet = null; + Set linkSet = null; + + while (resultSet.next()) { + String holdingsType = resultSet.getString("HOLDINGS_TYPE"); + String id = resultSet.getString("HOLDINGS_ID"); + if (id == null) { + continue; + } + OleHoldings oleHoldings = null; + if (map.containsKey(id)) { + oleHoldings = map.get(id).getHoldings().getContentObject(); + } else { + oleHoldings = new OleHoldings(); + Location location = RebuildIndexUtil.getLocationDetails(resultSet.getString("LOCATION"), resultSet.getString("LOCATION_LEVEL")); + oleHoldings.setLocation(location); + oleHoldings.setHoldingsType(holdingsType); + CallNumber callNumber = new CallNumber(); + callNumber.setNumber(resultSet.getString("CALL_NUMBER")); + callNumber.setPrefix(resultSet.getString("CALL_NUMBER_PREFIX")); + ShelvingOrder shelvingOrder = new ShelvingOrder(); + if (resultSet.getString("SHELVING_ORDER") != null && !"null".equals(resultSet.getString("SHELVING_ORDER"))) { + shelvingOrder.setCodeValue(resultSet.getString("SHELVING_ORDER")); + shelvingOrder.setFullValue(resultSet.getString("SHELVING_ORDER")); + } + callNumber.setShelvingOrder(shelvingOrder); + ShelvingScheme shelvingScheme = new ShelvingScheme(); + if (resultSet.getString("CALL_NUMBER_TYPE_ID") != null) { + String[] strings = callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")).split("[|]"); + shelvingScheme.setCodeValue(strings[0]); + shelvingScheme.setFullValue(strings[1]); + } + callNumber.setShelvingScheme(shelvingScheme); + oleHoldings.setCallNumber(callNumber); + oleHoldings.setCopyNumber(resultSet.getString("COPY_NUMBER")); + HoldingsTree holdingsTree = new HoldingsTree(); + + Holdings holdings = null; + + if (holdingsType.equalsIgnoreCase(PHoldings.PRINT)) { + holdings = new PHoldings(); + + if (resultSet.getString("RECEIPT_STATUS_ID") != null) { + oleHoldings.setReceiptStatus(resultSet.getString("RECEIPT_STATUS_ID")); + } + extentOfOwnershipNoteSet = new HashSet<>(); + donorSet = coverageSet = perpetualSet = null; + } else { + holdings = new EHoldings(); + oleHoldings.setAccessStatus(resultSet.getString("ACCESS_STATUS")); + oleHoldings.setImprint(resultSet.getString("IMPRINT")); + Platform platform = new Platform(); + platform.setPlatformName(resultSet.getString("PLATFORM")); + platform.setAdminUrl(resultSet.getString("ADMIN_URL")); + platform.setAdminUserName(resultSet.getString("ADMIN_USERNAME")); + platform.setAdminPassword(resultSet.getString("ADMIN_PASSWORD")); + oleHoldings.setPlatform(platform); + + oleHoldings.setPublisher(resultSet.getString("PUBLISHER")); + HoldingsAccessInformation holdingsAccessInformation = new HoldingsAccessInformation(); + + holdingsAccessInformation.setProxiedResource(resultSet.getString("PROXIED_RESOURCE")); + holdingsAccessInformation.setAccessUsername(resultSet.getString("ACCESS_USERNAME")); + holdingsAccessInformation.setAccessPassword(resultSet.getString("ACCESS_PASSWORD")); + holdingsAccessInformation.setNumberOfSimultaneousUser(resultSet.getString("NUMBER_SIMULT_USERS")); + holdingsAccessInformation.setAccessLocation(resultSet.getString("CODE")); + holdingsAccessInformation.setAuthenticationType(resultSet.getString("AUTHENTICATION_TYPE_ID")); + oleHoldings.setHoldingsAccessInformation(holdingsAccessInformation); + String statisticalSearchId = resultSet.getString("STAT_SEARCH_CODE_ID"); + if (StringUtils.isNotEmpty(statisticalSearchId)) { + String[] strings = statisticalSearchCodeMap.get(statisticalSearchId).split("[|]"); + StatisticalSearchingCode statisticalSearchingCode = new StatisticalSearchingCode(); + statisticalSearchingCode.setCodeValue(strings[0]); + statisticalSearchingCode.setFullValue(strings[1]); + oleHoldings.setStatisticalSearchingCode(statisticalSearchingCode); + } + oleHoldings.setLocalPersistentLink(resultSet.getString("LOCAL_PERSISTENT_URI")); + oleHoldings.setSubscriptionStatus(resultSet.getString("SUBSCRIPTION_STATUS")); + oleHoldings.setInterLibraryLoanAllowed(Boolean.valueOf(resultSet.getString("ALLOW_ILL"))); + coverageSet = new HashSet<>(); + perpetualSet = new HashSet<>(); + donorSet = new HashSet<>(); + ExtentOfOwnership extentOfOwnership = new ExtentOfOwnership(); + Coverages coverages = new Coverages(); + PerpetualAccesses perpetualAccesses = new PerpetualAccesses(); + extentOfOwnership.setCoverages(coverages); + extentOfOwnership.setPerpetualAccesses(perpetualAccesses); + oleHoldings.getExtentOfOwnership().add(extentOfOwnership); + + } + holdings.setHoldingsType(holdingsType); + holdings.setId("who-" + id); + holdings.setContentObject(oleHoldings); + holdings.setCreatedBy(resultSet.getString("CREATED_BY")); + holdings.setCreatedOn(resultSet.getString("DATE_CREATED")); + if (resultSet.getString("STAFF_ONLY") != null) { + holdings.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); + } + holdings.setUpdatedBy(resultSet.getString("UPDATED_BY")); + holdings.setUpdatedOn(resultSet.getString("DATE_UPDATED")); + holdings.setLastUpdated(resultSet.getString("DATE_UPDATED")); + + uriSet = new HashSet<>(); + noteSet = new HashSet<>(); + linkSet = new HashSet<>(); + + holdingsTree.setHoldings(holdings); + if (StringUtils.isNotBlank(id)) { + List itemList = fetchItemForHoldings(Integer.parseInt(id)); + holdingsTree.getItems().addAll(itemList); + } + map.put(id, holdingsTree); + + holdingsTrees.add(holdingsTree); + } + + if (StringUtils.isNotEmpty(holdingsType) && holdingsType.equalsIgnoreCase(PHoldings.PRINT)) { + if (uriSet.add(resultSet.getString("HOLDINGS_URI_ID"))) { + Uri uri = new Uri(); + uri.setValue(resultSet.getString("TEXT")); + oleHoldings.getUri().add(uri); + } + ExtentOfOwnership extentOfOwnership = null; + if (extentOfOwnershipMap.containsKey(resultSet.getString("EXT_OWNERSHIP_ID"))) { + extentOfOwnership = extentOfOwnershipMap.get(resultSet.getString("EXT_OWNERSHIP_ID")); + } else { + extentOfOwnership = new ExtentOfOwnership(); + if (StringUtils.isNotEmpty(resultSet.getString("EXT_OWNERSHIP_TYPE_ID"))) { + String[] strings = extentOfOwnershipTypeMap.get(resultSet.getString("EXT_OWNERSHIP_TYPE_ID")).split("[|]"); + extentOfOwnership.setType(strings[1]); + } + extentOfOwnershipMap.put(resultSet.getString("EXT_OWNERSHIP_ID"), extentOfOwnership); + oleHoldings.getExtentOfOwnership().add(extentOfOwnership); + } + String extOwnershipNoteId = resultSet.getString("EXT_OWNERSHIP_NOTE_ID"); + if (extentOfOwnershipNoteSet != null && StringUtils.isNotEmpty(extOwnershipNoteId) && extentOfOwnershipNoteSet.add(resultSet.getString("EXT_OWNERSHIP_NOTE_ID"))) { + Note note = new Note(); + note.setValue((String) resultSet.getString("EXNOTE")); + note.setType((String) resultSet.getString("EXTYPE")); + + extentOfOwnership.getNote().add(note); + } + } else { + if (linkSet.add(resultSet.getString("HOLDINGS_URI_ID"))) { + Link link = new Link(); + link.setUrl(resultSet.getString("URI")); + link.setText(resultSet.getString("TEXT")); + oleHoldings.getLink().add(link); + } + if (oleHoldings.getExtentOfOwnership() != null && oleHoldings.getExtentOfOwnership().size() > 0) { + if (coverageSet != null && coverageSet.add(resultSet.getString("HOLDINGS_COVERAGE_ID"))) { + Coverage coverage = new Coverage(); + coverage.setCoverageStartIssue(resultSet.getString("COVERAGE_START_ISSUE")); + coverage.setCoverageStartDate(resultSet.getString("COVERAGE_START_DATE")); + coverage.setCoverageStartVolume(resultSet.getString("COVERAGE_START_VOLUME")); + coverage.setCoverageEndIssue(resultSet.getString("HOLDINGS_COVERAGE_ID")); + coverage.setCoverageEndDate(resultSet.getString("COVERAGE_END_DATE")); + coverage.setCoverageEndVolume(resultSet.getString("COVERAGE_END_VOLUME")); + oleHoldings.getExtentOfOwnership().get(0).getCoverages().getCoverage().add(coverage); + } + + if (perpetualSet != null && perpetualSet.add(resultSet.getString("HOLDINGS_PERPETUAL_ACCESS_ID"))) { + PerpetualAccess perpetualAccess = new PerpetualAccess(); + perpetualAccess.setPerpetualAccessStartDate(resultSet.getString("PERPETUAL_ACCESS_START_DATE")); + perpetualAccess.setPerpetualAccessStartIssue(resultSet.getString("PERPETUAL_ACCESS_START_ISSUE")); + perpetualAccess.setPerpetualAccessStartVolume(resultSet.getString("PERPETUAL_ACCESS_START_VOLUME")); + perpetualAccess.setPerpetualAccessEndDate(resultSet.getString("PERPETUAL_ACCESS_END_DATE")); + perpetualAccess.setPerpetualAccessEndVolume(resultSet.getString("PERPETUAL_ACCESS_END_VOLUME")); + perpetualAccess.setPerpetualAccessEndIssue(resultSet.getString("PERPETUAL_ACCESS_END_ISSUE")); + oleHoldings.getExtentOfOwnership().get(0).getPerpetualAccesses().getPerpetualAccess().add(perpetualAccess); + } + } + if (donorSet != null && donorSet.add(resultSet.getString("HOLDINGS_DONOR_ID"))) { + DonorInfo donorInfo = new DonorInfo(); + donorInfo.setDonorCode(resultSet.getString("DONOR_CODE")); + donorInfo.setDonorNote(resultSet.getString("DONOR_NOTE")); + donorInfo.setDonorPublicDisplay(resultSet.getString("DONOR_DISPLAY_NOTE")); + oleHoldings.getDonorInfo().add(donorInfo); + } + } + + if (noteSet.add(resultSet.getString("HOLDINGS_NOTE_ID"))) { + Note note = new Note(); + note.setValue(resultSet.getString("NOTE")); + note.setType(resultSet.getString("TYPE")); + oleHoldings.getNote().add(note); + } + } + + return holdingsTrees; + } + + public List fetchItemForHoldings(int holdingsId) throws Exception { + List itemList = new ArrayList(); + Map itemHashMap = new HashMap<>(); + String itemQuery = getItemQuery(holdingsId); + SqlRowSet resultSet = jdbcTemplate.queryForRowSet(itemQuery); + Set highDensityStorageSet = null; + Set donorNoteSet = null; + Set itemNoteSet = null; + Set statisticalSearchSet = null; + org.kuali.ole.docstore.common.document.content.instance.Item itemObj = null; + + while (resultSet.next()) { + String id = resultSet.getString("ITEM_ID"); + if (id == null) { + continue; + } + if (itemHashMap.containsKey(id)) { + itemObj = (org.kuali.ole.docstore.common.document.content.instance.Item) itemHashMap.get(id).getContentObject(); + } else { + itemObj = new org.kuali.ole.docstore.common.document.content.instance.Item(); + org.kuali.ole.docstore.common.document.Item item = new org.kuali.ole.docstore.common.document.Item(); + itemList.add(item); + item.setId("wio-" + resultSet.getString("ITEM_ID")); + item.setContentObject(itemObj); + Location location = RebuildIndexUtil.getLocationDetails(resultSet.getString("LOCATION"), resultSet.getString("LOCATION_LEVEL")); + itemObj.setLocation(location); + CallNumber callNumber = new CallNumber(); + callNumber.setNumber(resultSet.getString("CALL_NUMBER")); + callNumber.setPrefix(resultSet.getString("CALL_NUMBER_PREFIX")); + ShelvingOrder shelvingOrder = new ShelvingOrder(); + if (resultSet.getString("SHELVING_ORDER") != null && !"null".equals(resultSet.getString("SHELVING_ORDER"))) { + shelvingOrder.setCodeValue(resultSet.getString("SHELVING_ORDER")); + shelvingOrder.setFullValue(resultSet.getString("SHELVING_ORDER")); + } + callNumber.setShelvingOrder(shelvingOrder); + ShelvingScheme shelvingScheme = new ShelvingScheme(); + if (callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")) != null) { + String[] strings = callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")).split("[|]"); + shelvingScheme.setCodeValue(strings[0]); + shelvingScheme.setFullValue(strings[1]); + } + callNumber.setShelvingScheme(shelvingScheme); + itemObj.setCallNumber(callNumber); + itemObj.setBarcodeARSL(resultSet.getString("BARCODE_ARSL")); + itemObj.setEnumeration(resultSet.getString("ENUMERATION")); + itemObj.setChronology(resultSet.getString("CHRONOLOGY")); + itemObj.setCopyNumber(resultSet.getString("COPY_NUMBER")); + AccessInformation accessInformation = new AccessInformation(); + accessInformation.setBarcode(resultSet.getString("BARCODE")); + Uri uri = new Uri(); + uri.setValue(resultSet.getString("URI")); + accessInformation.setUri(uri); + itemObj.setAccessInformation(accessInformation); + itemObj.setPurchaseOrderLineItemIdentifier(resultSet.getString("PURCHASE_ORDER_LINE_ITEM_ID")); + itemObj.setVendorLineItemIdentifier(resultSet.getString("VENDOR_LINE_ITEM_ID")); + itemObj.setFund(resultSet.getString("FUND")); + itemObj.setPrice(resultSet.getString("PRICE")); + itemObj.setItemStatusEffectiveDate(RebuildIndexUtil.convertDateFormat(resultSet.getString("ITEM_STATUS_DATE_UPDATED"))); + if (resultSet.getString("FAST_ADD") != null) { + itemObj.setFastAddFlag(resultSet.getString("FAST_ADD").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); + } + itemObj.setCheckinNote(resultSet.getString("CHECK_IN_NOTE")); + if (resultSet.getString("CLAIMS_RETURNED") != null) { + itemObj.setClaimsReturnedFlag(resultSet.getString("CLAIMS_RETURNED").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); + } + String claimsReturnFlagCreatedDate = resultSet.getString("CLAIMS_RETURNED_DATE_CREATED"); + itemObj.setClaimsReturnedFlagCreateDate(RebuildIndexUtil.convertDateFormat(claimsReturnFlagCreatedDate)); + itemObj.setClaimsReturnedNote(resultSet.getString("CLAIMS_RETURNED_NOTE")); + itemObj.setCurrentBorrower(resultSet.getString("CURRENT_BORROWER")); + itemObj.setProxyBorrower(resultSet.getString("PROXY_BORROWER")); + String dueDateTime = resultSet.getString("DUE_DATE_TIME"); + itemObj.setDueDateTime(RebuildIndexUtil.convertDateFormat(dueDateTime)); + String originalDueDate = resultSet.getString("ORG_DUE_DATE_TIME"); + itemObj.setOriginalDueDate(RebuildIndexUtil.convertDateFormat(originalDueDate)); + String checkOutDateTime = resultSet.getString("CHECK_OUT_DATE_TIME"); + itemObj.setCheckOutDateTime(RebuildIndexUtil.convertDateFormat(checkOutDateTime)); + itemObj.setDamagedItemNote(resultSet.getString("ITEM_DAMAGED_NOTE")); + if (resultSet.getString("ITEM_DAMAGED_STATUS") != null) { + itemObj.setItemDamagedStatus(resultSet.getString("ITEM_DAMAGED_STATUS").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); + } + if (resultSet.getString("MISSING_PIECES") != null) { + itemObj.setMissingPieceFlag(resultSet.getString("MISSING_PIECES").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); + } + itemObj.setMissingPiecesCount(resultSet.getString("MISSING_PIECES_COUNT")); + itemObj.setMissingPieceFlagNote(resultSet.getString("MISSING_PIECES_NOTE")); + itemObj.setMissingPieceEffectiveDate(resultSet.getString("MISSING_PIECES_EFFECTIVE_DATE")); + itemObj.setNumberOfPieces(resultSet.getString("NUM_PIECES")); + itemObj.setDescriptionOfPieces(resultSet.getString("DESC_OF_PIECES")); + itemObj.setNumberOfRenew(resultSet.getInt("NUM_OF_RENEW")); + highDensityStorageSet = new HashSet<>(); + itemNoteSet = new HashSet<>(); + statisticalSearchSet = new HashSet<>(); + donorNoteSet = new HashSet<>(); + + ItemStatus itemStatus = new ItemStatus(); + if (itemStatusMap.containsKey(resultSet.getString("ITEM_STATUS_ID"))) { + String[] strings = itemStatusMap.get(resultSet.getString("ITEM_STATUS_ID")).split("[|]"); + itemStatus.setCodeValue(strings[0]); + itemStatus.setFullValue(strings[1]); + } + + itemObj.setItemStatus(itemStatus); + ItemType itemType = new ItemType(); + if (itemTypeMap.containsKey(resultSet.getString("ITEM_TYPE_ID"))) { + String[] strings = itemTypeMap.get(resultSet.getString("ITEM_TYPE_ID")).split("[|]"); + itemType.setCodeValue(strings[0]); + itemType.setFullValue(strings[1]); + } + itemObj.setItemType(itemType); + ItemType tempItemType = new ItemType(); + if (itemTypeMap.containsKey(resultSet.getString("TEMP_ITEM_TYPE_ID"))) { + String[] strings = itemTypeMap.get(resultSet.getString("TEMP_ITEM_TYPE_ID")).split("[|]"); + tempItemType.setCodeValue(strings[0]); + tempItemType.setFullValue(strings[1]); + } + itemObj.setTemporaryItemType(tempItemType); + item.setContentObject(itemObj); + item.setCreatedBy(resultSet.getString("CREATED_BY")); + item.setCreatedOn(resultSet.getString("DATE_CREATED")); + if (resultSet.getString("STAFF_ONLY") != null) { + item.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); + } + item.setUpdatedBy(resultSet.getString("UPDATED_BY")); + item.setUpdatedOn(resultSet.getString("DATE_UPDATED")); + item.setLastUpdated(resultSet.getString("DATE_UPDATED")); + + } + if (itemNoteSet.add(resultSet.getString("ITEM_NOTE_ID"))) { + Note note = new Note(); + note.setValue(resultSet.getString("NOTE")); + note.setType(resultSet.getString("TYPE")); + itemObj.getNote().add(note); + } + if (resultSet.getString("STAT_SEARCH_CODE_ID") != null && statisticalSearchSet.add(resultSet.getString("STAT_SEARCH_CODE_ID"))) { + String[] strings = statisticalSearchCodeMap.get(resultSet.getString("STAT_SEARCH_CODE_ID")).split("[|]"); + StatisticalSearchingCode statisticalSearchingCode = new StatisticalSearchingCode(); + statisticalSearchingCode.setCodeValue(strings[0]); + statisticalSearchingCode.setFullValue(strings[1]); + itemObj.getStatisticalSearchingCode().add(statisticalSearchingCode); + } + if (donorNoteSet.add(resultSet.getString("ITEM_DONOR_ID"))) { + DonorInfo donorInfo = new DonorInfo(); + donorInfo.setDonorCode(resultSet.getString("DONOR_CODE")); + donorInfo.setDonorNote(resultSet.getString("DONOR_NOTE")); + donorInfo.setDonorPublicDisplay(resultSet.getString("DONOR_DISPLAY_NOTE")); + itemObj.getDonorInfo().add(donorInfo); + } + if (highDensityStorageSet.add(resultSet.getString("ITEM_DONOR_ID"))) { + HighDensityStorage highDensityStorage = new HighDensityStorage(); + highDensityStorage.setRow(resultSet.getString("HIGH_DENSITY_ROW")); + itemObj.setHighDensityStorage(highDensityStorage); + } + } + return itemList; + } + + private String getItemQuery(int holdingsId) { + return "SELECT I.*,N.ITEM_NOTE_ID,N.NOTE,N.TYPE, S.STAT_SEARCH_CODE_ID," + + "D.ITEM_DONOR_ID,D.DONOR_CODE,D.DONOR_DISPLAY_NOTE,D.DONOR_NOTE,HD.HIGH_DENSITY_ROW " + + "FROM ole_ds_item_t I " + + "LEFT JOIN ole_ds_item_donor_t D ON I.item_id=D.item_id " + + "LEFT JOIN ole_ds_item_note_t N ON I.item_id = N.item_id " + + "LEFT JOIN ole_ds_item_stat_search_t S ON I.item_id=S.item_id " + + "LEFT JOIN OLE_DS_HIGH_DENSITY_STORAGE_T HD ON I.HIGH_DENSITY_STORAGE_ID =HD.HIGH_DENSITY_STORAGE_ID " + + "WHERE I.HOLDINGS_ID= " + holdingsId; + } + + private String getHoldingsQuery(int bibId) { + return "SELECT H.* ,LOC.CODE,STAT.STAT_SEARCH_CODE_ID,URI.HOLDINGS_URI_ID,URI.TEXT,URI.URI,EXT.EXT_OWNERSHIP_ID,EXT.EXT_OWNERSHIP_TYPE_ID," + + "EXTN.EXT_OWNERSHIP_NOTE_ID,EXTN.NOTE AS EXNOTE,EXTN.TYPE AS EXTYPE,COV.HOLDINGS_COVERAGE_ID,COV.COVERAGE_START_ISSUE, COV.COVERAGE_START_DATE," + + "COV.COVERAGE_START_VOLUME,COV.COVERAGE_END_DATE,COV.COVERAGE_END_VOLUME,PA.HOLDINGS_PERPETUAL_ACCESS_ID,PA.PERPETUAL_ACCESS_START_DATE,PA.PERPETUAL_ACCESS_START_ISSUE," + + "PA.PERPETUAL_ACCESS_START_VOLUME,PA.PERPETUAL_ACCESS_END_DATE,PA.PERPETUAL_ACCESS_END_VOLUME,PA.PERPETUAL_ACCESS_END_ISSUE,DONOR.HOLDINGS_DONOR_ID, " + + "DONOR.DONOR_CODE,DONOR.DONOR_NOTE,DONOR.DONOR_DISPLAY_NOTE,NOTE.HOLDINGS_NOTE_ID,NOTE.NOTE,NOTE.TYPE " + + "FROM OLE_DS_HOLDINGS_T H " + + "LEFT JOIN ole_ds_holdings_uri_t URI ON H.HOLDINGS_ID=URI.HOLDINGS_ID " + + "LEFT JOIN ole_ds_holdings_note_t NOTE ON H.HOLDINGS_ID=NOTE.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_HOLDINGS_DONOR_T DONOR ON H.HOLDINGS_ID=DONOR.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_HOLDINGS_COVERAGE_T COV ON H.HOLDINGS_ID=COV.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_PERPETUAL_ACCESS_T PA ON H.HOLDINGS_ID=PA.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_HOLDINGS_STAT_SEARCH_T STAT ON H.HOLDINGS_ID=STAT.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_ACCESS_LOCATION_T ACC ON H.HOLDINGS_ID=ACC.HOLDINGS_ID " + + "LEFT JOIN OLE_DS_ACCESS_LOCATION_CODE_T LOC ON ACC.ACCESS_LOCATION_CODE_ID=LOC.ACCESS_LOCATION_CODE_ID " + + "LEFT JOIN ole_ds_ext_ownership_t ext ON H.HOLDINGS_ID=ext.holdings_id " + + "LEFT JOIN ole_ds_ext_ownership_note_t EXTN ON ext.EXT_OWNERSHIP_ID=EXTN.EXT_OWNERSHIP_ID WHERE H.BIB_ID= " + bibId; + } + + public PlatformTransactionManager getTransactionManager() { + if (transactionManager == null) { + transactionManager = GlobalResourceLoader.getService("transactionManager"); + } + return this.transactionManager; + } +} diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java index 6b485807c7..0a3f0e89e3 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDao.java @@ -53,7 +53,7 @@ private void init() throws Exception { } public void export(BatchExportHandler batchExportHandler, String query, BatchProcessTxObject batchProcessTxObject, - OleNGBatchExportResponse oleNGBatchExportResponse) { + OleNGBatchExportResponse oleNGBatchExportResponse, boolean isIncremental) { try { init(); this.chunkSize = batchExportHandler.getBatchChunkSize(); @@ -62,32 +62,47 @@ public void export(BatchExportHandler batchExportHandler, String query, BatchPro int fileCount = 1; int fileSize = batchProcessTxObject.getBatchJobDetails().getNumOfRecordsInFile(); int numOfRecordsInFile = 0; - SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); + SolrDocumentList solrDocumentList = batchExportHandler.getSolrRequestReponseHandler().getSolrDocumentList( + query, null, null, OleNGConstants.BIB_IDENTIFIER); totalCount = solrDocumentList.getNumFound(); - List bibIds = getBibIds(query, totalCount, chunkSize, batchExportHandler, batchProcessTxObject); + List bibIds = new ArrayList<>(); - totalCount = bibIds.size(); - - if(fileSize < chunkSize) { - chunkSize = fileSize; + if(isIncremental) { + bibIds = getBibIds(query, totalCount, chunkSize, batchExportHandler, batchProcessTxObject); + totalCount = bibIds.size(); + if(fileSize < chunkSize) { + chunkSize = fileSize; + } } batchProcessTxObject.getBatchJobDetails().setTotalRecords(String.valueOf(totalCount)); - oleNGBatchExportResponse.setTotalNumberOfRecords(((int)totalCount)); + oleNGBatchExportResponse.setTotalNumberOfRecords((int) totalCount); batchExportHandler.updateBatchJob(batchProcessTxObject.getBatchJobDetails()); - List> partition = Lists.partition(bibIds, chunkSize); - - for (Iterator> iterator = partition.iterator(); iterator.hasNext(); ) { - List bibIdLists = iterator.next(); - futures.add(executorService.submit(new ExportDaoCallableImpl(commonFields, getJdbcTemplate(),bibIdLists, - fileCount, batchExportHandler, batchProcessTxObject))); - numOfRecordsInFile += chunkSize; - if (numOfRecordsInFile >= fileSize) { - fileCount++; - numOfRecordsInFile = 0; + if(isIncremental) { + List> partition = Lists.partition(bibIds, chunkSize); + for (Iterator> iterator = partition.iterator(); iterator.hasNext(); ) { + List bibIdLists = iterator.next(); + futures.add(executorService.submit(new IncrementalExportCallableImpl(commonFields, getJdbcTemplate(),bibIdLists, + fileCount, batchExportHandler, batchProcessTxObject))); + numOfRecordsInFile += chunkSize; + if (numOfRecordsInFile >= fileSize) { + fileCount++; + numOfRecordsInFile = 0; + } } + } else { + do { + futures.add(executorService.submit(new ExportDaoCallableImpl(commonFields, getJdbcTemplate(), query, + start, chunkSize, fileCount, batchExportHandler, batchProcessTxObject))); + numOfRecordsInFile += chunkSize; + if (numOfRecordsInFile == fileSize) { + fileCount++; + numOfRecordsInFile = 0; + } + start += chunkSize; + } while (start <= totalCount); } prepareBatchExportResponse(futures, batchExportHandler, batchProcessTxObject, oleNGBatchExportResponse); executorService.shutdown(); diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java index fcfcc05c6d..b3639738a3 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/ExportDaoCallableImpl.java @@ -1,585 +1,47 @@ package org.kuali.ole.oleng.dao.export; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.kuali.ole.DocumentUniqueIDPrefix; -import org.kuali.ole.constants.OleNGConstants; -import org.kuali.ole.docstore.common.document.*; -import org.kuali.ole.docstore.common.document.HoldingsTree; -import org.kuali.ole.docstore.common.document.Item; -import org.kuali.ole.docstore.common.document.content.instance.*; -import org.kuali.ole.docstore.common.response.OleNGBatchExportResponse; -import org.kuali.ole.docstore.util.RebuildIndexUtil; import org.kuali.ole.oleng.batch.process.model.BatchProcessTxObject; -import org.kuali.ole.oleng.batch.profile.model.BatchProcessProfile; -import org.kuali.ole.oleng.batch.profile.model.BatchProfileDataTransformer; import org.kuali.ole.oleng.handler.BatchExportHandler; -import org.kuali.ole.oleng.helper.ExportEholdingsMappingHelper; -import org.kuali.ole.oleng.helper.ExportHoldingsMappingHelper; -import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader; -import org.marc4j.marc.Record; +import org.kuali.ole.oleng.util.BatchExportUtil; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.support.rowset.SqlRowSet; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.Callable; +import java.util.ArrayList; +import java.util.Map; +import java.util.Set; /** * Created by rajeshbabuk on 4/25/16. */ -public class ExportDaoCallableImpl implements Callable { +public class ExportDaoCallableImpl extends ExportCallable{ - public static Map callNumberType = new HashMap<>(); - public static Map receiptStatus = new HashMap<>(); - public static Map authenticationType = new HashMap<>(); - public static Map itemTypeMap = new HashMap<>(); - public static Map itemStatusMap = new HashMap<>(); - public static Map statisticalSearchCodeMap = new HashMap<>(); - public static Map extentOfOwnershipTypeMap = new HashMap<>(); + private String query; + private int start; + private int chunkSize; + private BatchExportUtil batchExportUtil; - - private PlatformTransactionManager transactionManager; - private String bibQuery = "SELECT * FROM OLE_DS_BIB_T WHERE BIB_ID "; - private JdbcTemplate jdbcTemplate; - private String bibIdsString; - private int fileNumber; - private OleNGBatchExportResponse oleNGBatchExportResponse; - private BatchExportHandler batchExportHandler; - private BatchProcessTxObject batchProcessTxObject; - private List bibIds; - - public ExportDaoCallableImpl(Map> commomFields, JdbcTemplate jdbcTemplate,List bibIds, int fileNumber, BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject) { - this.bibIds = new ArrayList<>(bibIds); - this.bibIdsString = StringUtils.join(bibIds, ','); - this.jdbcTemplate = jdbcTemplate; - this.callNumberType = commomFields.get("callNumberType"); - this.receiptStatus = commomFields.get("receiptStatus"); - this.authenticationType = commomFields.get("authenticationType"); - this.itemTypeMap = commomFields.get("itemTypeMap"); - this.itemStatusMap = commomFields.get("itemStatusMap"); - this.statisticalSearchCodeMap = commomFields.get("statisticalSearchCodeMap"); - this.extentOfOwnershipTypeMap = commomFields.get("extentOfOwnershipTypeMap"); - this.fileNumber = fileNumber; - this.batchExportHandler = batchExportHandler; - this.batchProcessTxObject = batchProcessTxObject; - this.oleNGBatchExportResponse = new OleNGBatchExportResponse(); + public ExportDaoCallableImpl(Map> commomFields, + JdbcTemplate jdbcTemplate, String query, int start, int chunkSize, int fileNumber, + BatchExportHandler batchExportHandler, BatchProcessTxObject batchProcessTxObject) { + super(commomFields, jdbcTemplate, fileNumber, batchExportHandler, batchProcessTxObject); + this.query = query; + this.start = start; + this.chunkSize = chunkSize; } @Override public Object call() throws Exception { - final TransactionTemplate template = new TransactionTemplate(getTransactionManager()); - final SqlRowSet bibResultSet = this.jdbcTemplate.queryForRowSet(bibQuery + " IN ( " + bibIdsString + " )"); - try { - template.execute(new TransactionCallback() { - @Override - public Object doInTransaction(TransactionStatus status) { - List marcRecords = new ArrayList<>(); - Bib bib = null; - List holdingsTreeList = null; - List marcRecord = null; - int numberOfSuccesssRecord = 0; - int numberOfFailureRecord = 0; - while (bibResultSet.next()) { - try { - bibIds.remove(bibResultSet.getString(1)); - bib = fetchBibRecord(bibResultSet); - holdingsTreeList = fetchHoldingsTreeForBib(Integer.parseInt(bib.getLocalId())); - marcRecord = batchExportHandler.getMarcRecordUtil().convertMarcXmlContentToMarcRecord(bib.getContent()); - processDataMappings(bib.getId(), marcRecord.get(0), holdingsTreeList, batchProcessTxObject.getBatchProcessProfile()); - processDataTransformations(bib.getId(), marcRecord.get(0), batchProcessTxObject.getBatchProcessProfile(), batchExportHandler); - numberOfSuccesssRecord++; - oleNGBatchExportResponse.addSuccessRecord(bib.getLocalId(), bib.getId(), OleNGConstants.SUCCESS); - marcRecords.addAll(marcRecord); - - if (bib.isStaffOnly()) { - // For Incremental Except Staff Only - oleNGBatchExportResponse.getDeletedBibIds().add(bib.getId()); - } - } catch (Exception e) { - e.printStackTrace(); - numberOfFailureRecord++; - oleNGBatchExportResponse.addFailureRecord(bib.getLocalId(), bib.getId(), e.getMessage()); - batchExportHandler.addBatchExportFailureResponseToExchange(e, bib.getId(), batchProcessTxObject.getExchangeObjectForBatchExport()); - } - } - preparedReportForUnSyncRecords(bibIds, oleNGBatchExportResponse); - oleNGBatchExportResponse.addNoOfSuccessRecords(numberOfSuccesssRecord); - oleNGBatchExportResponse.addNoOfFailureRecords(numberOfFailureRecord + bibIds.size()); - batchExportHandler.generateFileForMarcRecords(fileNumber, marcRecords, batchProcessTxObject); - return oleNGBatchExportResponse; - } - }); - } catch (Exception ex) { - throw ex; - } finally { - this.transactionManager = null; - - } - return oleNGBatchExportResponse; - } - - private void preparedReportForUnSyncRecords(List bibIds, OleNGBatchExportResponse oleNGBatchExportResponse) { - if(CollectionUtils.isNotEmpty(bibIds)) { - for (Iterator iterator = bibIds.iterator(); iterator.hasNext(); ) { - String bibId = iterator.next(); - String bibIdWithPrefix = DocumentUniqueIDPrefix.getPrefixedId(DocumentUniqueIDPrefix.PREFIX_WORK_BIB_MARC, bibId); - oleNGBatchExportResponse.addFailureRecord(bibId, bibIdWithPrefix, OleNGConstants.SOLR_RECORD_NOT_SYNC_WITH_DB_RECORD); - } - } - } - - private void processDataMappings(String bibId, Record marcRecord, List holdingsTreeList, BatchProcessProfile batchProcessProfile) { - for (HoldingsTree holdingsTree : holdingsTreeList) { - try { - if (holdingsTree.getHoldings().getHoldingsType().equalsIgnoreCase(OleNGConstants.PRINT)) { - marcRecord.getDataFields().addAll(new ExportHoldingsMappingHelper().generateDataFieldForHolding(holdingsTree, batchProcessProfile)); - } else { - marcRecord.getDataFields().addAll(new ExportEholdingsMappingHelper().generateDataFieldForEHolding(holdingsTree, batchProcessProfile)); - } - } catch (Exception e) { - e.printStackTrace(); - batchExportHandler.addBatchExportFailureResponseToExchange(e, bibId, batchProcessTxObject.getExchangeObjectForBatchExport()); - } - } - } - - private void processDataTransformations(String bibId, Record marcRecord, BatchProcessProfile batchProcessProfile, BatchExportHandler batchExportHandler) { - List dataTransformerList = batchProcessProfile.getBatchProfileDataTransformerList(); - for (BatchProfileDataTransformer dataTransformer : dataTransformerList) { - try { - if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.DELETE_FIELD)) { - batchExportHandler.getMarcRecordUtil().removeFieldFromRecord(marcRecord, dataTransformer.getDataField()); - } else if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.DELETE_SUBFIELD)) { - batchExportHandler.getMarcRecordUtil().removeSubField(marcRecord, dataTransformer.getDataField(), dataTransformer.getInd1(), dataTransformer.getInd2(), dataTransformer.getSubField()); - } else if (dataTransformer.getActionType().equalsIgnoreCase(OleNGConstants.CUT_PASTE)) { - batchExportHandler.getMarcRecordUtil().renameDataField(marcRecord, dataTransformer.getDataField(), dataTransformer.getDestDataField()); - } - } catch (Exception e) { - e.printStackTrace(); - batchExportHandler.addBatchExportFailureResponseToExchange(e, bibId, batchProcessTxObject.getExchangeObjectForBatchExport()); - } - } - } - - private Bib fetchBibRecord(SqlRowSet bibResultSet) throws SQLException { - Bib bib = new BibMarc(); - bib.setCreatedBy(bibResultSet.getString("CREATED_BY")); - bib.setCreatedOn(bibResultSet.getString("DATE_CREATED")); - if (bibResultSet.getString("STAFF_ONLY") != null) { - bib.setStaffOnly((bibResultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); - } - bib.setContent(bibResultSet.getString("CONTENT")); - bib.setUpdatedBy(bibResultSet.getString("UPDATED_BY")); - bib.setUpdatedOn(bibResultSet.getString("DATE_UPDATED")); - bib.setStatus(bibResultSet.getString("STATUS")); - bib.setStatusUpdatedBy(bibResultSet.getString("STATUS_UPDATED_BY")); - bib.setStatusUpdatedOn(bibResultSet.getString("STATUS_UPDATED_DATE")); - bib.setLastUpdated(bibResultSet.getString("DATE_UPDATED")); - String uuid = bibResultSet.getString("UNIQUE_ID_PREFIX") + "-" + bibResultSet.getString(1); - bib.setId(uuid); - bib.setLocalId(bibResultSet.getString(1)); - return bib; - } - - - private List fetchHoldingsTreeForBib(int bibId) throws Exception { - List holdingsTrees = new ArrayList(); - String holdingsQuery = getHoldingsQuery(bibId); - SqlRowSet resultSet = jdbcTemplate.queryForRowSet(holdingsQuery); - Map map = new HashMap<>(); - Map extentOfOwnershipMap = new HashMap<>(); - Set uriSet = null; - Set noteSet = null; - Set donorSet = null; - Set coverageSet = null; - Set perpetualSet = null; - Set extentOfOwnershipNoteSet = null; - Set linkSet = null; - - while (resultSet.next()) { - String holdingsType = resultSet.getString("HOLDINGS_TYPE"); - String id = resultSet.getString("HOLDINGS_ID"); - if (id == null) { - continue; - } - OleHoldings oleHoldings = null; - if (map.containsKey(id)) { - oleHoldings = map.get(id).getHoldings().getContentObject(); - } else { - oleHoldings = new OleHoldings(); - Location location = RebuildIndexUtil.getLocationDetails(resultSet.getString("LOCATION"), resultSet.getString("LOCATION_LEVEL")); - oleHoldings.setLocation(location); - oleHoldings.setHoldingsType(holdingsType); - CallNumber callNumber = new CallNumber(); - callNumber.setNumber(resultSet.getString("CALL_NUMBER")); - callNumber.setPrefix(resultSet.getString("CALL_NUMBER_PREFIX")); - ShelvingOrder shelvingOrder = new ShelvingOrder(); - if (resultSet.getString("SHELVING_ORDER") != null && !"null".equals(resultSet.getString("SHELVING_ORDER"))) { - shelvingOrder.setCodeValue(resultSet.getString("SHELVING_ORDER")); - shelvingOrder.setFullValue(resultSet.getString("SHELVING_ORDER")); - } - callNumber.setShelvingOrder(shelvingOrder); - ShelvingScheme shelvingScheme = new ShelvingScheme(); - if (resultSet.getString("CALL_NUMBER_TYPE_ID") != null) { - String[] strings = callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")).split("[|]"); - shelvingScheme.setCodeValue(strings[0]); - shelvingScheme.setFullValue(strings[1]); - } - callNumber.setShelvingScheme(shelvingScheme); - oleHoldings.setCallNumber(callNumber); - oleHoldings.setCopyNumber(resultSet.getString("COPY_NUMBER")); - HoldingsTree holdingsTree = new HoldingsTree(); - - Holdings holdings = null; - - if (holdingsType.equalsIgnoreCase(PHoldings.PRINT)) { - holdings = new PHoldings(); - - if (resultSet.getString("RECEIPT_STATUS_ID") != null) { - oleHoldings.setReceiptStatus(resultSet.getString("RECEIPT_STATUS_ID")); - } - extentOfOwnershipNoteSet = new HashSet<>(); - donorSet = coverageSet = perpetualSet = null; - } else { - holdings = new EHoldings(); - oleHoldings.setAccessStatus(resultSet.getString("ACCESS_STATUS")); - oleHoldings.setImprint(resultSet.getString("IMPRINT")); - Platform platform = new Platform(); - platform.setPlatformName(resultSet.getString("PLATFORM")); - platform.setAdminUrl(resultSet.getString("ADMIN_URL")); - platform.setAdminUserName(resultSet.getString("ADMIN_USERNAME")); - platform.setAdminPassword(resultSet.getString("ADMIN_PASSWORD")); - oleHoldings.setPlatform(platform); - - oleHoldings.setPublisher(resultSet.getString("PUBLISHER")); - HoldingsAccessInformation holdingsAccessInformation = new HoldingsAccessInformation(); - - holdingsAccessInformation.setProxiedResource(resultSet.getString("PROXIED_RESOURCE")); - holdingsAccessInformation.setAccessUsername(resultSet.getString("ACCESS_USERNAME")); - holdingsAccessInformation.setAccessPassword(resultSet.getString("ACCESS_PASSWORD")); - holdingsAccessInformation.setNumberOfSimultaneousUser(resultSet.getString("NUMBER_SIMULT_USERS")); - holdingsAccessInformation.setAccessLocation(resultSet.getString("CODE")); - holdingsAccessInformation.setAuthenticationType(resultSet.getString("AUTHENTICATION_TYPE_ID")); - oleHoldings.setHoldingsAccessInformation(holdingsAccessInformation); - String statisticalSearchId = resultSet.getString("STAT_SEARCH_CODE_ID"); - if (StringUtils.isNotEmpty(statisticalSearchId)) { - String[] strings = statisticalSearchCodeMap.get(statisticalSearchId).split("[|]"); - StatisticalSearchingCode statisticalSearchingCode = new StatisticalSearchingCode(); - statisticalSearchingCode.setCodeValue(strings[0]); - statisticalSearchingCode.setFullValue(strings[1]); - oleHoldings.setStatisticalSearchingCode(statisticalSearchingCode); - } - oleHoldings.setLocalPersistentLink(resultSet.getString("LOCAL_PERSISTENT_URI")); - oleHoldings.setSubscriptionStatus(resultSet.getString("SUBSCRIPTION_STATUS")); - oleHoldings.setInterLibraryLoanAllowed(Boolean.valueOf(resultSet.getString("ALLOW_ILL"))); - coverageSet = new HashSet<>(); - perpetualSet = new HashSet<>(); - donorSet = new HashSet<>(); - ExtentOfOwnership extentOfOwnership = new ExtentOfOwnership(); - Coverages coverages = new Coverages(); - PerpetualAccesses perpetualAccesses = new PerpetualAccesses(); - extentOfOwnership.setCoverages(coverages); - extentOfOwnership.setPerpetualAccesses(perpetualAccesses); - oleHoldings.getExtentOfOwnership().add(extentOfOwnership); - - } - holdings.setHoldingsType(holdingsType); - holdings.setId("who-" + id); - holdings.setContentObject(oleHoldings); - holdings.setCreatedBy(resultSet.getString("CREATED_BY")); - holdings.setCreatedOn(resultSet.getString("DATE_CREATED")); - if (resultSet.getString("STAFF_ONLY") != null) { - holdings.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); - } - holdings.setUpdatedBy(resultSet.getString("UPDATED_BY")); - holdings.setUpdatedOn(resultSet.getString("DATE_UPDATED")); - holdings.setLastUpdated(resultSet.getString("DATE_UPDATED")); - - uriSet = new HashSet<>(); - noteSet = new HashSet<>(); - linkSet = new HashSet<>(); - - holdingsTree.setHoldings(holdings); - if (StringUtils.isNotBlank(id)) { - List itemList = fetchItemForHoldings(Integer.parseInt(id)); - holdingsTree.getItems().addAll(itemList); - } - map.put(id, holdingsTree); - - holdingsTrees.add(holdingsTree); - } - - if (StringUtils.isNotEmpty(holdingsType) && holdingsType.equalsIgnoreCase(PHoldings.PRINT)) { - if (uriSet.add(resultSet.getString("HOLDINGS_URI_ID"))) { - Uri uri = new Uri(); - uri.setValue(resultSet.getString("TEXT")); - oleHoldings.getUri().add(uri); - } - ExtentOfOwnership extentOfOwnership = null; - if (extentOfOwnershipMap.containsKey(resultSet.getString("EXT_OWNERSHIP_ID"))) { - extentOfOwnership = extentOfOwnershipMap.get(resultSet.getString("EXT_OWNERSHIP_ID")); - } else { - extentOfOwnership = new ExtentOfOwnership(); - if (StringUtils.isNotEmpty(resultSet.getString("EXT_OWNERSHIP_TYPE_ID"))) { - String[] strings = extentOfOwnershipTypeMap.get(resultSet.getString("EXT_OWNERSHIP_TYPE_ID")).split("[|]"); - extentOfOwnership.setType(strings[1]); - } - extentOfOwnershipMap.put(resultSet.getString("EXT_OWNERSHIP_ID"), extentOfOwnership); - oleHoldings.getExtentOfOwnership().add(extentOfOwnership); - } - String extOwnershipNoteId = resultSet.getString("EXT_OWNERSHIP_NOTE_ID"); - if (extentOfOwnershipNoteSet != null && StringUtils.isNotEmpty(extOwnershipNoteId) && extentOfOwnershipNoteSet.add(resultSet.getString("EXT_OWNERSHIP_NOTE_ID"))) { - Note note = new Note(); - note.setValue((String) resultSet.getString("EXNOTE")); - note.setType((String) resultSet.getString("EXTYPE")); - - extentOfOwnership.getNote().add(note); - } - } else { - if (linkSet.add(resultSet.getString("HOLDINGS_URI_ID"))) { - Link link = new Link(); - link.setUrl(resultSet.getString("URI")); - link.setText(resultSet.getString("TEXT")); - oleHoldings.getLink().add(link); - } - if (oleHoldings.getExtentOfOwnership() != null && oleHoldings.getExtentOfOwnership().size() > 0) { - if (coverageSet != null && coverageSet.add(resultSet.getString("HOLDINGS_COVERAGE_ID"))) { - Coverage coverage = new Coverage(); - coverage.setCoverageStartIssue(resultSet.getString("COVERAGE_START_ISSUE")); - coverage.setCoverageStartDate(resultSet.getString("COVERAGE_START_DATE")); - coverage.setCoverageStartVolume(resultSet.getString("COVERAGE_START_VOLUME")); - coverage.setCoverageEndIssue(resultSet.getString("HOLDINGS_COVERAGE_ID")); - coverage.setCoverageEndDate(resultSet.getString("COVERAGE_END_DATE")); - coverage.setCoverageEndVolume(resultSet.getString("COVERAGE_END_VOLUME")); - oleHoldings.getExtentOfOwnership().get(0).getCoverages().getCoverage().add(coverage); - } - - if (perpetualSet != null && perpetualSet.add(resultSet.getString("HOLDINGS_PERPETUAL_ACCESS_ID"))) { - PerpetualAccess perpetualAccess = new PerpetualAccess(); - perpetualAccess.setPerpetualAccessStartDate(resultSet.getString("PERPETUAL_ACCESS_START_DATE")); - perpetualAccess.setPerpetualAccessStartIssue(resultSet.getString("PERPETUAL_ACCESS_START_ISSUE")); - perpetualAccess.setPerpetualAccessStartVolume(resultSet.getString("PERPETUAL_ACCESS_START_VOLUME")); - perpetualAccess.setPerpetualAccessEndDate(resultSet.getString("PERPETUAL_ACCESS_END_DATE")); - perpetualAccess.setPerpetualAccessEndVolume(resultSet.getString("PERPETUAL_ACCESS_END_VOLUME")); - perpetualAccess.setPerpetualAccessEndIssue(resultSet.getString("PERPETUAL_ACCESS_END_ISSUE")); - oleHoldings.getExtentOfOwnership().get(0).getPerpetualAccesses().getPerpetualAccess().add(perpetualAccess); - } - } - if (donorSet != null && donorSet.add(resultSet.getString("HOLDINGS_DONOR_ID"))) { - DonorInfo donorInfo = new DonorInfo(); - donorInfo.setDonorCode(resultSet.getString("DONOR_CODE")); - donorInfo.setDonorNote(resultSet.getString("DONOR_NOTE")); - donorInfo.setDonorPublicDisplay(resultSet.getString("DONOR_DISPLAY_NOTE")); - oleHoldings.getDonorInfo().add(donorInfo); - } - } - - if (noteSet.add(resultSet.getString("HOLDINGS_NOTE_ID"))) { - Note note = new Note(); - note.setValue(resultSet.getString("NOTE")); - note.setType(resultSet.getString("TYPE")); - oleHoldings.getNote().add(note); - } - } - - return holdingsTrees; + Set bibIdentifiers = getBatchExportUtil().getBibIdentifiersForQuery(query, start, chunkSize); + this.bibIdsString = StringUtils.join(bibIdentifiers, ','); + bibIds = new ArrayList<>(bibIdentifiers); + return processRecords(); } - public List fetchItemForHoldings(int holdingsId) throws Exception { - List itemList = new ArrayList(); - Map itemHashMap = new HashMap<>(); - String itemQuery = getItemQuery(holdingsId); - SqlRowSet resultSet = jdbcTemplate.queryForRowSet(itemQuery); - Set highDensityStorageSet = null; - Set donorNoteSet = null; - Set itemNoteSet = null; - Set statisticalSearchSet = null; - org.kuali.ole.docstore.common.document.content.instance.Item itemObj = null; - - while (resultSet.next()) { - String id = resultSet.getString("ITEM_ID"); - if (id == null) { - continue; - } - if (itemHashMap.containsKey(id)) { - itemObj = (org.kuali.ole.docstore.common.document.content.instance.Item) itemHashMap.get(id).getContentObject(); - } else { - itemObj = new org.kuali.ole.docstore.common.document.content.instance.Item(); - Item item = new Item(); - itemList.add(item); - item.setId("wio-" + resultSet.getString("ITEM_ID")); - item.setContentObject(itemObj); - Location location = RebuildIndexUtil.getLocationDetails(resultSet.getString("LOCATION"), resultSet.getString("LOCATION_LEVEL")); - itemObj.setLocation(location); - CallNumber callNumber = new CallNumber(); - callNumber.setNumber(resultSet.getString("CALL_NUMBER")); - callNumber.setPrefix(resultSet.getString("CALL_NUMBER_PREFIX")); - ShelvingOrder shelvingOrder = new ShelvingOrder(); - if (resultSet.getString("SHELVING_ORDER") != null && !"null".equals(resultSet.getString("SHELVING_ORDER"))) { - shelvingOrder.setCodeValue(resultSet.getString("SHELVING_ORDER")); - shelvingOrder.setFullValue(resultSet.getString("SHELVING_ORDER")); - } - callNumber.setShelvingOrder(shelvingOrder); - ShelvingScheme shelvingScheme = new ShelvingScheme(); - if (callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")) != null) { - String[] strings = callNumberType.get(resultSet.getString("CALL_NUMBER_TYPE_ID")).split("[|]"); - shelvingScheme.setCodeValue(strings[0]); - shelvingScheme.setFullValue(strings[1]); - } - callNumber.setShelvingScheme(shelvingScheme); - itemObj.setCallNumber(callNumber); - itemObj.setBarcodeARSL(resultSet.getString("BARCODE_ARSL")); - itemObj.setEnumeration(resultSet.getString("ENUMERATION")); - itemObj.setChronology(resultSet.getString("CHRONOLOGY")); - itemObj.setCopyNumber(resultSet.getString("COPY_NUMBER")); - AccessInformation accessInformation = new AccessInformation(); - accessInformation.setBarcode(resultSet.getString("BARCODE")); - Uri uri = new Uri(); - uri.setValue(resultSet.getString("URI")); - accessInformation.setUri(uri); - itemObj.setAccessInformation(accessInformation); - itemObj.setPurchaseOrderLineItemIdentifier(resultSet.getString("PURCHASE_ORDER_LINE_ITEM_ID")); - itemObj.setVendorLineItemIdentifier(resultSet.getString("VENDOR_LINE_ITEM_ID")); - itemObj.setFund(resultSet.getString("FUND")); - itemObj.setPrice(resultSet.getString("PRICE")); - itemObj.setItemStatusEffectiveDate(RebuildIndexUtil.convertDateFormat(resultSet.getString("ITEM_STATUS_DATE_UPDATED"))); - if (resultSet.getString("FAST_ADD") != null) { - itemObj.setFastAddFlag(resultSet.getString("FAST_ADD").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); - } - itemObj.setCheckinNote(resultSet.getString("CHECK_IN_NOTE")); - if (resultSet.getString("CLAIMS_RETURNED") != null) { - itemObj.setClaimsReturnedFlag(resultSet.getString("CLAIMS_RETURNED").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); - } - String claimsReturnFlagCreatedDate = resultSet.getString("CLAIMS_RETURNED_DATE_CREATED"); - itemObj.setClaimsReturnedFlagCreateDate(RebuildIndexUtil.convertDateFormat(claimsReturnFlagCreatedDate)); - itemObj.setClaimsReturnedNote(resultSet.getString("CLAIMS_RETURNED_NOTE")); - itemObj.setCurrentBorrower(resultSet.getString("CURRENT_BORROWER")); - itemObj.setProxyBorrower(resultSet.getString("PROXY_BORROWER")); - String dueDateTime = resultSet.getString("DUE_DATE_TIME"); - itemObj.setDueDateTime(RebuildIndexUtil.convertDateFormat(dueDateTime)); - String originalDueDate = resultSet.getString("ORG_DUE_DATE_TIME"); - itemObj.setOriginalDueDate(RebuildIndexUtil.convertDateFormat(originalDueDate)); - String checkOutDateTime = resultSet.getString("CHECK_OUT_DATE_TIME"); - itemObj.setCheckOutDateTime(RebuildIndexUtil.convertDateFormat(checkOutDateTime)); - itemObj.setDamagedItemNote(resultSet.getString("ITEM_DAMAGED_NOTE")); - if (resultSet.getString("ITEM_DAMAGED_STATUS") != null) { - itemObj.setItemDamagedStatus(resultSet.getString("ITEM_DAMAGED_STATUS").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); - } - if (resultSet.getString("MISSING_PIECES") != null) { - itemObj.setMissingPieceFlag(resultSet.getString("MISSING_PIECES").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE); - } - itemObj.setMissingPiecesCount(resultSet.getString("MISSING_PIECES_COUNT")); - itemObj.setMissingPieceFlagNote(resultSet.getString("MISSING_PIECES_NOTE")); - itemObj.setMissingPieceEffectiveDate(resultSet.getString("MISSING_PIECES_EFFECTIVE_DATE")); - itemObj.setNumberOfPieces(resultSet.getString("NUM_PIECES")); - itemObj.setDescriptionOfPieces(resultSet.getString("DESC_OF_PIECES")); - itemObj.setNumberOfRenew(resultSet.getInt("NUM_OF_RENEW")); - highDensityStorageSet = new HashSet<>(); - itemNoteSet = new HashSet<>(); - statisticalSearchSet = new HashSet<>(); - donorNoteSet = new HashSet<>(); - - ItemStatus itemStatus = new ItemStatus(); - if (itemStatusMap.containsKey(resultSet.getString("ITEM_STATUS_ID"))) { - String[] strings = itemStatusMap.get(resultSet.getString("ITEM_STATUS_ID")).split("[|]"); - itemStatus.setCodeValue(strings[0]); - itemStatus.setFullValue(strings[1]); - } - - itemObj.setItemStatus(itemStatus); - ItemType itemType = new ItemType(); - if (itemTypeMap.containsKey(resultSet.getString("ITEM_TYPE_ID"))) { - String[] strings = itemTypeMap.get(resultSet.getString("ITEM_TYPE_ID")).split("[|]"); - itemType.setCodeValue(strings[0]); - itemType.setFullValue(strings[1]); - } - itemObj.setItemType(itemType); - ItemType tempItemType = new ItemType(); - if (itemTypeMap.containsKey(resultSet.getString("TEMP_ITEM_TYPE_ID"))) { - String[] strings = itemTypeMap.get(resultSet.getString("TEMP_ITEM_TYPE_ID")).split("[|]"); - tempItemType.setCodeValue(strings[0]); - tempItemType.setFullValue(strings[1]); - } - itemObj.setTemporaryItemType(tempItemType); - item.setContentObject(itemObj); - item.setCreatedBy(resultSet.getString("CREATED_BY")); - item.setCreatedOn(resultSet.getString("DATE_CREATED")); - if (resultSet.getString("STAFF_ONLY") != null) { - item.setStaffOnly((resultSet.getString("STAFF_ONLY").equalsIgnoreCase("Y") ? Boolean.TRUE : Boolean.FALSE)); - } - item.setUpdatedBy(resultSet.getString("UPDATED_BY")); - item.setUpdatedOn(resultSet.getString("DATE_UPDATED")); - item.setLastUpdated(resultSet.getString("DATE_UPDATED")); - - } - if (itemNoteSet.add(resultSet.getString("ITEM_NOTE_ID"))) { - Note note = new Note(); - note.setValue(resultSet.getString("NOTE")); - note.setType(resultSet.getString("TYPE")); - itemObj.getNote().add(note); - } - if (resultSet.getString("STAT_SEARCH_CODE_ID") != null && statisticalSearchSet.add(resultSet.getString("STAT_SEARCH_CODE_ID"))) { - String[] strings = statisticalSearchCodeMap.get(resultSet.getString("STAT_SEARCH_CODE_ID")).split("[|]"); - StatisticalSearchingCode statisticalSearchingCode = new StatisticalSearchingCode(); - statisticalSearchingCode.setCodeValue(strings[0]); - statisticalSearchingCode.setFullValue(strings[1]); - itemObj.getStatisticalSearchingCode().add(statisticalSearchingCode); - } - if (donorNoteSet.add(resultSet.getString("ITEM_DONOR_ID"))) { - DonorInfo donorInfo = new DonorInfo(); - donorInfo.setDonorCode(resultSet.getString("DONOR_CODE")); - donorInfo.setDonorNote(resultSet.getString("DONOR_NOTE")); - donorInfo.setDonorPublicDisplay(resultSet.getString("DONOR_DISPLAY_NOTE")); - itemObj.getDonorInfo().add(donorInfo); - } - if (highDensityStorageSet.add(resultSet.getString("ITEM_DONOR_ID"))) { - HighDensityStorage highDensityStorage = new HighDensityStorage(); - highDensityStorage.setRow(resultSet.getString("HIGH_DENSITY_ROW")); - itemObj.setHighDensityStorage(highDensityStorage); - } - } - return itemList; - } - - private String getItemQuery(int holdingsId) { - return "SELECT I.*,N.ITEM_NOTE_ID,N.NOTE,N.TYPE, S.STAT_SEARCH_CODE_ID," + - "D.ITEM_DONOR_ID,D.DONOR_CODE,D.DONOR_DISPLAY_NOTE,D.DONOR_NOTE,HD.HIGH_DENSITY_ROW " + - "FROM ole_ds_item_t I " + - "LEFT JOIN ole_ds_item_donor_t D ON I.item_id=D.item_id " + - "LEFT JOIN ole_ds_item_note_t N ON I.item_id = N.item_id " + - "LEFT JOIN ole_ds_item_stat_search_t S ON I.item_id=S.item_id " + - "LEFT JOIN OLE_DS_HIGH_DENSITY_STORAGE_T HD ON I.HIGH_DENSITY_STORAGE_ID =HD.HIGH_DENSITY_STORAGE_ID " + - "WHERE I.HOLDINGS_ID= " + holdingsId; - } - - private String getHoldingsQuery(int bibId) { - return "SELECT H.* ,LOC.CODE,STAT.STAT_SEARCH_CODE_ID,URI.HOLDINGS_URI_ID,URI.TEXT,URI.URI,EXT.EXT_OWNERSHIP_ID,EXT.EXT_OWNERSHIP_TYPE_ID," + - "EXTN.EXT_OWNERSHIP_NOTE_ID,EXTN.NOTE AS EXNOTE,EXTN.TYPE AS EXTYPE,COV.HOLDINGS_COVERAGE_ID,COV.COVERAGE_START_ISSUE, COV.COVERAGE_START_DATE," + - "COV.COVERAGE_START_VOLUME,COV.COVERAGE_END_DATE,COV.COVERAGE_END_VOLUME,PA.HOLDINGS_PERPETUAL_ACCESS_ID,PA.PERPETUAL_ACCESS_START_DATE,PA.PERPETUAL_ACCESS_START_ISSUE," + - "PA.PERPETUAL_ACCESS_START_VOLUME,PA.PERPETUAL_ACCESS_END_DATE,PA.PERPETUAL_ACCESS_END_VOLUME,PA.PERPETUAL_ACCESS_END_ISSUE,DONOR.HOLDINGS_DONOR_ID, " + - "DONOR.DONOR_CODE,DONOR.DONOR_NOTE,DONOR.DONOR_DISPLAY_NOTE,NOTE.HOLDINGS_NOTE_ID,NOTE.NOTE,NOTE.TYPE " + - "FROM OLE_DS_HOLDINGS_T H " + - "LEFT JOIN ole_ds_holdings_uri_t URI ON H.HOLDINGS_ID=URI.HOLDINGS_ID " + - "LEFT JOIN ole_ds_holdings_note_t NOTE ON H.HOLDINGS_ID=NOTE.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_HOLDINGS_DONOR_T DONOR ON H.HOLDINGS_ID=DONOR.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_HOLDINGS_COVERAGE_T COV ON H.HOLDINGS_ID=COV.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_PERPETUAL_ACCESS_T PA ON H.HOLDINGS_ID=PA.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_HOLDINGS_STAT_SEARCH_T STAT ON H.HOLDINGS_ID=STAT.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_ACCESS_LOCATION_T ACC ON H.HOLDINGS_ID=ACC.HOLDINGS_ID " + - "LEFT JOIN OLE_DS_ACCESS_LOCATION_CODE_T LOC ON ACC.ACCESS_LOCATION_CODE_ID=LOC.ACCESS_LOCATION_CODE_ID " + - "LEFT JOIN ole_ds_ext_ownership_t ext ON H.HOLDINGS_ID=ext.holdings_id " + - "LEFT JOIN ole_ds_ext_ownership_note_t EXTN ON ext.EXT_OWNERSHIP_ID=EXTN.EXT_OWNERSHIP_ID WHERE H.BIB_ID= " + bibId; - } - public PlatformTransactionManager getTransactionManager() { - if (transactionManager == null) { - transactionManager = GlobalResourceLoader.getService("transactionManager"); + public BatchExportUtil getBatchExportUtil() { + if(null == batchExportUtil) { + batchExportUtil = new BatchExportUtil(); } - return this.transactionManager; + return batchExportUtil; } } diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/IncrementalExportCallableImpl.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/IncrementalExportCallableImpl.java new file mode 100644 index 0000000000..dcc047ca0f --- /dev/null +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/dao/export/IncrementalExportCallableImpl.java @@ -0,0 +1,30 @@ +package org.kuali.ole.oleng.dao.export; + +import org.apache.commons.lang.StringUtils; +import org.kuali.ole.oleng.batch.process.model.BatchProcessTxObject; +import org.kuali.ole.oleng.handler.BatchExportHandler; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Created by sheiks on 20/09/16. + */ +public class IncrementalExportCallableImpl extends ExportCallable { + + public IncrementalExportCallableImpl(Map> commomFields, JdbcTemplate jdbcTemplate, + List bibIds, int fileNumber, BatchExportHandler batchExportHandler, + BatchProcessTxObject batchProcessTxObject) { + super(commomFields, jdbcTemplate, fileNumber, batchExportHandler, batchProcessTxObject); + this.bibIds = new ArrayList<>(bibIds); + this.bibIdsString = StringUtils.join(bibIds, ','); + } + + @Override + public Object call() throws Exception { + return processRecords(); + } +} + diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/handler/BatchExportHandler.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/handler/BatchExportHandler.java index 6b3c1b31fb..82836c421c 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/handler/BatchExportHandler.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/handler/BatchExportHandler.java @@ -44,31 +44,31 @@ public void processExport(BatchProcessTxObject batchProcessTxObject, OleNGBatchE private void processFullExport(BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { String query = "(DocType:bibliographic)"; - exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse); + exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse, false); } private void processFullExceptStaffOnly(BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { String query = "(DocType:bibliographic)AND(staffOnlyFlag:false)"; - exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse); + exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse, false); } private void processIncremental(BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { Date lastExportDate = getLastExportDateForProfile(batchProcessTxObject); String query = getIncrementalSolrQuery(lastExportDate); - exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse); + exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse, true); processDeletedAndStaffOnlyBibs(lastExportDate, batchProcessTxObject); } private void processIncrementalExceptStaffOnly(BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { Date lastExportDateForProfile = getLastExportDateForProfile(batchProcessTxObject); String query = getIncrementalExceptStaffOnlySolrQuery(lastExportDateForProfile); - exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse); + exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse, true); processDeletedAndStaffOnlyBibs(lastExportDateForProfile, batchProcessTxObject); } private void processFilterExport(BatchProcessTxObject batchProcessTxObject, OleNGBatchExportResponse oleNGBatchExportResponse) { String query = getFilterSolrQuery(batchProcessTxObject, oleNGBatchExportResponse); - exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse); + exportDao.export(this, query, batchProcessTxObject, oleNGBatchExportResponse, false); } } diff --git a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/util/BatchExportUtil.java b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/util/BatchExportUtil.java index c09e85f07f..510f040e54 100644 --- a/ole-app/olefs/src/main/java/org/kuali/ole/oleng/util/BatchExportUtil.java +++ b/ole-app/olefs/src/main/java/org/kuali/ole/oleng/util/BatchExportUtil.java @@ -291,4 +291,20 @@ public void processDeletedAndStaffOnlyBibs(Date lastExportDate, BatchProcessTxOb generateFileForBibIds(deletedBibIds, batchProcessTxObject); } + public Set getBibIdentifiersForQuery(String query, int start, int chunkSize) { + Set bibIdentifiers = new HashSet<>(); + SolrDocumentList solrDocumentList = getSolrRequestReponseHandler().getSolrDocumentList(query, start, chunkSize, OleNGConstants.BIB_IDENTIFIER); + if (solrDocumentList.size() > 0) { + for (SolrDocument solrDocument : solrDocumentList) { + if (solrDocument.containsKey(OleNGConstants.BIB_IDENTIFIER)) { + List bibIds = (List) solrDocument.getFieldValue(OleNGConstants.BIB_IDENTIFIER); + for (String bibId : bibIds) { + bibIdentifiers.add(DocumentUniqueIDPrefix.getDocumentId(bibId)); + } + } + } + } + return bibIdentifiers; + } + } diff --git a/ole-common/ole-solr-client/src/main/java/org/kuali/incubator/SolrRequestReponseHandler.java b/ole-common/ole-solr-client/src/main/java/org/kuali/incubator/SolrRequestReponseHandler.java index 511a1c1c72..7b38b25711 100644 --- a/ole-common/ole-solr-client/src/main/java/org/kuali/incubator/SolrRequestReponseHandler.java +++ b/ole-common/ole-solr-client/src/main/java/org/kuali/incubator/SolrRequestReponseHandler.java @@ -115,7 +115,7 @@ public SolrDocumentList getSolrDocumentList(String queryString) { return sdl; } - public SolrDocumentList getSolrDocumentList(String queryString, int start, int rows, String fieldList) { + public SolrDocumentList getSolrDocumentList(String queryString, Integer start, Integer rows, String fieldList) { ArrayList> hitsOnPage = new ArrayList<>(); SolrDocumentList sdl = null; @@ -123,8 +123,12 @@ public SolrDocumentList getSolrDocumentList(String queryString, int start, int r SolrQuery query = new SolrQuery(); query.setQuery(queryString); - query.setStart(start); - query.setRows(rows); + if(null != start) { + query.setStart(start); + } + if(null != rows) { + query.setRows(rows); + } query.setFields(fieldList); query.setIncludeScore(true); diff --git a/ole-common/ole-solr-client/src/test/java/org/kuali/incubator/SolrRequestReponseHandlerTest.java b/ole-common/ole-solr-client/src/test/java/org/kuali/incubator/SolrRequestReponseHandlerTest.java new file mode 100644 index 0000000000..ee38b44da0 --- /dev/null +++ b/ole-common/ole-solr-client/src/test/java/org/kuali/incubator/SolrRequestReponseHandlerTest.java @@ -0,0 +1,28 @@ +package org.kuali.incubator; + +import org.apache.solr.common.SolrDocumentList; +import org.junit.Test; +import org.kuali.ole.constants.OleNGConstants; + +import static org.junit.Assert.*; + +/** + * Created by sheiks on 20/09/16. + */ +public class SolrRequestReponseHandlerTest { + + @Test + public void testTotalRecordCount() { + String query = "(dateUpdated:[2016-06-16T15:06:26Z TO NOW])AND(staffOnlyFlag:false)"; + SolrRequestReponseHandler solrRequestReponseHandler = new MockSolrRequestReponseHandler(); + SolrDocumentList solrDocumentList = solrRequestReponseHandler.getSolrDocumentList(query,null, null, OleNGConstants.BIB_IDENTIFIER); + assertNotNull(solrDocumentList); + } + + class MockSolrRequestReponseHandler extends SolrRequestReponseHandler { + @Override + public String getSolrUrl() { + return "http://localhost:8080/oledocstore/bib"; + } + } +} \ No newline at end of file